karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity
Files
f7f67afff9f97ec3dfa2deed7dcd2fe8e35b6622
searchi/fixtures/media/pipeline-10s-summary.json
T
karl b8400c4fbf
Core CI / core (ubuntu-latest) (push) Failing after 3m31s
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Successful in 5m39s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s
feat: add multimodal search and granular performance diagnostics
2026-10-12 19:16:23 +02:00

561 lines
16 KiB
JSON
Raw Blame History

{
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
"worker": {
"device": "cpu",
"dimensions": 768,
"media_supported": true,
"model": "google/embeddinggemma-2",
"ready": true,
"runtime": "b11461-no-logits-v1",
"threads": 1
},
"overall": {
"positive_queries": 58,
"unsupported_queries": 1,
"recall_at1": 0.9655172413793104,
"recall_at5": 1.0,
"mrr": 0.9827586206896551,
"temporal_queries": 22,
"segment_overlap_hit_at1": 0.8181818181818182,
"timestamp_point_hit_at1": 0.5909090909090909,
"segment_overlap_hit_at5": 0.8181818181818182,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 3.777000000000001,
"no_answer_queries": 6,
"no_answer_return_rate": 1.0
},
"by_category": {
"content-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"exact-filter": {
"positive_queries": 1,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"exact-metadata": {
"positive_queries": 3,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"filter-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"image-memory": {
"positive_queries": 24,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"metadata-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"music-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 0.5,
"recall_at5": 1.0,
"mrr": 0.75,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"sound-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"speech-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"temporal-negative": {
"positive_queries": 0,
"unsupported_queries": 1,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-action": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-scene": {
"positive_queries": 6,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 6,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-speech": {
"positive_queries": 8,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 8,
"segment_overlap_hit_at1": 0.75,
"timestamp_point_hit_at1": 0.125,
"segment_overlap_hit_at5": 0.75,
"timestamp_distance_median_s": 1.551,
"timestamp_distance_max_s": 3.777000000000001,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-temporal": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 0.5,
"timestamp_point_hit_at1": 0.5,
"segment_overlap_hit_at5": 0.5,
"timestamp_distance_median_s": 1.005,
"timestamp_distance_max_s": 1.005,
"no_answer_queries": 0,
"no_answer_return_rate": null
}
},
"by_language": {
"de": {
"positive_queries": 27,
"unsupported_queries": 0,
"recall_at1": 0.9629629629629629,
"recall_at5": 1.0,
"mrr": 0.9814814814814815,
"temporal_queries": 11,
"segment_overlap_hit_at1": 0.8181818181818182,
"timestamp_point_hit_at1": 0.6363636363636364,
"segment_overlap_hit_at5": 0.8181818181818182,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 1.551,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"en": {
"positive_queries": 31,
"unsupported_queries": 1,
"recall_at1": 0.967741935483871,
"recall_at5": 1.0,
"mrr": 0.9838709677419355,
"temporal_queries": 11,
"segment_overlap_hit_at1": 0.8181818181818182,
"timestamp_point_hit_at1": 0.5454545454545454,
"segment_overlap_hit_at5": 0.8181818181818182,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 3.777000000000001,
"no_answer_queries": 4,
"no_answer_return_rate": 1.0
}
},
"encoding_failures": [],
"encoding_ms": 671510.0948000001,
"query_p50_ms": 90.6467,
"query_p95_ms": 110.4152,
"failures": [
{
"query": "q029",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip03",
"end_s": 4.005,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 2005
},
"start_s": 2.005
},
"expected_intervals": {
"clip03": [
[
0,
1.0
]
]
},
"timestamp_distance_s": 1.005
},
{
"query": "q030",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip03",
"end_s": 4.005,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 2005
},
"start_s": 2.005
},
"expected_intervals": {
"clip03": [
[
0,
1.0
]
]
},
"timestamp_distance_s": 1.005
},
{
"query": "q039",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 10.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 0
},
"start_s": 0.0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q040",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 10.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 0
},
"start_s": 0.0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q041",
"category": "wrong-media-channel",
"candidate": {
"asset": "clip04",
"end_s": 26.167,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 24167
},
"start_s": 24.167
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": null
},
{
"query": "q042",
"category": "wrong-media-channel",
"candidate": {
"asset": "clip04",
"end_s": 14.467,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 12467
},
"start_s": 12.467
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": null
},
{
"query": "q043",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 50.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 40000
},
"start_s": 40.0
},
"expected_intervals": {
"clip04": [
[
43.777,
52.419
]
]
},
"timestamp_distance_s": 3.777000000000001
},
{
"query": "q045",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 66.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 56000
},
"start_s": 56.0
},
"expected_intervals": {
"clip04": [
[
56.539,
63.496
]
]
},
"timestamp_distance_s": 0.5390000000000015
},
{
"query": "q046",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 66.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 56000
},
"start_s": 56.0
},
"expected_intervals": {
"clip04": [
[
56.539,
63.496
]
]
},
"timestamp_distance_s": 0.5390000000000015
},
{
"query": "q049",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q050",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q059",
"category": "unsupported-query-capability",
"detail": "Production query API has no temporal-range filter yet"
},
{
"query": "q060",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.01639344262295082
},
{
"query": "q061",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.01639344262295082
},
{
"query": "q062",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.01639344262295082
},
{
"query": "q063",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.01639344262295082
},
{
"query": "q064",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.01639344262295082
},
{
"query": "q065",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.01639344262295082
}
],
"source_scope": "Actual Searchi hybrid pipeline and native extension filters; temporal-range filter unsupported; segment ends inferred from configured audio/frame policy, exact point timestamps come from actual result API",
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
}
Reference in new issue View Git Blame Copy Permalink