searchi/fixtures/media/reference-summary.json

{
  "fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
  "worker": {
    "device": "cpu",
    "dimensions": 768,
    "model": "google/embeddinggemma-2",
    "ready": true,
    "runtime": "b11461-no-logits-v1",
    "threads": 1
  },
  "overall": {
    "positive_queries": 55,
    "unsupported_queries": 4,
    "recall_at1": 0.9636363636363636,
    "recall_at5": 0.9636363636363636,
    "mrr": 0.9636363636363636,
    "temporal_queries": 22,
    "segment_overlap_hit_at1": 0.9545454545454546,
    "timestamp_point_hit_at1": 0.6818181818181818,
    "segment_overlap_hit_at5": 1.0,
    "timestamp_distance_median_s": 0,
    "timestamp_distance_max_s": 13.777000000000001,
    "no_answer_queries": 6,
    "no_answer_return_rate": 1.0
  },
  "by_category": {
    "content-no-answer": {
      "positive_queries": 0,
      "unsupported_queries": 0,
      "recall_at1": null,
      "recall_at5": null,
      "mrr": null,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 2,
      "no_answer_return_rate": 1.0
    },
    "exact-filter": {
      "positive_queries": 1,
      "unsupported_queries": 0,
      "recall_at1": 1.0,
      "recall_at5": 1.0,
      "mrr": 1.0,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    },
    "exact-metadata": {
      "positive_queries": 0,
      "unsupported_queries": 3,
      "recall_at1": null,
      "recall_at5": null,
      "mrr": null,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    },
    "filter-no-answer": {
      "positive_queries": 0,
      "unsupported_queries": 0,
      "recall_at1": null,
      "recall_at5": null,
      "mrr": null,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 1,
      "no_answer_return_rate": 1.0
    },
    "image-memory": {
      "positive_queries": 24,
      "unsupported_queries": 0,
      "recall_at1": 1.0,
      "recall_at5": 1.0,
      "mrr": 1.0,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    },
    "metadata-no-answer": {
      "positive_queries": 0,
      "unsupported_queries": 1,
      "recall_at1": null,
      "recall_at5": null,
      "mrr": null,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    },
    "music-memory": {
      "positive_queries": 4,
      "unsupported_queries": 0,
      "recall_at1": 0.5,
      "recall_at5": 0.5,
      "mrr": 0.5,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    },
    "sound-memory": {
      "positive_queries": 4,
      "unsupported_queries": 0,
      "recall_at1": 1.0,
      "recall_at5": 1.0,
      "mrr": 1.0,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    },
    "speech-no-answer": {
      "positive_queries": 0,
      "unsupported_queries": 0,
      "recall_at1": null,
      "recall_at5": null,
      "mrr": null,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 2,
      "no_answer_return_rate": 1.0
    },
    "temporal-negative": {
      "positive_queries": 0,
      "unsupported_queries": 0,
      "recall_at1": null,
      "recall_at5": null,
      "mrr": null,
      "temporal_queries": 0,
      "segment_overlap_hit_at1": null,
      "timestamp_point_hit_at1": null,
      "segment_overlap_hit_at5": null,
      "timestamp_distance_median_s": null,
      "timestamp_distance_max_s": null,
      "no_answer_queries": 1,
      "no_answer_return_rate": 1.0
    },
    "video-action": {
      "positive_queries": 4,
      "unsupported_queries": 0,
      "recall_at1": 1.0,
      "recall_at5": 1.0,
      "mrr": 1.0,
      "temporal_queries": 4,
      "segment_overlap_hit_at1": 1.0,
      "timestamp_point_hit_at1": 1.0,
      "segment_overlap_hit_at5": 1.0,
      "timestamp_distance_median_s": 0,
      "timestamp_distance_max_s": 0,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    },
    "video-scene": {
      "positive_queries": 6,
      "unsupported_queries": 0,
      "recall_at1": 1.0,
      "recall_at5": 1.0,
      "mrr": 1.0,
      "temporal_queries": 6,
      "segment_overlap_hit_at1": 1.0,
      "timestamp_point_hit_at1": 1.0,
      "segment_overlap_hit_at5": 1.0,
      "timestamp_distance_median_s": 0,
      "timestamp_distance_max_s": 0,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    },
    "video-speech": {
      "positive_queries": 8,
      "unsupported_queries": 0,
      "recall_at1": 1.0,
      "recall_at5": 1.0,
      "mrr": 1.0,
      "temporal_queries": 8,
      "segment_overlap_hit_at1": 1.0,
      "timestamp_point_hit_at1": 0.25,
      "segment_overlap_hit_at5": 1.0,
      "timestamp_distance_median_s": 13.129,
      "timestamp_distance_max_s": 13.777000000000001,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    },
    "video-temporal": {
      "positive_queries": 4,
      "unsupported_queries": 0,
      "recall_at1": 1.0,
      "recall_at5": 1.0,
      "mrr": 1.0,
      "temporal_queries": 4,
      "segment_overlap_hit_at1": 0.75,
      "timestamp_point_hit_at1": 0.75,
      "segment_overlap_hit_at5": 1.0,
      "timestamp_distance_median_s": 0,
      "timestamp_distance_max_s": 1.0,
      "no_answer_queries": 0,
      "no_answer_return_rate": null
    }
  },
  "by_language": {
    "de": {
      "positive_queries": 27,
      "unsupported_queries": 0,
      "recall_at1": 0.9629629629629629,
      "recall_at5": 0.9629629629629629,
      "mrr": 0.9629629629629629,
      "temporal_queries": 11,
      "segment_overlap_hit_at1": 0.9090909090909091,
      "timestamp_point_hit_at1": 0.6363636363636364,
      "segment_overlap_hit_at5": 1.0,
      "timestamp_distance_median_s": 0,
      "timestamp_distance_max_s": 13.777000000000001,
      "no_answer_queries": 2,
      "no_answer_return_rate": 1.0
    },
    "en": {
      "positive_queries": 28,
      "unsupported_queries": 4,
      "recall_at1": 0.9642857142857143,
      "recall_at5": 0.9642857142857143,
      "mrr": 0.9642857142857143,
      "temporal_queries": 11,
      "segment_overlap_hit_at1": 1.0,
      "timestamp_point_hit_at1": 0.7272727272727273,
      "segment_overlap_hit_at5": 1.0,
      "timestamp_distance_median_s": 0,
      "timestamp_distance_max_s": 13.777000000000001,
      "no_answer_queries": 4,
      "no_answer_return_rate": 1.0
    }
  },
  "encoding_failures": [
    {
      "candidate": {
        "asset": "music02",
        "end_s": 30,
        "modality": "audio",
        "path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\music02-audio-0000.f32",
        "start_s": 0
      },
      "error": "Invalid normalized audio PCM"
    }
  ],
  "encoding_ms": 498657.0532,
  "query_p50_ms": 92.78150000000001,
  "query_p95_ms": 122.9277,
  "failures": [
    {
      "query": "q030",
      "category": "timestamp-localization-miss",
      "candidate": {
        "asset": "clip03",
        "end_s": 4,
        "modality": "image",
        "path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip03-frame-0002.png",
        "start_s": 2,
        "token_budget": 70
      },
      "expected_intervals": {
        "clip03": [
          [
            0,
            1.0
          ]
        ]
      },
      "timestamp_distance_s": 1.0
    },
    {
      "query": "q039",
      "category": "timestamp-localization-miss",
      "candidate": {
        "asset": "clip04",
        "end_s": 30,
        "modality": "audio",
        "path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
        "start_s": 0
      },
      "expected_intervals": {
        "clip04": [
          [
            1.551,
            3.603
          ]
        ]
      },
      "timestamp_distance_s": 1.551
    },
    {
      "query": "q040",
      "category": "timestamp-localization-miss",
      "candidate": {
        "asset": "clip04",
        "end_s": 30,
        "modality": "audio",
        "path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
        "start_s": 0
      },
      "expected_intervals": {
        "clip04": [
          [
            1.551,
            3.603
          ]
        ]
      },
      "timestamp_distance_s": 1.551
    },
    {
      "query": "q041",
      "category": "timestamp-localization-miss",
      "candidate": {
        "asset": "clip04",
        "end_s": 30,
        "modality": "audio",
        "path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
        "start_s": 0
      },
      "expected_intervals": {
        "clip04": [
          [
            13.129,
            16.316
          ]
        ]
      },
      "timestamp_distance_s": 13.129
    },
    {
      "query": "q042",
      "category": "timestamp-localization-miss",
      "candidate": {
        "asset": "clip04",
        "end_s": 30,
        "modality": "audio",
        "path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
        "start_s": 0
      },
      "expected_intervals": {
        "clip04": [
          [
            13.129,
            16.316
          ]
        ]
      },
      "timestamp_distance_s": 13.129
    },
    {
      "query": "q043",
      "category": "timestamp-localization-miss",
      "candidate": {
        "asset": "clip04",
        "end_s": 60,
        "modality": "audio",
        "path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0030.f32",
        "start_s": 30
      },
      "expected_intervals": {
        "clip04": [
          [
            43.777,
            52.419
          ]
        ]
      },
      "timestamp_distance_s": 13.777000000000001
    },
    {
      "query": "q044",
      "category": "timestamp-localization-miss",
      "candidate": {
        "asset": "clip04",
        "end_s": 60,
        "modality": "audio",
        "path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0030.f32",
        "start_s": 30
      },
      "expected_intervals": {
        "clip04": [
          [
            43.777,
            52.419
          ]
        ]
      },
      "timestamp_distance_s": 13.777000000000001
    },
    {
      "query": "q049",
      "category": "hard-negative-confusion",
      "winner": "music01",
      "expected": [
        "music02"
      ]
    },
    {
      "query": "q050",
      "category": "hard-negative-confusion",
      "winner": "music01",
      "expected": [
        "music02"
      ]
    },
    {
      "query": "q055",
      "category": "unsupported-query-capability",
      "detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
    },
    {
      "query": "q056",
      "category": "unsupported-query-capability",
      "detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
    },
    {
      "query": "q057",
      "category": "unsupported-query-capability",
      "detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
    },
    {
      "query": "q059",
      "category": "no-answer-nearest-neighbor-return",
      "winner": "clip01",
      "score": 0.750250822387441
    },
    {
      "query": "q060",
      "category": "no-answer-nearest-neighbor-return",
      "winner": "music01",
      "score": 0.5716409940327322
    },
    {
      "query": "q061",
      "category": "no-answer-nearest-neighbor-return",
      "winner": "music01",
      "score": 0.6223961635527826
    },
    {
      "query": "q062",
      "category": "no-answer-nearest-neighbor-return",
      "winner": "music01",
      "score": 0.6181057666050869
    },
    {
      "query": "q063",
      "category": "no-answer-nearest-neighbor-return",
      "winner": "clip04",
      "score": 0.6229413326862353
    },
    {
      "query": "q064",
      "category": "no-answer-nearest-neighbor-return",
      "winner": "clip03",
      "score": 0.5715326739972073
    },
    {
      "query": "q065",
      "category": "unsupported-query-capability",
      "detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
    }
  ],
  "source_scope": "Direct real-model dense reference baseline; candidate filters here are benchmark reference constraints, not production-filter certification; no rejection threshold is tuned",
  "interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
}