{ "version": 1, "label_policy": "Frozen before any mixed-model run. One annotator, source implementation inspections plus reused independently frozen media labels. Previously observed media outcomes mean this is a new mixed diagnostic, not an untouched held-out media benchmark. Never change v1 labels after mixed rankings.", "assets": [ { "id": "source01", "path": "sources/crates__searchi-service__src__coordination.rs", "kind": "code", "source_path": "crates/searchi-service/src/coordination.rs", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "481b4a33fdba844bd0677dc85a273eb21d10c9fd0667086215dd206972aef827", "bytes": 15233, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "source02", "path": "sources/crates__searchi-service__src__routing.rs", "kind": "code", "source_path": "crates/searchi-service/src/routing.rs", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "613299748f08248f6f9be694f19698434029aa8f4440fbf1a5618730314abbd6", "bytes": 6647, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "source03", "path": "sources/crates__searchi-core__src__extraction.rs", "kind": "code", "source_path": "crates/searchi-core/src/extraction.rs", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "269ba55dd648d2ad396488a30ace3d670b149969506f04387fd89b0b4ed1372a", "bytes": 10899, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "source04", "path": "sources/apps__desktop__src__selection-state.ts", "kind": "code", "source_path": "apps/desktop/src/selection-state.ts", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "9793541bc5da0ad1fd27e74ab9e5ad646242a717cebe0fb0e058a23d1959befd", "bytes": 705, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "source05", "path": "sources/apps__desktop__src__search-performance.ts", "kind": "code", "source_path": "apps/desktop/src/search-performance.ts", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "9e07ef01da623b34d3dc9f36f68488f19422b425bde3f98f9031a4dc67646fe1", "bytes": 7198, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "source06", "path": "sources/crates__searchi-service__src__store__incremental.rs", "kind": "code", "source_path": "crates/searchi-service/src/store/incremental.rs", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "597b8c30f7f86be13f36b16f8e2db992ee56cb6591532fb1b9e3be24dc7e60f0", "bytes": 45030, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "source07", "path": "sources/docs__PERFORMANCE.md", "kind": "document", "source_path": "docs/PERFORMANCE.md", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "a5147832daf8c9c1d51e297b6498e4627e069ab9de58862c9baa3e51b8566321", "bytes": 12757, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "source08", "path": "sources/docs__INCREMENTAL-SEMANTIC.md", "kind": "document", "source_path": "docs/INCREMENTAL-SEMANTIC.md", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "0291ab5cabce5ef0cb503dadc39696b6bb9198d17247673a24ce6c5d3f60cb9a", "bytes": 6883, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "source09", "path": "sources/docs__AUTOMATIC-INDEXING.md", "kind": "document", "source_path": "docs/AUTOMATIC-INDEXING.md", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "9de756e96c04a9d82915c78d851c928b375a9b6c044f672e6d23a487b0e15fa8", "bytes": 6106, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "source10", "path": "sources/docs__CODE-SEARCH.md", "kind": "document", "source_path": "docs/CODE-SEARCH.md", "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", "sha256": "de3f230d33403ecc8627b18c048a0ca406dbec25c969b40c857f9fddcb79b3e9", "bytes": 4216, "license": "MIT; repository LICENSE", "changes": "Exact checked-out bytes, no edits" }, { "id": "image01", "path": "../media/assets/image01.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/126407/pexels-photo-126407.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/126407/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Long-haired tabby cat lying flat indoors, eyes toward camera.", "independent_inspection": "view_image before any media embedding inference", "bytes": 27390, "sha256": "c0b2ebddafcff4cd60ae1d2f1b8e53d4e0f11643ac53d0113605f333523b21ed" }, { "id": "image02", "path": "../media/assets/image02.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/1099680/pexels-photo-1099680.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/1099680/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Fruit bowl with raspberries, mango, blackberries, strawberries and almonds, top down.", "independent_inspection": "view_image before any media embedding inference", "bytes": 68905, "sha256": "72b46006e7f55dcad5bec5af977171a369917030cf2ff99e544806af939ffec1" }, { "id": "image03", "path": "../media/assets/image03.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/459225/pexels-photo-459225.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/459225/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Jagged mountains and fir trees reflected in a daylight lake.", "independent_inspection": "view_image before any media embedding inference", "bytes": 34309, "sha256": "c4abbb061e6fcbcef25d36d7bb7f24018c0c768b23176fb2a77a9eb8f5b86f21" }, { "id": "image04", "path": "../media/assets/image04.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/355465/pexels-photo-355465.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/355465/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Dense blue star field and Milky Way above a dark mountain landscape.", "independent_inspection": "view_image before any media embedding inference", "bytes": 37735, "sha256": "8999f22131c22ee1c29d79dda352c2654131eb1b4f61ba7f8b8d21dbbc5cc791" }, { "id": "image05", "path": "../media/assets/image05.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/414612/pexels-photo-414612.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/414612/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Wooden pier into still lake at night under a bright moon.", "independent_inspection": "view_image before any media embedding inference", "bytes": 51106, "sha256": "1cfd9d9c3dea4ec4e1733c55424eef93983bce1167f60777768cae6c549fd889" }, { "id": "image06", "path": "../media/assets/image06.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/1323550/pexels-photo-1323550.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/1323550/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Layered purple mountains under a yellow-orange sunset, tiny person on ridge.", "independent_inspection": "view_image before any media embedding inference", "bytes": 27259, "sha256": "91947e87067c923db2e560f67cdd0c1edec55c9bb0fbab6a83ee7c7adf88860d" }, { "id": "image07", "path": "../media/assets/image07.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/841303/pexels-photo-841303.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/841303/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Cows grazing in green rolling hills.", "independent_inspection": "view_image before any media embedding inference", "bytes": 50633, "sha256": "0b0cc8c211680d19ebdaf2b797af430120c4e65f951143bbe06544eaa0691942" }, { "id": "image08", "path": "../media/assets/image08.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/210019/pexels-photo-210019.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/210019/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Red sports coupe on a wooded road, motion-blurred background.", "independent_inspection": "view_image before any media embedding inference", "bytes": 65362, "sha256": "12b5fd662c054d1337e4adf733751da3e32c1fd40f88dfccc0bd73af4f620369" }, { "id": "image09", "path": "../media/assets/image09.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/1640777/pexels-photo-1640777.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/1640777/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Bowl of quinoa and vegetables, sliced carrots and zucchini on pale table.", "independent_inspection": "view_image before any media embedding inference", "bytes": 68469, "sha256": "6112af5f9ed170d91c7f8e1e5de95142c0190e2e2f8fbbbccf2f106553371817" }, { "id": "image10", "path": "../media/assets/image10.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/302899/pexels-photo-302899.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/302899/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Milk pouring from metal jug into cup of coffee with latte art.", "independent_inspection": "view_image before any media embedding inference", "bytes": 23609, "sha256": "632926d3bd0f4691ae7fd0a8e2cb8b455e0252511eebf322e91c64c17cdf0e2a" }, { "id": "image11", "path": "../media/assets/image11.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/248797/pexels-photo-248797.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/248797/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Turquoise tropical sea and white beach fringed by green vegetation.", "independent_inspection": "view_image before any media embedding inference", "bytes": 27716, "sha256": "c02ad4a8bbe8091cbc3a9a0abc602a905cb46056efa94141950f3af1c749c038" }, { "id": "image12", "path": "../media/assets/image12.jpg", "kind": "image", "source_url": "https://images.pexels.com/photos/167964/pexels-photo-167964.jpeg?auto=compress&cs=tinysrgb&w=640", "source_page": "https://www.pexels.com/photo/167964/", "license": "Pexels License", "license_url": "https://www.pexels.com/license/", "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", "observation": "Black and white silhouette of a person reaching a hand toward camera.", "independent_inspection": "view_image before any media embedding inference", "bytes": 14256, "sha256": "dd3bc409128a8702522bac7b953a5eb691662eebf21ac3ba0b3b15c70f86d54b" }, { "id": "clip01", "path": "../media/assets/clip01.webm", "kind": "video", "duration_s": 22.204, "source_url": "https://upload.wikimedia.org/wikipedia/commons/c/cc/Cat_opening_door.webm", "source_page": "https://commons.wikimedia.org/wiki/File:Cat_opening_door.webm", "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "author": "Astropulse", "changes": "None; downloaded original file.", "observation": "Orange tabby cat twice jumps against white door near its lever, then crouches.", "independent_inspection": "Real frames viewed before inference; NASA speech labels use publisher SRT, never model outputs.", "bytes": 2253125, "sha256": "1a1f420559b306574ba8a04aee0175c38c2e20815b1bd46a45aae86d17f3ad53" }, { "id": "clip02", "path": "../media/assets/clip02.webm", "kind": "video", "duration_s": 13.81, "source_url": "https://upload.wikimedia.org/wikipedia/commons/c/c9/Water_flowing_strem.webm", "source_page": "https://commons.wikimedia.org/wiki/File:Water_flowing_strem.webm", "license": "CC0-1.0", "license_url": "https://creativecommons.org/publicdomain/zero/1.0/", "author": "BhotaRohit", "changes": "None; downloaded original file.", "observation": "Tall waterfall descending misty rocky hillside; a tiny person appears near its base.", "independent_inspection": "Real frames viewed before inference; NASA speech labels use publisher SRT, never model outputs.", "bytes": 6001205, "sha256": "5652e6490bea85dfb7a1da05e00fc7f3d8cc2cbd54790c03a48223f2256bc3d1" }, { "id": "clip03", "path": "../media/assets/clip03.webm", "kind": "video", "duration_s": 5.056, "source_url": "https://interactive-examples.mdn.mozilla.net/media/cc0-videos/flower.webm", "source_page": "https://github.com/mdn/interactive-examples", "license": "CC0-1.0", "license_url": "https://raw.githubusercontent.com/mdn/interactive-examples/main/LICENSE", "author": "MDN contributors", "changes": "None; downloaded original file.", "observation": "Timelapse of closed red flower bud opening among green purple-edged leaves.", "independent_inspection": "Real frames viewed before inference; NASA speech labels use publisher SRT, never model outputs.", "bytes": 554058, "sha256": "c6f8a348953395598a9a73b9bab1676436410797bce9f398f4be1531d6e76dda" }, { "id": "clip04", "path": "../media/assets/clip04.mp4", "kind": "video", "duration_s": 80.63, "source_url": "https://images-assets.nasa.gov/video/Artemis%20Origins/Artemis%20Origins~small.mp4", "source_page": "https://images.nasa.gov/details/Artemis%20Origins", "license": "NASA informational media permission", "license_url": "https://www.nasa.gov/nasa-brand-center/images-and-media/", "author": "NASA", "changes": "None; downloaded original file.", "observation": "Artemis Origins montage: Apollo astronauts, lunar surface/rover, a speech and future-space imagery. Speech intervals independently labelled from NASA publisher SRT.", "independent_inspection": "Real frames viewed before inference; NASA speech labels use publisher SRT, never model outputs.", "bytes": 16710310, "sha256": "5fa4af30f984f4d7d53b50990d1e21e41df50b62a8b3ec3a2a1d44f30431f0f1" }, { "id": "music01", "path": "../media/assets/music01.ogg", "kind": "music", "source_url": "https://upload.wikimedia.org/wikipedia/commons/4/49/Kimiko_Ishizaka_-_14_-_Variatio_13_a_2_Clav.ogg", "source_page": "https://commons.wikimedia.org/wiki/File:Kimiko_Ishizaka_-_14_-_Variatio_13_a_2_Clav.ogg", "license": "CC0-1.0", "license_url": "https://creativecommons.org/publicdomain/zero/1.0/", "author": "Kimiko Ishizaka", "changes": "None; original waveform and embedded metadata retained.", "observation": "Soft solo piano performance of Bach Goldberg Variation13, original Vorbis artist/title tags retained.", "duration_s": 254.453333, "tags": { "encoder": "Lavc58.54.100 libvorbis", "title": "Variatio 13 a 2 Clav.", "artist": "Kimiko Ishizaka", "track": "14", "album": "The Open Goldberg Variations", "disc": "1", "date": "2012-05-29T17:49:26", "genre": "Classical", "copyright": "CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/", "tdat": "2012-05-29 17:49:26", "comment": "URL: http://freemusicarchive.org/music/Kimiko_Ishizaka/The_Open_Goldberg_Variations/KIMIKO_ISHIZAKA_-_Goldberg_Variations_BWV_988_-_14_-_Variatio_13_a_2_Clav__44k-24b\r\nComments: http://freemusicarchive.org/\r\nCurator: ccCommunity\r\nCopyright: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/\r\nComposer: J.S. Bach", "tlen": "254453", "tsrc": "QMNYZ1200014", "composer": "J.S. Bach", "album_artist": "Kimiko Ishizaka" }, "independent_inspection": "Publisher descriptions and original FFprobe tags checked before inference. Audio perception not claimed as human listening.", "bytes": 4697594, "sha256": "29fcd20414ee19c2a5aa4339eaf6375079dc87959051ab0aea32c76df52f34dc" }, { "id": "music02", "path": "../media/assets/music02.ogg", "kind": "music", "source_url": "https://upload.wikimedia.org/wikipedia/commons/3/37/00_Jazz_Violin_Solo.ogg", "source_page": "https://commons.wikimedia.org/wiki/File:00_Jazz_Violin_Solo.ogg", "license": "PD-self", "license_url": "https://commons.wikimedia.org/wiki/File:00_Jazz_Violin_Solo.ogg", "author": "VWD", "changes": "None; original waveform and embedded metadata retained.", "observation": "Jazz violin solo, original Vorbis tags identify artist Violinist WTD and title Jazz Violin Solo.", "duration_s": 30.0, "tags": { "album": "2007 Spring Concert Series", "artist": "Violinist WTD", "composer": "Violinist WTD/LPC Jazz Band", "date": "2007", "encoder": "Max 0.7.1", "encoding": "Vorbis settings: VBR(q=3.000000)", "genre": "Jazz", "title": "Jazz Violin Solo" }, "independent_inspection": "Publisher descriptions and original FFprobe tags checked before inference. Audio perception not claimed as human listening.", "bytes": 198070, "sha256": "9ce9c5d07f941c07cab575e58016d6ad1b09219a36477652118d8e85c3401dd8" }, { "id": "sound01", "path": "../media/assets/sound01.wav", "kind": "sound", "source_url": "https://opengameart.org/sites/default/files/dog_barking_mono.wav", "source_page": "https://opengameart.org/content/dog-barking-mono", "license": "CC0-1.0", "license_url": "https://creativecommons.org/publicdomain/zero/1.0/", "author": "Brandon Morris (uploaded by HaelDB)", "changes": "None; original waveform and embedded metadata retained.", "observation": "Dog bark recording from independently described source; real waveform retained.", "duration_s": 2.038594, "tags": {}, "independent_inspection": "Publisher descriptions and original FFprobe tags checked before inference. Audio perception not claimed as human listening.", "bytes": 179848, "sha256": "bbd0f908b3514dd3bd7d2bc04dcf64f8d360a161e7f43cac5d6761e7add79451" }, { "id": "sound02", "path": "../media/assets/sound02.mp3", "kind": "sound", "source_url": "https://interactive-examples.mdn.mozilla.net/media/cc0-audio/t-rex-roar.mp3", "source_page": "https://github.com/mdn/interactive-examples", "license": "CC0-1.0", "license_url": "https://creativecommons.org/publicdomain/zero/1.0/", "author": "MDN contributors", "changes": "None; original waveform and embedded metadata retained.", "observation": "Roaring creature sound effect from independently described CC0 source; real waveform retained.", "duration_s": 2.074218, "tags": { "encoder": "LAME3.98r" }, "independent_inspection": "Publisher descriptions and original FFprobe tags checked before inference. Audio perception not claimed as human listening.", "bytes": 39868, "sha256": "41191d0727073bf848bcc8f0bd851d71a0b0058e901abb1c1b236ad327bda52e" }, { "id": "llama-unicode", "path": "sources/llama__unicode.cpp", "kind": "code", "source_path": "src/unicode.cpp", "source_repository": "https://github.com/ggml-org/llama.cpp", "source_commit": "4d756bc72bf00a4aacf410ae15a2d315f3db400d", "sha256": "2818ca5657cff7b2b4f50bc976bb9bdc4e75e92fd595a5a75555b22946e149da", "bytes": 61285, "license": "MIT; fixtures/mixed/LLAMA-LICENSE.txt", "changes": "Exact committed upstream bytes retrieved with git show; local runtime patches excluded" }, { "id": "llama-allocator", "path": "sources/llama__ggml-alloc.c", "kind": "code", "source_path": "ggml/src/ggml-alloc.c", "source_repository": "https://github.com/ggml-org/llama.cpp", "source_commit": "4d756bc72bf00a4aacf410ae15a2d315f3db400d", "sha256": "9664bda3e9c4223f5c94ebfeb3c0381f0065a5091ddae707c09a800f7e419a39", "bytes": 45068, "license": "MIT; fixtures/mixed/LLAMA-LICENSE.txt", "changes": "Exact committed upstream bytes retrieved with git show; local runtime patches excluded" } ], "queries": [ { "id": "code-budget-en", "text": "share a CPU thread budget between foreground search and background indexing", "language": "en", "category": "code", "expected": [ "source01" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "code-budget-de", "text": "CPU-Threads zwischen Suche und Hintergrundindexierung aufteilen", "language": "de", "category": "code", "expected": [ "source01" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "code-audio-en", "text": "code that chooses audio search when a query mentions speech", "language": "en", "category": "code", "expected": [ "source02" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "code-audio-de", "text": "Code der Audiosuche auswählt wenn die Anfrage Sprache erwähnt", "language": "de", "category": "code", "expected": [ "source02" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "code-extract-en", "text": "read a document once and hash the exact bytes consumed by the parser", "language": "en", "category": "code", "expected": [ "source03" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "code-extract-de", "text": "Dokument einmal lesen und die vom Parser verwendeten Bytes hashen", "language": "de", "category": "code", "expected": [ "source03" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "code-selection-en", "text": "restore a selected result after it temporarily disappears from search results", "language": "en", "category": "code", "expected": [ "source04" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "code-selection-de", "text": "Ausgewähltes Ergebnis wiederherstellen nachdem es kurz aus Suchergebnissen verschwindet", "language": "de", "category": "code", "expected": [ "source04" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "code-timing-en", "text": "measure queue delay separately from native search and rendering time", "language": "en", "category": "code", "expected": [ "source05" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "code-timing-de", "text": "Wartezeit getrennt von nativer Suche und Darstellung messen", "language": "de", "category": "code", "expected": [ "source05" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "doc-timing-en", "text": "documentation explaining why nested trace durations must not be added together", "language": "en", "category": "document", "expected": [ "source07" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "doc-timing-de", "text": "Dokumentation warum verschachtelte Trace-Zeiten nicht addiert werden dürfen", "language": "de", "category": "document", "expected": [ "source07" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "doc-incremental-en", "text": "how completed embedding work is reused after an interrupted index build", "language": "en", "category": "document", "expected": [ "source08" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "doc-incremental-de", "text": "Wie fertige Embeddings nach unterbrochener Indexierung wiederverwendet werden", "language": "de", "category": "document", "expected": [ "source08" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "doc-watch-en", "text": "limitations of detecting media changes with unchanged file size and modification time", "language": "en", "category": "document", "expected": [ "source09" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "doc-watch-de", "text": "Grenzen der Erkennung von Medienänderungen bei gleicher Größe und Änderungszeit", "language": "de", "category": "document", "expected": [ "source09" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "doc-code-en", "text": "how to search source code using the code retrieval task prefix", "language": "en", "category": "document", "expected": [ "source10" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "doc-code-de", "text": "Wie Quellcodesuche das Code-Retrieval-Präfix verwendet", "language": "de", "category": "document", "expected": [ "source10" ], "split": "frozen-diagnostic", "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." }, { "id": "media-q001", "text": "the fluffy striped cat resting with its chin on the floor", "language": "en", "expected": [ "image01" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q002", "text": "die flauschige gestreifte Katze liegt mit dem Kinn auf dem Boden", "language": "de", "expected": [ "image01" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q003", "text": "the overhead breakfast bowl with berries and diced mango", "language": "en", "expected": [ "image02" ], "category": "image-memory", "hard_negatives": [ "image09" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q004", "text": "die Frühstücksschüssel mit Beeren und gewürfelter Mango von oben", "language": "de", "expected": [ "image02" ], "category": "image-memory", "hard_negatives": [ "image09" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q005", "text": "the daytime lake reflecting jagged mountains and pine trees", "language": "en", "expected": [ "image03" ], "category": "image-memory", "hard_negatives": [ "image05", "image06", "image04" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q006", "text": "der See bei Tageslicht spiegelt schroffe Berge und Tannen", "language": "de", "expected": [ "image03" ], "category": "image-memory", "hard_negatives": [ "image05", "image06", "image04" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q007", "text": "the Milky Way over a dark mountain valley", "language": "en", "expected": [ "image04" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q008", "text": "die Milchstraße über einem dunklen Bergtal", "language": "de", "expected": [ "image04" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q009", "text": "the wooden jetty pointing into a lake under moonlight", "language": "en", "expected": [ "image05" ], "category": "image-memory", "hard_negatives": [ "image03", "image04" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q010", "text": "der Holzsteg führt bei Mondlicht in einen See", "language": "de", "expected": [ "image05" ], "category": "image-memory", "hard_negatives": [ "image03", "image04" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q011", "text": "the pink layered mountain ridges beneath a golden sunset", "language": "en", "expected": [ "image06" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q012", "text": "die rosa Bergketten unter einem goldenen Sonnenuntergang", "language": "de", "expected": [ "image06" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q013", "text": "the herd of cows on green hills", "language": "en", "expected": [ "image07" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q014", "text": "die Kuhherde auf grünen Hügeln", "language": "de", "expected": [ "image07" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q015", "text": "the red sports car driving through woods", "language": "en", "expected": [ "image08" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q016", "text": "das rote Sportauto fährt durch den Wald", "language": "de", "expected": [ "image08" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q017", "text": "the vegetable and grain bowl beside carrot slices", "language": "en", "expected": [ "image09" ], "category": "image-memory", "hard_negatives": [ "image02" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q018", "text": "die Gemüse-Getreide-Schüssel neben Karottenscheiben", "language": "de", "expected": [ "image09" ], "category": "image-memory", "hard_negatives": [ "image02" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q019", "text": "milk being poured into coffee to make a pattern", "language": "en", "expected": [ "image10" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q020", "text": "Milch wird für ein Muster in den Kaffee gegossen", "language": "de", "expected": [ "image10" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q021", "text": "the empty tropical beach with turquoise water", "language": "en", "expected": [ "image11" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q022", "text": "der leere tropische Strand mit türkisfarbenem Wasser", "language": "de", "expected": [ "image11" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q023", "text": "the black and white photo with a hand reaching towards us", "language": "en", "expected": [ "image12" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q024", "text": "das Schwarzweißfoto mit einer Hand, die nach uns greift", "language": "de", "expected": [ "image12" ], "category": "image-memory", "hard_negatives": [], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q025", "text": "the cat jumping up beside a door handle", "language": "en", "expected": [ "clip01" ], "category": "video-action", "intervals": { "clip01": [ [ 2, 4.9 ], [ 15.5, 17.5 ] ] }, "hard_negatives": [ "image01" ], "negative_intervals": { "clip01": [ [ 7, 14 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q026", "text": "die Katze springt neben dem Türgriff hoch", "language": "de", "expected": [ "clip01" ], "category": "video-action", "intervals": { "clip01": [ [ 2, 4.9 ], [ 15.5, 17.5 ] ] }, "hard_negatives": [ "image01" ], "negative_intervals": { "clip01": [ [ 7, 14 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q027", "text": "the cat sitting low by the door after trying to open it", "language": "en", "expected": [ "clip01" ], "category": "video-action", "intervals": { "clip01": [ [ 7, 14 ] ] }, "negative_intervals": { "clip01": [ [ 2, 4.9 ], [ 15.5, 17.5 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q028", "text": "die Katze sitzt nach dem Öffnungsversuch unten an der Tür", "language": "de", "expected": [ "clip01" ], "category": "video-action", "intervals": { "clip01": [ [ 7, 14 ] ] }, "negative_intervals": { "clip01": [ [ 2, 4.9 ], [ 15.5, 17.5 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q029", "text": "the red flower bud before its petals open", "language": "en", "expected": [ "clip03" ], "category": "video-temporal", "intervals": { "clip03": [ [ 0, 1.0 ] ] }, "negative_intervals": { "clip03": [ [ 3.5, 5.05 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q030", "text": "die rote Blütenknospe bevor sich die Blätter öffnen", "language": "de", "expected": [ "clip03" ], "category": "video-temporal", "intervals": { "clip03": [ [ 0, 1.0 ] ] }, "negative_intervals": { "clip03": [ [ 3.5, 5.05 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q031", "text": "the red flower with its petals open wide", "language": "en", "expected": [ "clip03" ], "category": "video-temporal", "intervals": { "clip03": [ [ 3.5, 5.05 ] ] }, "negative_intervals": { "clip03": [ [ 0, 1.0 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q032", "text": "die rote Blume mit weit geöffneten Blütenblättern", "language": "de", "expected": [ "clip03" ], "category": "video-temporal", "intervals": { "clip03": [ [ 3.5, 5.05 ] ] }, "negative_intervals": { "clip03": [ [ 0, 1.0 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q033", "text": "the tall waterfall on a foggy rocky mountain", "language": "en", "expected": [ "clip02" ], "category": "video-scene", "intervals": { "clip02": [ [ 0, 13.81 ] ] }, "hard_negatives": [ "image03", "image05", "image11" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q034", "text": "der hohe Wasserfall an einem nebligen felsigen Berg", "language": "de", "expected": [ "clip02" ], "category": "video-scene", "intervals": { "clip02": [ [ 0, 13.81 ] ] }, "hard_negatives": [ "image03", "image05", "image11" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q035", "text": "the astronaut standing on the Moon next to landing equipment", "language": "en", "expected": [ "clip04" ], "category": "video-scene", "intervals": { "clip04": [ [ 10, 20 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q036", "text": "der Astronaut auf dem Mond neben der Landeausrüstung", "language": "de", "expected": [ "clip04" ], "category": "video-scene", "intervals": { "clip04": [ [ 10, 20 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q037", "text": "the lunar rover driving over grey Moon terrain", "language": "en", "expected": [ "clip04" ], "category": "video-scene", "intervals": { "clip04": [ [ 24, 33 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q038", "text": "der Mondrover fährt über graues Mondgelände", "language": "de", "expected": [ "clip04" ], "category": "video-scene", "intervals": { "clip04": [ [ 24, 33 ] ] }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q039", "text": "someone saying that we choose to go to the Moon", "language": "en", "expected": [ "clip04" ], "category": "video-speech", "intervals": { "clip04": [ [ 1.551, 3.603 ] ] }, "required_modality": "audio", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q040", "text": "jemand sagt, dass wir uns entscheiden, zum Mond zu fliegen", "language": "de", "expected": [ "clip04" ], "category": "video-speech", "intervals": { "clip04": [ [ 1.551, 3.603 ] ] }, "required_modality": "audio", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q041", "text": "someone finding orange soil on the Moon", "language": "en", "expected": [ "clip04" ], "category": "video-speech", "intervals": { "clip04": [ [ 13.129, 16.316 ] ] }, "required_modality": "audio", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q042", "text": "jemand findet orangefarbenen Boden auf dem Mond", "language": "de", "expected": [ "clip04" ], "category": "video-speech", "intervals": { "clip04": [ [ 13.129, 16.316 ] ] }, "required_modality": "audio", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q043", "text": "a request to fund the Artemis program for the first woman on the Moon", "language": "en", "expected": [ "clip04" ], "category": "video-speech", "intervals": { "clip04": [ [ 43.777, 52.419 ] ] }, "required_modality": "audio", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q044", "text": "eine Bitte um Finanzierung von Artemis für die erste Frau auf dem Mond", "language": "de", "expected": [ "clip04" ], "category": "video-speech", "intervals": { "clip04": [ [ 43.777, 52.419 ] ] }, "required_modality": "audio", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q045", "text": "someone talking about planting a flag on Mars", "language": "en", "expected": [ "clip04" ], "category": "video-speech", "intervals": { "clip04": [ [ 56.539, 63.496 ] ] }, "required_modality": "audio", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q046", "text": "jemand spricht davon, eine Flagge auf dem Mars aufzustellen", "language": "de", "expected": [ "clip04" ], "category": "video-speech", "intervals": { "clip04": [ [ 56.539, 63.496 ] ] }, "required_modality": "audio", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q047", "text": "gentle solo piano music without anyone singing", "language": "en", "expected": [ "music01" ], "category": "music-memory", "intervals": { "music01": [ [ 0, 254.45 ] ] }, "required_modality": "audio", "hard_negatives": [ "music02" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q048", "text": "sanfte Klaviermusik ohne Gesang", "language": "de", "expected": [ "music01" ], "category": "music-memory", "intervals": { "music01": [ [ 0, 254.45 ] ] }, "required_modality": "audio", "hard_negatives": [ "music02" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q049", "text": "a jazzy violin melody", "language": "en", "expected": [ "music02" ], "category": "music-memory", "intervals": { "music02": [ [ 0, 30 ] ] }, "required_modality": "audio", "hard_negatives": [ "music01" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q050", "text": "eine jazzige Geigenmelodie", "language": "de", "expected": [ "music02" ], "category": "music-memory", "intervals": { "music02": [ [ 0, 30 ] ] }, "required_modality": "audio", "hard_negatives": [ "music01" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q051", "text": "a dog barking nearby", "language": "en", "expected": [ "sound01" ], "category": "sound-memory", "intervals": { "sound01": [ [ 0, 2.039 ] ] }, "required_modality": "audio", "hard_negatives": [ "sound02" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q052", "text": "ein Hund bellt in der Nähe", "language": "de", "expected": [ "sound01" ], "category": "sound-memory", "intervals": { "sound01": [ [ 0, 2.039 ] ] }, "required_modality": "audio", "hard_negatives": [ "sound02" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q053", "text": "a growling roaring creature sound effect", "language": "en", "expected": [ "sound02" ], "category": "sound-memory", "intervals": { "sound02": [ [ 0, 2.074 ] ] }, "required_modality": "audio", "hard_negatives": [ "sound01" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q054", "text": "ein knurrender brüllender Kreatur-Soundeffekt", "language": "de", "expected": [ "sound02" ], "category": "sound-memory", "intervals": { "sound02": [ [ 0, 2.074 ] ] }, "required_modality": "audio", "hard_negatives": [ "sound01" ], "label_source": "fixtures/media/fixture.json" }, { "id": "media-q055", "text": "Kimiko Ishizaka", "language": "en", "expected": [ "music01" ], "category": "exact-metadata", "metadata": { "artist": "Kimiko Ishizaka" }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q056", "text": "Jazz Violin Solo", "language": "en", "expected": [ "music02" ], "category": "exact-metadata", "metadata": { "title": "Jazz Violin Solo" }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q057", "text": "Violinist WTD", "language": "en", "expected": [ "music02" ], "category": "exact-metadata", "metadata": { "artist": "Violinist WTD" }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q058", "text": "wooden pier under the moon", "language": "en", "expected": [ "image05" ], "category": "exact-filter", "filters": { "extension": "jpg" }, "label_source": "fixtures/media/fixture.json" }, { "id": "media-q059", "text": "the cat jumping beside the handle", "language": "en", "expected": [], "category": "temporal-negative", "filters": { "asset": "clip01", "time_range_s": [ 7, 14 ] }, "required_modality": "image", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q060", "text": "someone mentioning Mars", "language": "en", "expected": [], "category": "filter-no-answer", "filters": { "extension": "ogg" }, "required_modality": "audio", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q061", "text": "a purple elephant playing a violin on the beach", "language": "en", "expected": [], "category": "content-no-answer", "split": "held-out-no-answer", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q062", "text": "ein violetter Elefant spielt Geige am Strand", "language": "de", "expected": [], "category": "content-no-answer", "split": "held-out-no-answer", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q063", "text": "someone explains quantum entanglement", "language": "en", "expected": [], "category": "speech-no-answer", "required_modality": "audio", "split": "held-out-no-answer", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q064", "text": "jemand erklärt Quantenverschränkung", "language": "de", "expected": [], "category": "speech-no-answer", "required_modality": "audio", "split": "held-out-no-answer", "label_source": "fixtures/media/fixture.json" }, { "id": "media-q065", "text": "Adele", "language": "en", "expected": [], "category": "metadata-no-answer", "metadata": { "artist": "Adele" }, "label_source": "fixtures/media/fixture.json" }, { "id": "absent-83", "text": "implementation of a chess engine using alpha beta pruning", "language": "en", "category": "mixed-no-answer", "expected": [], "split": "frozen-diagnostic" }, { "id": "absent-84", "text": "Implementierung einer Schachengine mit Alpha-Beta-Suche", "language": "de", "category": "mixed-no-answer", "expected": [], "split": "frozen-diagnostic" }, { "id": "absent-85", "text": "signed employment agreement with annual salary and holiday allowance", "language": "en", "category": "mixed-no-answer", "expected": [], "split": "frozen-diagnostic" }, { "id": "absent-86", "text": "Unterschriebener Arbeitsvertrag mit Jahresgehalt und Urlaubsanspruch", "language": "de", "category": "mixed-no-answer", "expected": [], "split": "frozen-diagnostic" }, { "id": "upstream-unicode-en", "text": "decode a UTF-8 sequence into a Unicode code point and advance the byte offset", "language": "en", "category": "code", "expected": [ "llama-unicode" ], "split": "frozen-diagnostic" }, { "id": "upstream-unicode-de", "text": "UTF-8-Sequenz in einen Unicode-Codepoint dekodieren und Byteposition erhöhen", "language": "de", "category": "code", "expected": [ "llama-unicode" ], "split": "frozen-diagnostic" }, { "id": "upstream-allocator-en", "text": "reuse memory buffers for tensors in a computational graph", "language": "en", "category": "code", "expected": [ "llama-allocator" ], "split": "frozen-diagnostic" }, { "id": "upstream-allocator-de", "text": "Speicherpuffer für Tensoren in einem Rechengraphen wiederverwenden", "language": "de", "category": "code", "expected": [ "llama-allocator" ], "split": "frozen-diagnostic" } ] }