karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Freeze independently reviewed relevance calibration corpus

Browse Source
This commit is contained in:
karl committed 2026-10-13 10:15:32 +02:00
1 parent b6dd973a88
commit 4e60038c2f
36 files changed
+6565
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

fixtures/relevance-v2/blind-review-input.json
+581
View File
@@ -0,0 +1,581 @@
{
"version": 2,
"scope": "text/code only; no inference performed",
"assets": [
{
"id": "code-pattern",
"path": "sources/BurntSushi__aho-corasick__src__packed__api.rs",
"kind": "code",
"split": "calibration",
"repository": "BurntSushi/aho-corasick",
"commit": "6c0abf5681bfc30bb9d8f7f52b68a350b436fffa",
"source_path": "src/packed/api.rs",
"url": "https://raw.githubusercontent.com/BurntSushi/aho-corasick/6c0abf5681bfc30bb9d8f7f52b68a350b436fffa/src/packed/api.rs",
"sha256": "7bcb23b99bbf87e41348ef8d4db72a88dff744de503dcb684ac9d40659656f96",
"bytes": 23091,
"license": "MIT OR Unlicense",
"license_files": [
{
"path": "licenses/BurntSushi__aho-corasick__LICENSE-MIT.txt",
"sha256": "0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f",
"url": "https://raw.githubusercontent.com/BurntSushi/aho-corasick/6c0abf5681bfc30bb9d8f7f52b68a350b436fffa/LICENSE-MIT"
},
{
"path": "licenses/BurntSushi__aho-corasick__UNLICENSE.txt",
"sha256": "7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c",
"url": "https://raw.githubusercontent.com/BurntSushi/aho-corasick/6c0abf5681bfc30bb9d8f7f52b68a350b436fffa/UNLICENSE"
},
{
"path": "licenses/BurntSushi__aho-corasick__COPYING.txt",
"sha256": "01c266bced4a434da0051174d6bee16a4c82cf634e2679b6155d40d75012390f",
"url": "https://raw.githubusercontent.com/BurntSushi/aho-corasick/6c0abf5681bfc30bb9d8f7f52b68a350b436fffa/COPYING"
}
],
"changes": "Exact upstream bytes; no edits"
},
{
"id": "code-version",
"path": "sources/dtolnay__semver__src__parse.rs",
"kind": "code",
"split": "held_out",
"repository": "dtolnay/semver",
"commit": "280ebcb6edac3aa4cdc545dbff8a26c5ac4861fe",
"source_path": "src/parse.rs",
"url": "https://raw.githubusercontent.com/dtolnay/semver/280ebcb6edac3aa4cdc545dbff8a26c5ac4861fe/src/parse.rs",
"sha256": "2d9cb28a72a813968c16da29a9690f8e53f1ce00727a1dbe67aec3ddb5ed59ed",
"bytes": 12187,
"license": "MIT OR Apache-2.0",
"license_files": [
{
"path": "licenses/dtolnay__semver__LICENSE-MIT.txt",
"sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3",
"url": "https://raw.githubusercontent.com/dtolnay/semver/280ebcb6edac3aa4cdc545dbff8a26c5ac4861fe/LICENSE-MIT"
},
{
"path": "licenses/dtolnay__semver__LICENSE-APACHE.txt",
"sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a",
"url": "https://raw.githubusercontent.com/dtolnay/semver/280ebcb6edac3aa4cdc545dbff8a26c5ac4861fe/LICENSE-APACHE"
}
],
"changes": "Exact upstream bytes; no edits"
},
{
"id": "bicycle-a",
"path": "sources/bicycle-a.md",
"kind": "document",
"split": "calibration",
"topic_family": "bicycle",
"sha256": "2d77a47673d5900784efa2eb7943efb29abcf36b19ef81e4291ce52dc47f1f70",
"bytes": 274,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "bicycle-b",
"path": "sources/bicycle-b.md",
"kind": "document",
"split": "calibration",
"topic_family": "bicycle",
"sha256": "6302d4a3832b65d25a46e8502cfa5dcaeb6256ef22cae9b7f6eda9d01e5d3375",
"bytes": 247,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "bakery-a",
"path": "sources/bakery-a.md",
"kind": "document",
"split": "calibration",
"topic_family": "bakery",
"sha256": "158ee5091f8c6cdb6775da7244441731869da3645059f10048c975504003484d",
"bytes": 267,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "bakery-b",
"path": "sources/bakery-b.md",
"kind": "document",
"split": "calibration",
"topic_family": "bakery",
"sha256": "c17ce0846c41b46b4346ed7e83815677dddd678cc6268297d865bca912df4920",
"bytes": 270,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "meter-a",
"path": "sources/meter-a.md",
"kind": "document",
"split": "calibration",
"topic_family": "meter",
"sha256": "ab41afc2781a045ab2eac305a68074830c9878cd951a8ee4fcd65335b4a91aab",
"bytes": 307,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "meter-b",
"path": "sources/meter-b.md",
"kind": "document",
"split": "calibration",
"topic_family": "meter",
"sha256": "6044b38f140d5302619f450fa6a7ad5ae08133e2d748a4279b325ca42b28fbb9",
"bytes": 255,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "museum-a",
"path": "sources/museum-a.md",
"kind": "document",
"split": "held_out",
"topic_family": "museum",
"sha256": "4e88213ad17ac4eb63f8acd13d9842ededebd6f211226876bd817bbd296246d0",
"bytes": 278,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "museum-b",
"path": "sources/museum-b.md",
"kind": "document",
"split": "held_out",
"topic_family": "museum",
"sha256": "51072d9e67336305851f74100583e748a4b9cbc817a83a4ec3575d5bdb792a15",
"bytes": 260,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "irrigation-a",
"path": "sources/irrigation-a.md",
"kind": "document",
"split": "held_out",
"topic_family": "irrigation",
"sha256": "e57ab03ca777a20f8a281ea700f1117d56b25439a12e05f08af5d3a928e78488",
"bytes": 306,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "irrigation-b",
"path": "sources/irrigation-b.md",
"kind": "document",
"split": "held_out",
"topic_family": "irrigation",
"sha256": "9a9b28cc09bc8f1468c6933f1d0969909e7f58478084fcbbda51b96dabb83c6c",
"bytes": 251,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "ferry-a",
"path": "sources/ferry-a.md",
"kind": "document",
"split": "held_out",
"topic_family": "ferry",
"sha256": "c88a26ac20a896b2f815861e2fc10d6ccee8a572ff3ee637b4eecdcac4adce86",
"bytes": 261,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
},
{
"id": "ferry-b",
"path": "sources/ferry-b.md",
"kind": "document",
"split": "held_out",
"topic_family": "ferry",
"sha256": "47c14d1b88e3f38167778347ec1978b4346555b66f81f321bf18e5d7ce868a41",
"bytes": 244,
"license": "MIT; searchi repository LICENSE",
"provenance": "Explicitly authored fictional diagnostic; no real customer data"
}
],
"queries": [
{
"id": "bicycle-target-en",
"pair_id": "bicycle-target",
"split": "calibration",
"language": "en",
"query": "warranty instructions for a cracked bicycle frame weld",
"required_artifact": "document with the requested operative rule"
},
{
"id": "bicycle-target-de",
"pair_id": "bicycle-target",
"split": "calibration",
"language": "de",
"query": "Garantieanleitung für eine gerissene Schweißnaht am Fahrradrahmen",
"required_artifact": "document with the requested operative rule"
},
{
"id": "bicycle-contrast-en",
"pair_id": "bicycle-contrast",
"split": "calibration",
"language": "en",
"query": "price list for replacing bicycle brake pads",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "bicycle-contrast-de",
"pair_id": "bicycle-contrast",
"split": "calibration",
"language": "de",
"query": "Preisliste für den Austausch von Fahrradbremsbelägen",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "bicycle-absent-en",
"pair_id": "bicycle-absent",
"split": "calibration",
"language": "en",
"query": "signed recall notice for Aster electric bicycle battery fires",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "bicycle-absent-de",
"pair_id": "bicycle-absent",
"split": "calibration",
"language": "de",
"query": "unterschriebene Rückrufmitteilung zu Akkubränden bei Aster-Elektrofahrrädern",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "bakery-target-en",
"pair_id": "bakery-target",
"split": "calibration",
"language": "en",
"query": "procedure for checking almond warnings before biscuit batches leave the kitchen",
"required_artifact": "document with the requested operative rule"
},
{
"id": "bakery-target-de",
"pair_id": "bakery-target",
"split": "calibration",
"language": "de",
"query": "Verfahren zur Prüfung von Mandelwarnungen vor der Freigabe von Keks-Chargen",
"required_artifact": "document with the requested operative rule"
},
{
"id": "bakery-contrast-en",
"pair_id": "bakery-contrast",
"split": "calibration",
"language": "en",
"query": "checklist for loading biscuit deliveries into reusable crates",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "bakery-contrast-de",
"pair_id": "bakery-contrast",
"split": "calibration",
"language": "de",
"query": "Checkliste zum Verladen von Kekslieferungen in Mehrwegkisten",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "bakery-absent-en",
"pair_id": "bakery-absent",
"split": "calibration",
"language": "en",
"query": "laboratory certificate proving Juniper biscuits contain no gluten",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "bakery-absent-de",
"pair_id": "bakery-absent",
"split": "calibration",
"language": "de",
"query": "Laborzertifikat zum Nachweis glutenfreier Juniper-Kekse",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "meter-target-en",
"pair_id": "meter-target",
"split": "calibration",
"language": "en",
"query": "instructions for photographing an electricity meter when moving out",
"required_artifact": "document with the requested operative rule"
},
{
"id": "meter-target-de",
"pair_id": "meter-target",
"split": "calibration",
"language": "de",
"query": "Anleitung zum Fotografieren des Stromzählers beim Auszug",
"required_artifact": "document with the requested operative rule"
},
{
"id": "meter-contrast-en",
"pair_id": "meter-contrast",
"split": "calibration",
"language": "en",
"query": "November schedule for checking leaking radiator valves",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "meter-contrast-de",
"pair_id": "meter-contrast",
"split": "calibration",
"language": "de",
"query": "Novemberplan zur Prüfung undichter Heizkörperventile",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "meter-absent-en",
"pair_id": "meter-absent",
"split": "calibration",
"language": "en",
"query": "solar installation permit for the roof of Rowan Court",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "meter-absent-de",
"pair_id": "meter-absent",
"split": "calibration",
"language": "de",
"query": "Genehmigung für eine Solaranlage auf dem Dach von Rowan Court",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "museum-target-en",
"pair_id": "museum-target",
"split": "held_out",
"language": "en",
"query": "loan terms requiring insurance for the return journey of a ceramic bowl",
"required_artifact": "document with the requested operative rule"
},
{
"id": "museum-target-de",
"pair_id": "museum-target",
"split": "held_out",
"language": "de",
"query": "Leihbedingungen mit Versicherungspflicht für den Rücktransport einer Keramikschale",
"required_artifact": "document with the requested operative rule"
},
{
"id": "museum-contrast-en",
"pair_id": "museum-contrast",
"split": "held_out",
"language": "en",
"query": "log instructions for recording gallery humidity twice daily",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "museum-contrast-de",
"pair_id": "museum-contrast",
"split": "held_out",
"language": "de",
"query": "Anleitung zur zweimal täglichen Erfassung der Luftfeuchtigkeit im Museumssaal",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "museum-absent-en",
"pair_id": "museum-absent",
"split": "held_out",
"language": "en",
"query": "customs export permit for the Elm Museum bronze statue",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "museum-absent-de",
"pair_id": "museum-absent",
"split": "held_out",
"language": "de",
"query": "Zollausfuhrgenehmigung für die Bronzestatue des Elm Museums",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "irrigation-target-en",
"pair_id": "irrigation-target",
"split": "held_out",
"language": "en",
"query": "controller instructions that stop the irrigation pump when the tank is nearly empty",
"required_artifact": "document with the requested operative rule"
},
{
"id": "irrigation-target-de",
"pair_id": "irrigation-target",
"split": "held_out",
"language": "de",
"query": "Steuerungsanleitung zum Abschalten der Bewässerungspumpe bei fast leerem Tank",
"required_artifact": "document with the requested operative rule"
},
{
"id": "irrigation-contrast-en",
"pair_id": "irrigation-contrast",
"split": "held_out",
"language": "en",
"query": "winter checklist for draining and storing sprinkler hoses",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "irrigation-contrast-de",
"pair_id": "irrigation-contrast",
"split": "held_out",
"language": "de",
"query": "Wintercheckliste zum Entleeren und Lagern von Beregnungsschläuchen",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "irrigation-absent-en",
"pair_id": "irrigation-absent",
"split": "held_out",
"language": "en",
"query": "water quality laboratory report certifying Linden tank water is safe to drink",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "irrigation-absent-de",
"pair_id": "irrigation-absent",
"split": "held_out",
"language": "de",
"query": "Laborbericht zur Trinkwassereignung des Wassers im Linden-Tank",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "ferry-target-en",
"pair_id": "ferry-target",
"split": "held_out",
"language": "en",
"query": "rota rules for cancelling Saturday ferry trips when a crew member is unavailable",
"required_artifact": "document with the requested operative rule"
},
{
"id": "ferry-target-de",
"pair_id": "ferry-target",
"split": "held_out",
"language": "de",
"query": "Dienstplanregeln zur Absage von Samstagsfahrten bei Ausfall eines Fährbesatzungsmitglieds",
"required_artifact": "document with the requested operative rule"
},
{
"id": "ferry-contrast-en",
"pair_id": "ferry-contrast",
"split": "held_out",
"language": "en",
"query": "fare sheet listing the cost of taking a bicycle on the ferry",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "ferry-contrast-de",
"pair_id": "ferry-contrast",
"split": "held_out",
"language": "de",
"query": "Tarifblatt mit den Kosten für die Fahrradmitnahme auf der Fähre",
"required_artifact": "document containing the requested procedure or tariff"
},
{
"id": "ferry-absent-en",
"pair_id": "ferry-absent",
"split": "held_out",
"language": "en",
"query": "signed engineering certificate for replacing the Alder Ferry diesel engine",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "ferry-absent-de",
"pair_id": "ferry-absent",
"split": "held_out",
"language": "de",
"query": "unterschriebenes Ingenieurzertifikat zum Austausch des Dieselmotors der Alder-Fähre",
"required_artifact": "specific certificate, permit or signed notice described in the query"
},
{
"id": "pattern-build-en",
"pair_id": "pattern-build",
"split": "calibration",
"language": "en",
"query": "implementation that builds a packed matcher from several byte patterns",
"required_artifact": "source implementation of packed matcher construction"
},
{
"id": "pattern-build-de",
"pair_id": "pattern-build",
"split": "calibration",
"language": "de",
"query": "Implementierung zum Aufbau eines gepackten Suchers aus mehreren Byte-Mustern",
"required_artifact": "source implementation of packed matcher construction"
},
{
"id": "pattern-range-en",
"pair_id": "pattern-range",
"split": "calibration",
"language": "en",
"query": "code that returns the first match among several byte patterns inside a bounded span of a byte string",
"required_artifact": "source implementation of bounded pattern search"
},
{
"id": "pattern-range-de",
"pair_id": "pattern-range",
"split": "calibration",
"language": "de",
"query": "Code, der den ersten Treffer unter mehreren Byte-Mustern innerhalb eines begrenzten Bytebereichs zurückgibt",
"required_artifact": "source implementation of bounded pattern search"
},
{
"id": "pattern-absent-en",
"pair_id": "pattern-absent",
"split": "calibration",
"language": "en",
"query": "implementation of TLS certificate hostname verification",
"required_artifact": "source implementation of TLS hostname verification"
},
{
"id": "pattern-absent-de",
"pair_id": "pattern-absent",
"split": "calibration",
"language": "de",
"query": "Implementierung der Hostnamenprüfung für TLS-Zertifikate",
"required_artifact": "source implementation of TLS hostname verification"
},
{
"id": "version-parse-en",
"pair_id": "version-parse",
"split": "held_out",
"language": "en",
"query": "parser for semantic version numbers with prerelease and build metadata",
"required_artifact": "source implementation parsing SemVer components"
},
{
"id": "version-parse-de",
"pair_id": "version-parse",
"split": "held_out",
"language": "de",
"query": "Parser für semantische Versionsnummern mit Vorabversion und Build-Metadaten",
"required_artifact": "source implementation parsing SemVer components"
},
{
"id": "version-wildcard-en",
"pair_id": "version-wildcard",
"split": "held_out",
"language": "en",
"query": "implementation rejecting a lone wildcard followed by additional version comparators, for example *, >=1.0.0",
"required_artifact": "source implementation checking wildcard comparator constraints"
},
{
"id": "version-wildcard-de",
"pair_id": "version-wildcard",
"split": "held_out",
"language": "de",
"query": "Implementierung zum Ablehnen eines alleinstehenden Platzhalters mit weiteren Versionsvergleichern, zum Beispiel *, >=1.0.0",
"required_artifact": "source implementation checking wildcard comparator constraints"
},
{
"id": "version-absent-en",
"pair_id": "version-absent",
"split": "held_out",
"language": "en",
"query": "implementation that verifies an Ed25519 digital signature",
"required_artifact": "source implementation of Ed25519 verification"
},
{
"id": "version-absent-de",
"pair_id": "version-absent",
"split": "held_out",
"language": "de",
"query": "Implementierung zur Prüfung einer digitalen Ed25519-Signatur",
"required_artifact": "source implementation of Ed25519 verification"
}
]
}