Freeze independently reviewed relevance calibration corpus
This commit is contained in:
karl committed
1 parent
b6dd973a88
commit
4e60038c2f
36 files changed
+6565
No files matched your search
@@ -0,0 +1,357 @@
|
||||
{
|
||||
"status": "draft pending independent agent review; not frozen",
|
||||
"labels": [
|
||||
{
|
||||
"query_id": "bicycle-target-en",
|
||||
"expected_ids": [
|
||||
"bicycle-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "bicycle-target-de",
|
||||
"expected_ids": [
|
||||
"bicycle-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "bicycle-contrast-en",
|
||||
"expected_ids": [
|
||||
"bicycle-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "bicycle-contrast-de",
|
||||
"expected_ids": [
|
||||
"bicycle-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "bicycle-absent-en",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "bicycle-absent-de",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "bakery-target-en",
|
||||
"expected_ids": [
|
||||
"bakery-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "bakery-target-de",
|
||||
"expected_ids": [
|
||||
"bakery-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "bakery-contrast-en",
|
||||
"expected_ids": [
|
||||
"bakery-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "bakery-contrast-de",
|
||||
"expected_ids": [
|
||||
"bakery-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "bakery-absent-en",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "bakery-absent-de",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "meter-target-en",
|
||||
"expected_ids": [
|
||||
"meter-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "meter-target-de",
|
||||
"expected_ids": [
|
||||
"meter-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "meter-contrast-en",
|
||||
"expected_ids": [
|
||||
"meter-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "meter-contrast-de",
|
||||
"expected_ids": [
|
||||
"meter-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "meter-absent-en",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "meter-absent-de",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "museum-target-en",
|
||||
"expected_ids": [
|
||||
"museum-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "museum-target-de",
|
||||
"expected_ids": [
|
||||
"museum-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "museum-contrast-en",
|
||||
"expected_ids": [
|
||||
"museum-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "museum-contrast-de",
|
||||
"expected_ids": [
|
||||
"museum-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "museum-absent-en",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "museum-absent-de",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "irrigation-target-en",
|
||||
"expected_ids": [
|
||||
"irrigation-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "irrigation-target-de",
|
||||
"expected_ids": [
|
||||
"irrigation-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "irrigation-contrast-en",
|
||||
"expected_ids": [
|
||||
"irrigation-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "irrigation-contrast-de",
|
||||
"expected_ids": [
|
||||
"irrigation-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "irrigation-absent-en",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "irrigation-absent-de",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "ferry-target-en",
|
||||
"expected_ids": [
|
||||
"ferry-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "ferry-target-de",
|
||||
"expected_ids": [
|
||||
"ferry-a"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Target document explicitly states the requested rule; the paired document concerns a different procedure."
|
||||
},
|
||||
{
|
||||
"query_id": "ferry-contrast-en",
|
||||
"expected_ids": [
|
||||
"ferry-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "ferry-contrast-de",
|
||||
"expected_ids": [
|
||||
"ferry-b"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Contrasting document explicitly states this requested procedure or tariff."
|
||||
},
|
||||
{
|
||||
"query_id": "ferry-absent-en",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "ferry-absent-de",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source contains this named certificate, permit or signed notice; related topic alone does not satisfy the artifact request."
|
||||
},
|
||||
{
|
||||
"query_id": "pattern-build-en",
|
||||
"expected_ids": [
|
||||
"code-pattern"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Builder::build and Builder::add assemble packed searchers."
|
||||
},
|
||||
{
|
||||
"query_id": "pattern-build-de",
|
||||
"expected_ids": [
|
||||
"code-pattern"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Builder::build and Builder::add assemble packed searchers."
|
||||
},
|
||||
{
|
||||
"query_id": "pattern-range-en",
|
||||
"expected_ids": [
|
||||
"code-pattern"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Searcher::find_in implements bounded-span dispatch."
|
||||
},
|
||||
{
|
||||
"query_id": "pattern-range-de",
|
||||
"expected_ids": [
|
||||
"code-pattern"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "Searcher::find_in implements bounded-span dispatch."
|
||||
},
|
||||
{
|
||||
"query_id": "pattern-absent-en",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "Neither code source nor customer document implements certificate verification."
|
||||
},
|
||||
{
|
||||
"query_id": "pattern-absent-de",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "Neither code source nor customer document implements certificate verification."
|
||||
},
|
||||
{
|
||||
"query_id": "version-parse-en",
|
||||
"expected_ids": [
|
||||
"code-version"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "FromStr for Version parses major/minor/patch, prerelease and build."
|
||||
},
|
||||
{
|
||||
"query_id": "version-parse-de",
|
||||
"expected_ids": [
|
||||
"code-version"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "FromStr for Version parses major/minor/patch, prerelease and build."
|
||||
},
|
||||
{
|
||||
"query_id": "version-wildcard-en",
|
||||
"expected_ids": [
|
||||
"code-version"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "VersionReq parsing rejects WildcardNotTheOnlyComparator."
|
||||
},
|
||||
{
|
||||
"query_id": "version-wildcard-de",
|
||||
"expected_ids": [
|
||||
"code-version"
|
||||
],
|
||||
"classification": "positive",
|
||||
"rationale": "VersionReq parsing rejects WildcardNotTheOnlyComparator."
|
||||
},
|
||||
{
|
||||
"query_id": "version-absent-en",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source implements digital signature verification."
|
||||
},
|
||||
{
|
||||
"query_id": "version-absent-de",
|
||||
"expected_ids": [],
|
||||
"classification": "absent_artifact",
|
||||
"rationale": "No source implements digital signature verification."
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user