523 lines
23 KiB
JSON
523 lines
23 KiB
JSON
{
|
|
"version": 2,
|
|
"reviewer": "independent AI agent blind review; not human annotation",
|
|
"inputPath": "fixtures/relevance-v2/blind-review-input.json",
|
|
"inputSHA256": "d90a3b863c7985acfc93fce0b4c4b1325c109c80d012e81dda87e14aa6bcc94a",
|
|
"blindness": "No author labels, ranking results, model outputs or query runtime read. Required project instructions and input-listed local sources/licenses only.",
|
|
"assetSHA256Verified": true,
|
|
"queries": [
|
|
{
|
|
"id": "bicycle-target-en",
|
|
"expectedIDs": [
|
|
"bicycle-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "bicycle-a: five-year cracked-weld warranty; submit photo, serial and receipt.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bicycle-target-de",
|
|
"expectedIDs": [
|
|
"bicycle-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "bicycle-a: five-year cracked-weld warranty; submit photo, serial and receipt.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bicycle-contrast-en",
|
|
"expectedIDs": [
|
|
"bicycle-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "bicycle-b: brake-pad replacement costs 24 euros; bicycle-a exclusions are not a tariff.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bicycle-contrast-de",
|
|
"expectedIDs": [
|
|
"bicycle-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "bicycle-b: brake-pad replacement costs 24 euros; bicycle-a exclusions are not a tariff.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bicycle-absent-en",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bicycle-absent-de",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bakery-target-en",
|
|
"expectedIDs": [
|
|
"bakery-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "bakery-a: second staff member verifies almond warning before release; record batch and initials.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bakery-target-de",
|
|
"expectedIDs": [
|
|
"bakery-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "bakery-a: second staff member verifies almond warning before release; record batch and initials.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bakery-contrast-en",
|
|
"expectedIDs": [
|
|
"bakery-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "bakery-b: pack sealed cartons in reusable crates and check address/count before loading.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bakery-contrast-de",
|
|
"expectedIDs": [
|
|
"bakery-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "bakery-b: pack sealed cartons in reusable crates and check address/count before loading.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bakery-absent-en",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "bakery-absent-de",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "meter-target-en",
|
|
"expectedIDs": [
|
|
"meter-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "meter-a: moving-day electricity-meter photo must show reading and identifier; send within three days.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "meter-target-de",
|
|
"expectedIDs": [
|
|
"meter-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "meter-a: moving-day electricity-meter photo must show reading and identifier; send within three days.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "meter-contrast-en",
|
|
"expectedIDs": [
|
|
"meter-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "meter-b: November inspection checks radiator valve leaks and air; access 09:00-15:00.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "meter-contrast-de",
|
|
"expectedIDs": [
|
|
"meter-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "meter-b: November inspection checks radiator valve leaks and air; access 09:00-15:00.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "meter-absent-en",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "meter-absent-de",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "museum-target-en",
|
|
"expectedIDs": [
|
|
"museum-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity: query requests loan terms, which the source supplies.",
|
|
"reason": "museum-a states loan terms requiring insurance during both outward and return ceramic-bowl transport, directly satisfying the clarified operative-rule request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "museum-target-de",
|
|
"expectedIDs": [
|
|
"museum-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity: query requests loan terms, which the source supplies.",
|
|
"reason": "museum-a states loan terms requiring insurance during both outward and return ceramic-bowl transport, directly satisfying the clarified operative-rule request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "museum-contrast-en",
|
|
"expectedIDs": [
|
|
"museum-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "museum-b: conservators record relative humidity twice each day.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "museum-contrast-de",
|
|
"expectedIDs": [
|
|
"museum-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "museum-b: conservators record relative humidity twice each day.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "museum-absent-en",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "museum-absent-de",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "irrigation-target-en",
|
|
"expectedIDs": [
|
|
"irrigation-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "irrigation-a: stop pump below twelve centimetres, refill above twenty before reset.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "irrigation-target-de",
|
|
"expectedIDs": [
|
|
"irrigation-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "irrigation-a: stop pump below twelve centimetres, refill above twenty before reset.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "irrigation-contrast-en",
|
|
"expectedIDs": [
|
|
"irrigation-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "irrigation-b: drain hoses before frost, disconnect and store dry in shed.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "irrigation-contrast-de",
|
|
"expectedIDs": [
|
|
"irrigation-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "irrigation-b: drain hoses before frost, disconnect and store dry in shed.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "irrigation-absent-en",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "irrigation-absent-de",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "ferry-target-en",
|
|
"expectedIDs": [
|
|
"ferry-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "ferry-a: two qualified crew required Saturday; cancel and notify if either unavailable.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "ferry-target-de",
|
|
"expectedIDs": [
|
|
"ferry-a"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "ferry-a: two qualified crew required Saturday; cancel and notify if either unavailable.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "ferry-contrast-en",
|
|
"expectedIDs": [
|
|
"ferry-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "ferry-b: bicycle carriage costs one euro per crossing.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "ferry-contrast-de",
|
|
"expectedIDs": [
|
|
"ferry-b"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "ferry-b: bicycle carriage costs one euro per crossing.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "ferry-absent-en",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "ferry-absent-de",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "pattern-build-en",
|
|
"expectedIDs": [
|
|
"code-pattern"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "Builder::add/extend accumulates byte patterns; Builder::build constructs configured pattern and RabinKarp/Teddy state and returns Searcher. Searcher::new invokes construction. Actual executable implementation is present.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "pattern-build-de",
|
|
"expectedIDs": [
|
|
"code-pattern"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "Builder::add/extend accumulates byte patterns; Builder::build constructs configured pattern and RabinKarp/Teddy state and returns Searcher. Searcher::new invokes construction. Actual executable implementation is present.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "pattern-range-en",
|
|
"expectedIDs": [
|
|
"code-pattern"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity: clarified request explicitly asks for the first match among configured patterns in a bounded span.",
|
|
"reason": "Executable Searcher::find_in caps haystack at span.end and invokes Teddy/RabinKarp from span.start, with a slow short-span fallback, returning the first occurrence of any configured pattern according to match semantics.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "pattern-range-de",
|
|
"expectedIDs": [
|
|
"code-pattern"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity: clarified request explicitly asks for the first match among configured patterns in a bounded span.",
|
|
"reason": "Executable Searcher::find_in caps haystack at span.end and invokes Teddy/RabinKarp from span.start, with a slow short-span fallback, returning the first occurrence of any configured pattern according to match semantics.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "pattern-absent-en",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "pattern-absent-de",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "version-parse-en",
|
|
"expectedIDs": [
|
|
"code-version"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "Version::from_str validates major/minor/patch and optional prerelease/build identifiers, then constructs Version. Actual parser implementation is present.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "version-parse-de",
|
|
"expectedIDs": [
|
|
"code-version"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "Version::from_str validates major/minor/patch and optional prerelease/build identifiers, then constructs Version. Actual parser implementation is present.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "version-wildcard-en",
|
|
"expectedIDs": [
|
|
"code-version"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "Clarified lone wildcard example matches implemented rejection; numeric partial wildcards are outside the request.",
|
|
"reason": "VersionReq::from_str rejects lone */x/X followed by a comma using WildcardNotTheOnlyComparator; recursive version_req also checks a later lone wildcard. Matches clarified *, >=1.0.0 example.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "version-wildcard-de",
|
|
"expectedIDs": [
|
|
"code-version"
|
|
],
|
|
"confidence": "high",
|
|
"label": "relevant_operative_content",
|
|
"ambiguityReason": "Clarified lone wildcard example matches implemented rejection; numeric partial wildcards are outside the request.",
|
|
"reason": "VersionReq::from_str rejects lone */x/X followed by a comma using WildcardNotTheOnlyComparator; recursive version_req also checks a later lone wildcard. Matches clarified *, >=1.0.0 example.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "version-absent-en",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
},
|
|
{
|
|
"id": "version-absent-de",
|
|
"expectedIDs": [],
|
|
"confidence": "high",
|
|
"label": "clear_absent_artifact",
|
|
"ambiguityReason": "No material ambiguity under required_artifact operative-rule definition.",
|
|
"reason": "The requested specific artifact or implementation is absent from every inspected source; shared company/entity or topic does not satisfy the request.",
|
|
"sourceQueryMismatch": []
|
|
}
|
|
],
|
|
"evaluationAssessment": {
|
|
"meaningfulTextAbstention": true,
|
|
"meaningfulCodeAbstention": true,
|
|
"limits": [
|
|
"Small fictional corpus, unusually explicit rules and contrasts; not representative customer diversity.",
|
|
"Specific absent-artifact queries are distinct from broad-topic retrieval; no broad-topic queries are included.",
|
|
"Two real code files and far-away negatives do not cover near-miss implementations or overloaded symbols.",
|
|
"EN/DE pairs are correlated observations.",
|
|
"Provisional AI review; human annotation and inference remain unperformed."
|
|
],
|
|
"humanReviewRemaining": [
|
|
"Human-adjudicate all provisional agent labels before treating them as human reference data.",
|
|
"Add realistic customer documents, near-miss implementations and broad-topic queries for representative calibration; current small corpus remains a diagnostic."
|
|
]
|
|
},
|
|
"correctedInputReview": "Independently reread four precision clarifications: museum-target EN/DE asks loan terms and pattern-range EN/DE explicitly asks first bounded match. Both original caveats resolved; expected IDs unchanged. Verified current native German strings and all 14 source fixture SHA identities. Previous input/review preserved by coordinating author under review-history."
|
|
}
|