39 lines
1.7 KiB
Python
39 lines
1.7 KiB
Python
|
"""Verify frozen labels and every declared corpus input before inference."""
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
ROOT = Path(__file__).resolve().parents[2]
|
||
|
|
DIRECTORY = Path(__file__).parent
|
||
|
|
|
||
|
|
def verify():
|
||
|
|
target = DIRECTORY / "fixture.json"
|
||
|
|
fixture = json.loads(target.read_text(encoding="utf-8"))
|
||
|
|
lock = json.loads((DIRECTORY / "fixture.lock.json").read_text())
|
||
|
|
assert hashlib.sha256(target.read_bytes()).hexdigest() == lock["fixture_sha256"]
|
||
|
|
assert fixture["frozen_before_inference"] is True
|
||
|
|
assert len(fixture["assets"]) == lock["assets"] == 36
|
||
|
|
assert len(fixture["queries"]) == lock["queries"] == 36
|
||
|
|
ids = {asset["id"] for asset in fixture["assets"]}
|
||
|
|
assert len(ids) == 36
|
||
|
|
for asset in fixture["assets"]:
|
||
|
|
source = (ROOT / asset["path"]).resolve()
|
||
|
|
assert source.is_relative_to(ROOT)
|
||
|
|
assert source.stat().st_size == asset["bytes"]
|
||
|
|
assert hashlib.sha256(source.read_bytes()).hexdigest() == asset["sha256"], source
|
||
|
|
query_ids = set()
|
||
|
|
for query in fixture["queries"]:
|
||
|
|
assert query["id"] not in query_ids
|
||
|
|
query_ids.add(query["id"])
|
||
|
|
assert query["text"] and query["rationale"]
|
||
|
|
assert query["language"] in ("en", "de")
|
||
|
|
assert query["scope"] in ("all", "code")
|
||
|
|
assert query["split"] == "new_blind_holdout"
|
||
|
|
assert set(query["expected_ids"]) <= ids
|
||
|
|
assert (query["label"] == "absent") == (not query["expected_ids"])
|
||
|
|
for label, count_key in (("answerable", "strict_answerable"), ("absent", "strict_absent"), ("partial", "partial")):
|
||
|
|
assert sum(query["label"] == label for query in fixture["queries"]) == lock[count_key]
|
||
|
|
return lock
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
print(json.dumps(verify()))
|