"""Verify frozen labels and every declared corpus input before inference.""" import hashlib import json from pathlib import Path ROOT = Path(__file__).resolve().parents[2] DIRECTORY = Path(__file__).parent def verify(): target = DIRECTORY / "fixture.json" fixture = json.loads(target.read_text(encoding="utf-8")) lock = json.loads((DIRECTORY / "fixture.lock.json").read_text()) assert hashlib.sha256(target.read_bytes()).hexdigest() == lock["fixture_sha256"] assert fixture["frozen_before_inference"] is True assert len(fixture["assets"]) == lock["assets"] == 36 assert len(fixture["queries"]) == lock["queries"] == 36 ids = {asset["id"] for asset in fixture["assets"]} assert len(ids) == 36 for asset in fixture["assets"]: source = (ROOT / asset["path"]).resolve() assert source.is_relative_to(ROOT) assert source.stat().st_size == asset["bytes"] assert hashlib.sha256(source.read_bytes()).hexdigest() == asset["sha256"], source query_ids = set() for query in fixture["queries"]: assert query["id"] not in query_ids query_ids.add(query["id"]) assert query["text"] and query["rationale"] assert query["language"] in ("en", "de") assert query["scope"] in ("all", "code") assert query["split"] == "new_blind_holdout" assert set(query["expected_ids"]) <= ids assert (query["label"] == "absent") == (not query["expected_ids"]) for label, count_key in (("answerable", "strict_answerable"), ("absent", "strict_absent"), ("partial", "partial")): assert sum(query["label"] == label for query in fixture["queries"]) == lock[count_key] return lock if __name__ == "__main__": print(json.dumps(verify()))