Distinguish main matches from related suggestions and ship 0.1.9

fixtures/absence-v3/verify.py

@@ -0,0 +1,39 @@
+"""Verify frozen labels and every declared corpus input before inference."""
+import hashlib
+import json
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+DIRECTORY = Path(__file__).parent
+
+def verify():
+    target = DIRECTORY / "fixture.json"
+    fixture = json.loads(target.read_text(encoding="utf-8"))
+    lock = json.loads((DIRECTORY / "fixture.lock.json").read_text())
+    assert hashlib.sha256(target.read_bytes()).hexdigest() == lock["fixture_sha256"]
+    assert fixture["frozen_before_inference"] is True
+    assert len(fixture["assets"]) == lock["assets"] == 36
+    assert len(fixture["queries"]) == lock["queries"] == 36
+    ids = {asset["id"] for asset in fixture["assets"]}
+    assert len(ids) == 36
+    for asset in fixture["assets"]:
+        source = (ROOT / asset["path"]).resolve()
+        assert source.is_relative_to(ROOT)
+        assert source.stat().st_size == asset["bytes"]
+        assert hashlib.sha256(source.read_bytes()).hexdigest() == asset["sha256"], source
+    query_ids = set()
+    for query in fixture["queries"]:
+        assert query["id"] not in query_ids
+        query_ids.add(query["id"])
+        assert query["text"] and query["rationale"]
+        assert query["language"] in ("en", "de")
+        assert query["scope"] in ("all", "code")
+        assert query["split"] == "new_blind_holdout"
+        assert set(query["expected_ids"]) <= ids
+        assert (query["label"] == "absent") == (not query["expected_ids"])
+    for label, count_key in (("answerable", "strict_answerable"), ("absent", "strict_absent"), ("partial", "partial")):
+        assert sum(query["label"] == label for query in fixture["queries"]) == lock[count_key]
+    return lock
+
+if __name__ == "__main__":
+    print(json.dumps(verify()))