107 lines
8.0 KiB
Python
107 lines
8.0 KiB
Python
"""Freeze source-based labels without loading inference reports or model scores."""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
OUT = Path(__file__).parent
|
|
|
|
def digest(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
assets = []
|
|
for source in ("relevance-v2", "media"):
|
|
fixture_path = ROOT / "fixtures" / source / "fixture.json"
|
|
fixture = json.loads(fixture_path.read_text(encoding="utf-8-sig"))
|
|
for asset in fixture["assets"]:
|
|
path = fixture_path.parent / asset["path"]
|
|
assert digest(path) == asset["sha256"], path
|
|
assets.append({"id": asset["id"], "path": path.relative_to(ROOT).as_posix(),
|
|
"sha256": digest(path), "bytes": path.stat().st_size,
|
|
"kind": asset["kind"], "source_manifest": fixture_path.relative_to(ROOT).as_posix(),
|
|
"label_basis": asset.get("observation", "Exact source text inspected by label author")})
|
|
for language in ("en", "de"):
|
|
path = ROOT / ".searchi/qa/speech-service-validation-v1/core/corpus" / f"speech-{language}.wav"
|
|
assert path.is_file(), path
|
|
assets.append({"id": f"speech-{language}", "path": path.relative_to(ROOT).as_posix(),
|
|
"sha256": digest(path), "bytes": path.stat().st_size, "kind": "speech",
|
|
"source_manifest": "fixtures/speech-validation-v1/fixture.json",
|
|
"label_basis": "Publisher Article 1 text; derived PCM with 5 second prefix; no model transcript used"})
|
|
|
|
queries = []
|
|
def pair(category, en, de, expected, rationale, scope="all", label="answerable"):
|
|
for language, text in (("en", en), ("de", de)):
|
|
queries.append({"id": f"a3-{len(queries)+1:03}", "text": text,
|
|
"language": language, "category": category, "scope": scope,
|
|
"expected_ids": expected, "label": label, "rationale": rationale,
|
|
"split": "new_blind_holdout"})
|
|
|
|
pair("document", "instructions for proving when a tenant stopped consuming power",
|
|
"Wie belege ich den Stromverbrauch beim Auszug eines Mieters?", ["meter-a"],
|
|
"Moving-day photograph, identifier and handover date support departing resident final bill.")
|
|
pair("document", "the form for transferring a tenant's electricity direct debit to the next resident",
|
|
"Formular zur Übertragung des Stromlastschriftmandats auf den Nachmieter", [],
|
|
"Meter reading document exists but no direct-debit transfer form or procedure.", label="absent")
|
|
pair("document", "proof required before collecting the fragile exhibit on loan",
|
|
"Welchen Nachweis braucht man vor Abholung des zerbrechlichen Leihobjekts?", ["museum-a"],
|
|
"Insurance certificate before collection of ceramic bowl.")
|
|
pair("document", "export customs declaration for taking the borrowed museum bowl overseas",
|
|
"Zollausfuhranmeldung für die geliehene Museumsschale ins Ausland", [],
|
|
"Transit insurance exists; neither customs/export declaration nor overseas loan rule exists.", label="absent")
|
|
pair("code", "parser that separates the plus suffix from prerelease identifiers",
|
|
"Parser, der den Plus-Suffix von Vorabversionskennungen trennt", ["code-version"],
|
|
"Semver Version parser splits '-' prerelease and '+' build identifiers.", scope="code")
|
|
pair("code", "version parser that fetches the newest release from an HTTP package registry",
|
|
"Versionsparser, der die neueste Version aus einer HTTP-Paketregistrierung abruft", [],
|
|
"Semver parsing exists; no network registry retrieval code in fixed two source files.", scope="code", label="absent")
|
|
pair("image", "photo of tiny peaks repeating upside down across quiet water",
|
|
"Foto von Bergspitzen, die sich kopfüber in ruhigem Wasser wiederholen", ["image03"],
|
|
"Mountain and tree reflections in daylight lake.")
|
|
pair("image", "photo of an alpine lake with a red canoe and two paddlers",
|
|
"Foto eines Bergsees mit rotem Kanu und zwei Paddlern", [],
|
|
"Lake photos exist but no canoe or two paddlers in any photo.", label="absent")
|
|
pair("video", "video where a feline keeps reaching up at a room's exit",
|
|
"Video, in dem eine Katze wiederholt am Ausgang eines Zimmers hochspringt", ["clip01"],
|
|
"Orange tabby twice jumps at white door lever.")
|
|
pair("video", "video of a cat squeezing through a flap into a snowy garden",
|
|
"Video einer Katze, die durch eine Klappe in einen verschneiten Garten geht", [],
|
|
"Door jumping cat exists; no flap, snowy garden or passage through it.", label="absent")
|
|
pair("music", "music by Kimiko Ishizaka with the thirteenth Goldberg variation",
|
|
"Musik von Kimiko Ishizaka mit der dreizehnten Goldberg-Variation", ["music01"],
|
|
"Original Vorbis tags plus publisher description identify performer and composition.")
|
|
pair("music", "music by Kimiko Ishizaka playing a Goldberg variation on a violin",
|
|
"Musik von Kimiko Ishizaka mit einer Goldberg-Variation auf der Geige", [],
|
|
"Kimiko piano and separate jazz violin exist; requested performer/instrument/composition conjunction does not.", label="absent")
|
|
pair("sound", "audio of a household pet warning someone with repeated barks",
|
|
"Audio von einem Haustier, das jemanden mit Bellen warnt", ["sound01"],
|
|
"Publisher dog bark description supports barking; warning intent is colloquial query, not a labelled mood.")
|
|
pair("sound", "audio of a dog barking while church bells ring behind it",
|
|
"Audio eines bellenden Hundes mit Kirchenglocken im Hintergrund", [],
|
|
"Dog bark recording exists; no independently described church bells in any audio.", label="absent")
|
|
pair("speech", "spoken passage saying people possess an inner moral judgement",
|
|
"Gesprochene Passage darüber, dass Menschen ein inneres moralisches Urteilsvermögen haben",
|
|
["speech-en", "speech-de"], "Article 1 states endowed with reason and conscience.")
|
|
pair("speech", "spoken explanation of the right to paid holidays and limited working hours",
|
|
"Gesprochene Erklärung des Rechts auf bezahlten Urlaub und begrenzte Arbeitszeit", [],
|
|
"Only Article 1 is in speech files; requested Article 24 topic absent from corpus.", label="absent")
|
|
pair("ambiguous_partial", "pictures of somewhere peaceful near water",
|
|
"Bilder von einem ruhigen Ort am Wasser", ["image03", "image05", "image11"],
|
|
"Subjective broad request: lake and beach both plausible; excluded from strict no-answer metric.", label="partial")
|
|
pair("ambiguous_partial", "what do we know about keeping the loaned exhibit safe",
|
|
"Was wissen wir darüber, das geliehene Ausstellungsstück sicher zu halten?", ["museum-a"],
|
|
"Insurance is relevant partial evidence, but no physical packing/security instruction; do not score as absent.", label="partial")
|
|
|
|
fixture = {"schema": 1, "frozen_before_inference": True,
|
|
"label_policy": "New queries authored without opening inference outputs. Existing source content/independent media descriptions only. One AI annotator, not independent human truth. Never revise labels after outputs. Historical absence20 not reused. Partial queries scored separately. Extraction/indexing failures must be reported separately, not successful abstentions.",
|
|
"corpus_policy": "All 36 declared files form fixed distractor corpus. Copy only declared source bytes into a dedicated corpus; no repository-wide index. Speech PCM stays ignored and is referenced by exact hash, not added to Git.",
|
|
"assets": assets, "queries": queries}
|
|
target = OUT / "fixture.json"
|
|
assert not target.exists(), "Frozen fixture must not be overwritten"
|
|
target.write_text(json.dumps(fixture, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
lock = {"fixture_sha256": digest(target), "assets": len(assets), "queries": len(queries),
|
|
"strict_answerable": sum(q["label"] == "answerable" for q in queries),
|
|
"strict_absent": sum(q["label"] == "absent" for q in queries),
|
|
"partial": sum(q["label"] == "partial" for q in queries)}
|
|
(OUT / "fixture.lock.json").write_text(json.dumps(lock, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(lock))
|