"""Freeze source-based labels without loading inference reports or model scores.""" import hashlib import json from pathlib import Path ROOT = Path(__file__).resolve().parents[2] OUT = Path(__file__).parent def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() assets = [] for source in ("relevance-v2", "media"): fixture_path = ROOT / "fixtures" / source / "fixture.json" fixture = json.loads(fixture_path.read_text(encoding="utf-8-sig")) for asset in fixture["assets"]: path = fixture_path.parent / asset["path"] assert digest(path) == asset["sha256"], path assets.append({"id": asset["id"], "path": path.relative_to(ROOT).as_posix(), "sha256": digest(path), "bytes": path.stat().st_size, "kind": asset["kind"], "source_manifest": fixture_path.relative_to(ROOT).as_posix(), "label_basis": asset.get("observation", "Exact source text inspected by label author")}) for language in ("en", "de"): path = ROOT / ".searchi/qa/speech-service-validation-v1/core/corpus" / f"speech-{language}.wav" assert path.is_file(), path assets.append({"id": f"speech-{language}", "path": path.relative_to(ROOT).as_posix(), "sha256": digest(path), "bytes": path.stat().st_size, "kind": "speech", "source_manifest": "fixtures/speech-validation-v1/fixture.json", "label_basis": "Publisher Article 1 text; derived PCM with 5 second prefix; no model transcript used"}) queries = [] def pair(category, en, de, expected, rationale, scope="all", label="answerable"): for language, text in (("en", en), ("de", de)): queries.append({"id": f"a3-{len(queries)+1:03}", "text": text, "language": language, "category": category, "scope": scope, "expected_ids": expected, "label": label, "rationale": rationale, "split": "new_blind_holdout"}) pair("document", "instructions for proving when a tenant stopped consuming power", "Wie belege ich den Stromverbrauch beim Auszug eines Mieters?", ["meter-a"], "Moving-day photograph, identifier and handover date support departing resident final bill.") pair("document", "the form for transferring a tenant's electricity direct debit to the next resident", "Formular zur Übertragung des Stromlastschriftmandats auf den Nachmieter", [], "Meter reading document exists but no direct-debit transfer form or procedure.", label="absent") pair("document", "proof required before collecting the fragile exhibit on loan", "Welchen Nachweis braucht man vor Abholung des zerbrechlichen Leihobjekts?", ["museum-a"], "Insurance certificate before collection of ceramic bowl.") pair("document", "export customs declaration for taking the borrowed museum bowl overseas", "Zollausfuhranmeldung für die geliehene Museumsschale ins Ausland", [], "Transit insurance exists; neither customs/export declaration nor overseas loan rule exists.", label="absent") pair("code", "parser that separates the plus suffix from prerelease identifiers", "Parser, der den Plus-Suffix von Vorabversionskennungen trennt", ["code-version"], "Semver Version parser splits '-' prerelease and '+' build identifiers.", scope="code") pair("code", "version parser that fetches the newest release from an HTTP package registry", "Versionsparser, der die neueste Version aus einer HTTP-Paketregistrierung abruft", [], "Semver parsing exists; no network registry retrieval code in fixed two source files.", scope="code", label="absent") pair("image", "photo of tiny peaks repeating upside down across quiet water", "Foto von Bergspitzen, die sich kopfüber in ruhigem Wasser wiederholen", ["image03"], "Mountain and tree reflections in daylight lake.") pair("image", "photo of an alpine lake with a red canoe and two paddlers", "Foto eines Bergsees mit rotem Kanu und zwei Paddlern", [], "Lake photos exist but no canoe or two paddlers in any photo.", label="absent") pair("video", "video where a feline keeps reaching up at a room's exit", "Video, in dem eine Katze wiederholt am Ausgang eines Zimmers hochspringt", ["clip01"], "Orange tabby twice jumps at white door lever.") pair("video", "video of a cat squeezing through a flap into a snowy garden", "Video einer Katze, die durch eine Klappe in einen verschneiten Garten geht", [], "Door jumping cat exists; no flap, snowy garden or passage through it.", label="absent") pair("music", "music by Kimiko Ishizaka with the thirteenth Goldberg variation", "Musik von Kimiko Ishizaka mit der dreizehnten Goldberg-Variation", ["music01"], "Original Vorbis tags plus publisher description identify performer and composition.") pair("music", "music by Kimiko Ishizaka playing a Goldberg variation on a violin", "Musik von Kimiko Ishizaka mit einer Goldberg-Variation auf der Geige", [], "Kimiko piano and separate jazz violin exist; requested performer/instrument/composition conjunction does not.", label="absent") pair("sound", "audio of a household pet warning someone with repeated barks", "Audio von einem Haustier, das jemanden mit Bellen warnt", ["sound01"], "Publisher dog bark description supports barking; warning intent is colloquial query, not a labelled mood.") pair("sound", "audio of a dog barking while church bells ring behind it", "Audio eines bellenden Hundes mit Kirchenglocken im Hintergrund", [], "Dog bark recording exists; no independently described church bells in any audio.", label="absent") pair("speech", "spoken passage saying people possess an inner moral judgement", "Gesprochene Passage darüber, dass Menschen ein inneres moralisches Urteilsvermögen haben", ["speech-en", "speech-de"], "Article 1 states endowed with reason and conscience.") pair("speech", "spoken explanation of the right to paid holidays and limited working hours", "Gesprochene Erklärung des Rechts auf bezahlten Urlaub und begrenzte Arbeitszeit", [], "Only Article 1 is in speech files; requested Article 24 topic absent from corpus.", label="absent") pair("ambiguous_partial", "pictures of somewhere peaceful near water", "Bilder von einem ruhigen Ort am Wasser", ["image03", "image05", "image11"], "Subjective broad request: lake and beach both plausible; excluded from strict no-answer metric.", label="partial") pair("ambiguous_partial", "what do we know about keeping the loaned exhibit safe", "Was wissen wir darüber, das geliehene Ausstellungsstück sicher zu halten?", ["museum-a"], "Insurance is relevant partial evidence, but no physical packing/security instruction; do not score as absent.", label="partial") fixture = {"schema": 1, "frozen_before_inference": True, "label_policy": "New queries authored without opening inference outputs. Existing source content/independent media descriptions only. One AI annotator, not independent human truth. Never revise labels after outputs. Historical absence20 not reused. Partial queries scored separately. Extraction/indexing failures must be reported separately, not successful abstentions.", "corpus_policy": "All 36 declared files form fixed distractor corpus. Copy only declared source bytes into a dedicated corpus; no repository-wide index. Speech PCM stays ignored and is referenced by exact hash, not added to Git.", "assets": assets, "queries": queries} target = OUT / "fixture.json" assert not target.exists(), "Frozen fixture must not be overwritten" target.write_text(json.dumps(fixture, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") lock = {"fixture_sha256": digest(target), "assets": len(assets), "queries": len(queries), "strict_answerable": sum(q["label"] == "answerable" for q in queries), "strict_absent": sum(q["label"] == "absent" for q in queries), "partial": sum(q["label"] == "partial" for q in queries)} (OUT / "fixture.lock.json").write_text(json.dumps(lock, indent=2) + "\n", encoding="utf-8") print(json.dumps(lock))