diff --git a/.gitattributes b/.gitattributes index 4f13249..8917295 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,3 +11,5 @@ fixtures/mixed/sources/** -text fixtures/music-validation-v1/*.json -text whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol tools/rerank-eval/*-plan.json -text whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol tools/rerank-eval/*-manifest.json -text whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol + +fixtures/speech-validation-v1/*.json -text whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol diff --git a/fixtures/speech-validation-v1/README.md b/fixtures/speech-validation-v1/README.md new file mode 100644 index 0000000..ae31c2e --- /dev/null +++ b/fixtures/speech-validation-v1/README.md @@ -0,0 +1,100 @@ +# Independent English/German speech preparation v1 + +No ASR, VAD or Google inference has run on this fixture. The ten topic/absence +queries and two source recordings were frozen before preparation. The sources +are human LibriVox readings of Article 1 of the Universal Declaration of Human +Rights, unrelated to the earlier NASA topics. They share translated content and +are correlated; this is not a representative multilingual speech benchmark. + +| Source | Human reader | Recording permission | Original bytes | +| --- | --- | --- | ---: | +| [English Article 1](https://commons.wikimedia.org/wiki/File:Universal_Declaration_of_Human_Rights_-_eng_-_drb_-_Art1.ogg) | David Barnes | LibriVox public-domain dedication / copyrighted free use | 168186 | +| [German Article 1](https://commons.wikimedia.org/wiki/File:Universal_Declaration_of_Human_Rights_-_ger_-_robu_-_Art1.ogg) | Roman Buettner | LibriVox public-domain dedication / copyrighted free use | 151136 | + +The publisher pages explicitly identify the human readers, document and language. +The primary reference text is [UN Article 1 in English](https://www.un.org/about-us/universal-declaration-of-human-rights) +and [UNRIC Article 1 in German](https://unric.org/de/allgemeine-erklaerung-menschenrechte/). +The manifest records reference keywords and topic paraphrases, never model +captions or recognition-derived ground truth. Full primary text is externally +linked; it must not be supplied as an ASR prompt or indexed as if recognized. + +Only **319322 new bytes** were downloaded. Sources are checked against official +Commons API size/SHA-1 and acquired SHA-256. The existing independently licensed +US Navy drum cadence supplies the instrumental control, with its previously +locked source bytes. Including that reused source, original inputs total +1893869 bytes. Generated silence is exact zero PCM, released CC0. + +## Four-clip pool and honest timestamp scope + +Prepared audio is mono 16-kHz s16 WAV. Each human recording remains complete, +with five seconds of exact leading silence; no reference words are cropped. +The resulting English/German durations are 22.512 and 16.219 seconds. Music and +silence controls each cover exactly thirty seconds. Core WAVs total 3159568 bytes. +The music waveform is the first thirty seconds of the already locked drum source; +no descriptive tags are used to gate recognition. + +Actual source audio stream starts are verified as zero, and the five-second +padding/PCM source mapping is exact. **The publishers do not provide phrase-level +timed captions.** Neither the padding boundary nor the whole article bounds prove +the first phoneme or each sentence's start. Strict utterance-point scoring is +explicitly unavailable; do not count a start anywhere in the article as precise +localization success. Independent listening/timed annotations remain required. + +Six positive EN/DE queries describe equal dignity/rights, reason/conscience and +brotherly treatment. Either language's recording is relevant: translated topic +equivalence is intentional. Four absent queries ask about Mars funding and salary/ +paid leave. Silence and instrumental music remain in every candidate pool, and +any recognized hallucination remains an explicit measured failure/competitor. +Per-language recognized concept coverage is separate from semantic file recall. + +## Separate seam primitives + +`seam-plan.json` freezes two artificial repeat controls, outside the four-clip +retrieval pool. English source samples repeat twice and German samples three +times after the same known padding, producing 40.024 and 38.657 seconds. Their +2517880 additional WAV bytes and exact piecewise source maps stress a 30-second +window with two-second overlap. These joins are artificial repetitions, not +natural uninterrupted speech. Do not extrapolate their quality to long talks. + +Independent byte checks verified each derived piece equals the original decoded +source PCM and the prefix is all zeros. Compare full-source VAD on these short +controls with bounded independent windows, recording interval disagreement, +source coverage, seam omissions/duplicates and processing cost. Restarted VAD +need not reproduce a continuously initialized VAD's probabilities. Preserve +differences rather than changing thresholds or treating an unverified reference +as infallible. The same labels must not choose gate intervals or discard a seam. + +## Preparation and acceptance + +```powershell +python -m unittest discover -s tools/speech-eval -p test_prepare_validation.py +python tools/speech-eval/prepare_validation.py fixtures/speech-validation-v1/fixture.json .searchi/speech-validation-v1-seams .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe +``` + +Preparation already completed; existing files are preserved, so use a fresh +owned directory for another preparation. Five structural tests passed. Exact +padding and every seam's source pieces were checked independently without a +model. No recognition/quality score follows. Hashes are frozen in `inputs.lock.json`: + +- Fixture: `e3a850d7a2385e250408f57fe51f4620ce80a20f8625ced42205f02cc46d517d`. +- Seam plan: `5da4400814c4965fc12a87bb00c36665300730a702a69acd65a72b9f5201ac9e`. +- Prepared `.searchi/speech-validation-v1-seams/prepared.json`: + `085f34f0af863656a28e7a905746973a54d1123bb07053541e09c8af0a58bb3f`. + +The earlier four-clip-only plan +`4635e772834b0eed847459f5f4d47d7c99a591f0adba2c60ce75712557092279` +is preserved under its original ignored directory and was never inferred. + +Before accepted scoring, require every source/window and all ten queries, strict +UTF-8 and bounded times, zero excluded failed competitors, unchanged worker/model/ +decoder hashes and exact source-byte provenance. Report genuine German recognition +and all preregistered reference concepts, source-level recall/MRR, absent-return +rate and gate false positives separately. Both negative controls must retain +zero accepted recognized speech segments; raw Google audio search stays available. +Measure decode/VAD/ASR/alignment/Google/storage, cold model load, foreground impact +and warm reuse separately. ASR remains bounded to thirty seconds even for seam +controls. Pause between work units for foreground priority; do not claim native +preemption during one inference call. + +This plan does not adopt a provider, change production search or close fine +timestamp, long-source, accent, representative-noise or human-listening gates. diff --git a/fixtures/speech-validation-v1/fixture.json b/fixtures/speech-validation-v1/fixture.json new file mode 100644 index 0000000..e556651 --- /dev/null +++ b/fixtures/speech-validation-v1/fixture.json @@ -0,0 +1,185 @@ +{ + "schema": 1, + "frozen_before_inference": true, + "assets": [ + { + "id": "speech-en", + "kind": "speech", + "language": "en", + "source_url": "https://upload.wikimedia.org/wikipedia/commons/e/e7/Universal_Declaration_of_Human_Rights_-_eng_-_drb_-_Art1.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Universal_Declaration_of_Human_Rights_-_eng_-_drb_-_Art1.ogg", + "bytes": 168186, + "sha1": "4bfeec5eab9837eff3ca8318f2824e7e6982526d", + "license": "LibriVox public-domain dedication and copyrighted free use", + "author": "David Barnes", + "license_url": "https://librivox.org/pages/public-domain/", + "reference_text_url": "https://www.un.org/about-us/universal-declaration-of-human-rights", + "reference_section": "Article1", + "reference_keywords": [ + "free", + "equal", + "dignity", + "rights", + "reason", + "conscience", + "brotherhood" + ], + "prefix_silence_ms": 5000, + "fine_utterance_timestamps_available": false, + "speech_offset_note": "Exact padding5000ms does not establish first phoneme or individual sentence times." + }, + { + "id": "speech-de", + "kind": "speech", + "language": "de", + "source_url": "https://upload.wikimedia.org/wikipedia/commons/d/d8/Universal_Declaration_of_Human_Rights_-_ger_-_robu_-_Art1.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Universal_Declaration_of_Human_Rights_-_ger_-_robu_-_Art1.ogg", + "bytes": 151136, + "sha1": "7f5d3620b4939d44849639c10b415920071353a3", + "license": "LibriVox public-domain dedication and copyrighted free use", + "author": "Roman Buettner", + "license_url": "https://librivox.org/pages/public-domain/", + "reference_text_url": "https://unric.org/de/allgemeine-erklaerung-menschenrechte/", + "reference_section": "Article1", + "reference_keywords": [ + "frei", + "gleich", + "Würde", + "Rechten", + "Vernunft", + "Gewissen", + "Brüderlichkeit" + ], + "prefix_silence_ms": 5000, + "fine_utterance_timestamps_available": false, + "speech_offset_note": "Exact padding5000ms does not establish first phoneme or individual sentence times." + } + ], + "controls": [ + { + "id": "music-control", + "kind": "instrumental-negative", + "source_fixture": "fixtures/music-validation-v1/fixture.json", + "source_id": "unseen03", + "source_bytes": 1574547, + "source_sha256": "5083f07730c6ffb1cd2c3b5cc1a43714a0cfda04fe913935e83d1048466562e6", + "license": "PD-USGov USNavy", + "clip_duration_ms": 30000 + }, + { + "id": "silence-control", + "kind": "silence-negative", + "generation": "480000 exact zero s16 samples at16kHz mono", + "clip_duration_ms": 30000, + "license": "CC0-1.0" + } + ], + "queries": [ + { + "id": "equal-en", + "text": "someone explains that every person has equal dignity and rights", + "language": "en", + "expected": [ + "speech-en", + "speech-de" + ], + "category": "spoken-topic", + "strict_point_score_eligible": false + }, + { + "id": "equal-de", + "text": "jemand sagt dass alle Menschen gleiche Würde und Rechte haben", + "language": "de", + "expected": [ + "speech-en", + "speech-de" + ], + "category": "spoken-topic", + "strict_point_score_eligible": false + }, + { + "id": "reason-en", + "text": "a speaker mentions reason and conscience", + "language": "en", + "expected": [ + "speech-en", + "speech-de" + ], + "category": "spoken-topic", + "strict_point_score_eligible": false + }, + { + "id": "reason-de", + "text": "eine Person spricht über Vernunft und Gewissen", + "language": "de", + "expected": [ + "speech-en", + "speech-de" + ], + "category": "spoken-topic", + "strict_point_score_eligible": false + }, + { + "id": "solidarity-en", + "text": "people should treat one another with brotherhood", + "language": "en", + "expected": [ + "speech-en", + "speech-de" + ], + "category": "spoken-topic", + "strict_point_score_eligible": false + }, + { + "id": "solidarity-de", + "text": "Menschen sollen einander mit Brüderlichkeit begegnen", + "language": "de", + "expected": [ + "speech-en", + "speech-de" + ], + "category": "spoken-topic", + "strict_point_score_eligible": false + }, + { + "id": "absent-mars-en", + "text": "a speaker explains funding for a mission to Mars", + "language": "en", + "expected": [], + "category": "absent-topic", + "strict_point_score_eligible": false + }, + { + "id": "absent-mars-de", + "text": "jemand erklärt die Finanzierung einer Marsmission", + "language": "de", + "expected": [], + "category": "absent-topic", + "strict_point_score_eligible": false + }, + { + "id": "absent-salary-en", + "text": "someone explains their annual salary and paid leave", + "language": "en", + "expected": [], + "category": "absent-topic", + "strict_point_score_eligible": false + }, + { + "id": "absent-salary-de", + "text": "eine Person erklärt ihr Jahresgehalt und den bezahlten Urlaub", + "language": "de", + "expected": [], + "category": "absent-topic", + "strict_point_score_eligible": false + } + ], + "acceptance": { + "complete_pool": "All4clips and all10queries retained; zero excluded invalid windows; invalid outputs fail accepted scoring", + "recognition": "Report per-language transcript/reference concept coverage separately; publisher references never become inference prompts or indexed passages", + "retrieval": "ActualGoogle textadapter, allrecognized passages inclnegativehallucinations; positive fileR1/R5+MRR and absentreturn separate", + "timing": "No fine utterance publisher captions: precise startscore unavailable, do not credit wholeclip as precise. Independent listening/timedlabels required before fine localization acceptance", + "gate": "Zero accepted recognized segments for exactsilence and instrumentalcontrol; retain gatingdecision/source and failures; no endpoint/rawGoogle suppression", + "runtime": "CPU1thread on VM; ASRwindowmax30s; repeatedoperation/sourcehash guard/explicitoffsetmap; coldload separately measured" + } +} diff --git a/fixtures/speech-validation-v1/inputs.lock.json b/fixtures/speech-validation-v1/inputs.lock.json new file mode 100644 index 0000000..a2593ec --- /dev/null +++ b/fixtures/speech-validation-v1/inputs.lock.json @@ -0,0 +1,119 @@ +{ + "fixture_sha256": "e3a850d7a2385e250408f57fe51f4620ce80a20f8625ced42205f02cc46d517d", + "seam_plan_sha256": "5da4400814c4965fc12a87bb00c36665300730a702a69acd65a72b9f5201ac9e", + "prepared_sha256": "085f34f0af863656a28e7a905746973a54d1123bb07053541e09c8af0a58bb3f", + "frozen_before_inference": true, + "model_inference_performed": false, + "clips": [ + { + "id": "speech-en", + "sha256": "0aa1fa97d367d987bfb3bb01d7eca11db77a544552ccb8dc0a78ad74d0b496ea", + "samples": 360192, + "source_sha256": "d69ddbb1dbf64bae10bb03ff74edc840b08c1cd08c172ca524fa73cb2c98ee48", + "source_audio_start_ms": 0.0, + "prefix_silence_ms": 5000 + }, + { + "id": "speech-de", + "sha256": "4af09fe7d80056600e4c288161697a094108f343b069ffd581ccabc06a048afe", + "samples": 259504, + "source_sha256": "fb180307f9d14ced5687c6d4a59d389508d952ead568c2b83b7b8eea65e58f61", + "source_audio_start_ms": 0.0, + "prefix_silence_ms": 5000 + }, + { + "id": "music-control", + "sha256": "af76cb9e35948fcf81c72f5dcb95165c578a8bcc28d756c96aef0e3c1466b8f5", + "samples": 480000, + "source_audio_start_ms": 0 + }, + { + "id": "silence-control", + "sha256": "40abc924dc63f75aafb2400bb99fd66741e22eb9ff2450973a18db53a06286ac", + "samples": 480000, + "source_audio_start_ms": 0 + } + ], + "seam_controls": [ + { + "id": "seam-en", + "sha256": "d4553fac71e86adbd4eced7a1c4750782c63935501446697cf5bc7b48870b31d", + "source_sha256": "d69ddbb1dbf64bae10bb03ff74edc840b08c1cd08c172ca524fa73cb2c98ee48", + "samples": 640384, + "piecewise_map": [ + { + "derived_samples": [ + 80000, + 360192 + ], + "source": "speech-en", + "source_samples": [ + 0, + 280192 + ], + "copy_index": 0 + }, + { + "derived_samples": [ + 360192, + 640384 + ], + "source": "speech-en", + "source_samples": [ + 0, + 280192 + ], + "copy_index": 1 + } + ], + "prefix_silence_samples": 80000 + }, + { + "id": "seam-de", + "sha256": "6b2adaab5fd505dd313b49b8878fc0b372b5141d80c589dcf06fba37ad699c00", + "source_sha256": "fb180307f9d14ced5687c6d4a59d389508d952ead568c2b83b7b8eea65e58f61", + "samples": 618512, + "piecewise_map": [ + { + "derived_samples": [ + 80000, + 259504 + ], + "source": "speech-de", + "source_samples": [ + 0, + 179504 + ], + "copy_index": 0 + }, + { + "derived_samples": [ + 259504, + 439008 + ], + "source": "speech-de", + "source_samples": [ + 0, + 179504 + ], + "copy_index": 1 + }, + { + "derived_samples": [ + 439008, + 618512 + ], + "source": "speech-de", + "source_samples": [ + 0, + 179504 + ], + "copy_index": 2 + } + ], + "prefix_silence_samples": 80000 + } + ], + "ffmpeg_sha256": "b778ca5786ee4b0b4f37f34478617093d2fdb9a532d281fb76cda69d1d579d46", + "ffprobe_sha256": "4e1b0afd27bf02d79739f27b2d650137470925afb19d54b5e89a708b7a389236" +} diff --git a/fixtures/speech-validation-v1/labels.lock.json b/fixtures/speech-validation-v1/labels.lock.json new file mode 100644 index 0000000..40e770c --- /dev/null +++ b/fixtures/speech-validation-v1/labels.lock.json @@ -0,0 +1,7 @@ +{ + "fixture_sha256": "e3a850d7a2385e250408f57fe51f4620ce80a20f8625ced42205f02cc46d517d", + "frozen_before_inference": true, + "source_count": 2, + "derived_clip_count": 4, + "query_count": 10 +} diff --git a/fixtures/speech-validation-v1/seam-plan.json b/fixtures/speech-validation-v1/seam-plan.json new file mode 100644 index 0000000..52f89ad --- /dev/null +++ b/fixtures/speech-validation-v1/seam-plan.json @@ -0,0 +1,23 @@ +{ + "schema": 1, + "frozen_before_inference": true, + "purpose": "Artificial repeat/offset primitive stress, separate from4clipretrievalpool; no sentence-precise labels", + "policy": "ReferencefullsourceVAD versus independent30s VAD windows with2s overlap; no parityassumption or postscore parameter tuning", + "recipes": [ + { + "id": "seam-en", + "source": "speech-en", + "copies": 2, + "prefix_silence_ms": 5000, + "maximum_duration_ms": 45000 + }, + { + "id": "seam-de", + "source": "speech-de", + "copies": 3, + "prefix_silence_ms": 5000, + "maximum_duration_ms": 45000 + } + ], + "acceptance": "Report sourcecoverage/intervalmembership disagreement, boundary omissions/duplicates and cancellation/failure before publication; map padding exactlyonce, preserve known source offsets and no label-selected speech interval. Any whole-file probability difference is a finding, not automaticallya correctnesserror; functional acceptance requires no missing sourcewindows, validbounds and independently assessed speechrecall. Human finephoneme labels unavailable; do not claim precise localizedquality." +} diff --git a/fixtures/speech-validation-v1/seams.lock.json b/fixtures/speech-validation-v1/seams.lock.json new file mode 100644 index 0000000..45af84e --- /dev/null +++ b/fixtures/speech-validation-v1/seams.lock.json @@ -0,0 +1,4 @@ +{ + "seam_plan_sha256": "5da4400814c4965fc12a87bb00c36665300730a702a69acd65a72b9f5201ac9e", + "frozen_before_inference": true +} diff --git a/tools/speech-eval/prepare_validation.py b/tools/speech-eval/prepare_validation.py new file mode 100644 index 0000000..51a299c --- /dev/null +++ b/tools/speech-eval/prepare_validation.py @@ -0,0 +1,105 @@ +"""Acquire frozen small human speech inputs and prepare controls without inference.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import urllib.request +import wave + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def execute(args): + return subprocess.run(args, capture_output=True, check=True, timeout=60).stdout + + +def write_wav(path, samples, prefix_ms=0, maximum_ms=30000): + if not 0 < maximum_ms <= 45000 or len(samples) % 2 or not samples or len(samples) // 2 + prefix_ms * 16 > maximum_ms * 16: + raise ValueError("Invalid or oversized 30-second source") + if path.exists(): + raise ValueError("Preserve existing prepared clip") + with wave.open(str(path), "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(16000) + output.writeframes(bytes(prefix_ms * 32) + samples) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("fixture", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("ffmpeg", type=Path) + parser.add_argument("ffprobe", type=Path) + args = parser.parse_args() + raw = args.fixture.read_bytes() + fixture = json.loads(raw) + lock = json.loads(args.fixture.with_name("labels.lock.json").read_text()) + if sha(raw) != lock["fixture_sha256"]: + raise ValueError("Frozen plan changed") + args.output.mkdir(parents=True, exist_ok=True) + plan_path = args.output / "prepared.json" + if plan_path.exists(): + raise ValueError("Preserve existing preparation") + prepared = [] + for asset in fixture["assets"]: + path = args.output / (asset["id"] + "-source.ogg") + if not path.exists(): + request = urllib.request.Request(asset["source_url"] + "?download=1", headers={"User-Agent": "SearchiMediaEvaluation/1.0"}) + with urllib.request.urlopen(request, timeout=60) as response: + data = response.read(asset["bytes"] + 1) + if len(data) != asset["bytes"] or hashlib.sha1(data).hexdigest() != asset["sha1"]: + raise ValueError("Publisher source changed") + path.write_bytes(data) + original = path.read_bytes() + if len(original) != asset["bytes"] or hashlib.sha1(original).hexdigest() != asset["sha1"]: + raise ValueError("Cached source changed") + source_probe = json.loads(execute([str(args.ffprobe), "-v", "error", "-show_format", "-show_streams", "-of", "json", str(path)])) + audio = next(s for s in source_probe["streams"] if s["codec_type"] == "audio") + if "duration" not in audio or float(audio["duration"]) + asset["prefix_silence_ms"] / 1000 > 30: + raise ValueError("Full primary speech stream must fit 30 seconds; never silently crop reference speech") + samples = execute([str(args.ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-i", str(path), "-ac", "1", "-ar", "16000", "-f", "s16le", "pipe:1"]) + output = args.output / (asset["id"] + ".wav") + write_wav(output, samples, asset["prefix_silence_ms"]) + prepared.append({"id": asset["id"], "path": str(output.resolve()), "sha256": sha(output.read_bytes()), "samples": len(samples) // 2 + asset["prefix_silence_ms"] * 16, "source_sha256": sha(original), "source_probe": source_probe, "source_audio_start_ms": float(audio.get("start_time", "0")) * 1000, "prefix_silence_ms": asset["prefix_silence_ms"], "fine_utterance_timestamps_available": False}) + music_source = Path(".searchi/qa/music-validation-v1-canonical/unseen03-source.ogg") + expected_music = fixture["controls"][0] + if sha(music_source.read_bytes()) != expected_music["source_sha256"]: + raise ValueError("Frozen instrumental control changed") + samples = execute([str(args.ffmpeg), "-v", "error", "-nostdin", "-threads", "1", "-i", str(music_source), "-t", "30", "-ac", "1", "-ar", "16000", "-af", "aresample=16000,atrim=end_sample=480000,asetpts=PTS-STARTPTS", "-f", "s16le", "pipe:1"]) + if len(samples) != 960000: + raise ValueError("Instrumental control must cover full 30 seconds") + for id, data in [("music-control", samples), ("silence-control", bytes(960000))]: + output = args.output / (id + ".wav") + write_wav(output, data) + prepared.append({"id": id, "path": str(output.resolve()), "sha256": sha(output.read_bytes()), "samples": 480000, "source_audio_start_ms": 0, "expected_recognized_segments": 0}) + for item in prepared: + item["actual_probe"] = json.loads(execute([str(args.ffprobe), "-v", "error", "-show_format", "-show_streams", "-of", "json", item["path"]])) + seam_raw = args.fixture.with_name("seam-plan.json").read_bytes() + seam_plan = json.loads(seam_raw) + seam_lock = json.loads(args.fixture.with_name("seams.lock.json").read_text()) + if sha(seam_raw) != seam_lock["seam_plan_sha256"]: + raise ValueError("Frozen seam plan changed") + seams = [] + for recipe in seam_plan["recipes"]: + source = next(item for item in prepared if item["id"] == recipe["source"]) + with wave.open(source["path"], "rb") as clip: + clip.readframes(source["prefix_silence_ms"] * 16) + samples = clip.readframes(clip.getnframes()) + path = args.output / (recipe["id"] + ".wav") + write_wav(path, samples * recipe["copies"], recipe["prefix_silence_ms"], recipe["maximum_duration_ms"]) + count = len(samples) // 2 + prefix = recipe["prefix_silence_ms"] * 16 + mapping = [{"derived_samples": [prefix + index * count, prefix + (index + 1) * count], "source": recipe["source"], "source_samples": [0, count], "copy_index": index} for index in range(recipe["copies"])] + seams.append({"id": recipe["id"], "path": str(path.resolve()), "sha256": sha(path.read_bytes()), "source_sha256": source["source_sha256"], "samples": prefix + count * recipe["copies"], "piecewise_map": mapping, "prefix_silence_samples": prefix, "artificial_repeat_control": True}) + result = {"fixture_sha256": sha(raw), "seam_plan_sha256": sha(seam_raw), "frozen_before_inference": True, "model_inference_performed": False, "ffmpeg_sha256": sha(args.ffmpeg.read_bytes()), "ffprobe_sha256": sha(args.ffprobe.read_bytes()), "clips": prepared, "seam_controls": seams} + with plan_path.open("x", encoding="utf-8") as stream: + json.dump(result, stream, ensure_ascii=False, indent=2) + print("Prepared four clips without inference; plan SHA256", sha(plan_path.read_bytes())) + + +if __name__ == "__main__": + main() diff --git a/tools/speech-eval/test_prepare_validation.py b/tools/speech-eval/test_prepare_validation.py new file mode 100644 index 0000000..f6f61d9 --- /dev/null +++ b/tools/speech-eval/test_prepare_validation.py @@ -0,0 +1,45 @@ +import tempfile +import unittest +import wave +from pathlib import Path + +from prepare_validation import write_wav + + +class PreparationTests(unittest.TestCase): + def test_known_padding_does_not_change_source_samples(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "clip.wav" + samples = b"\x01\x00" * 16000 + write_wav(path, samples, 5000) + with wave.open(str(path), "rb") as clip: + self.assertEqual(clip.getnframes(), 96000) + self.assertEqual(clip.readframes(80000), bytes(160000)) + self.assertEqual(clip.readframes(16000), samples) + + def test_full_source_over_budget_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ValueError): + write_wav(Path(directory) / "clip.wav", bytes(960000), 5000) + + def test_existing_preparation_cannot_be_overwritten(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "clip.wav" + path.write_bytes(b"original") + with self.assertRaises(ValueError): + write_wav(path, bytes(32000)) + self.assertEqual(path.read_bytes(), b"original") + + def test_odd_pcm_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ValueError): + write_wav(Path(directory) / "clip.wav", b"\x00") + + def test_explicit_seam_budget_is_bounded(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "seam.wav" + write_wav(path, bytes(32000 * 35), maximum_ms=45000) + with wave.open(str(path), "rb") as clip: + self.assertEqual(clip.getnframes(), 16000 * 35) + with self.assertRaises(ValueError): + write_wav(Path(directory) / "oversized.wav", bytes(32000), maximum_ms=45001)