karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Validate audio stream timelines and add genuine Google transcript diagnostics

Browse Source
This commit is contained in:
karl committed 2026-10-13 10:39:31 +02:00
1 parent 8ef874c609
commit 5eeba29c0d
7 files changed
+1074 -23
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/speech-eval/run.py
+40 -21
View File
@@ -108,10 +108,10 @@ def recognized_window(raw, offset, length, decode_ms, asr_ms):
return report
def decode_command(ffmpeg, source, offset, length, wav):
def decode_command(ffmpeg, source, offset, length, wav, audio_index=None):
return [str(ffmpeg.resolve()), "-nostdin", "-hide_banner", "-v", "error",
"-protocol_whitelist", "file,pipe", "-threads", "1", "-ss", str(offset / 1000), "-t", str(length / 1000),
"-i", str(source.resolve()), "-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
"-protocol_whitelist", "file,pipe", "-threads", "1", "-seek_timestamp", "1", "-ss", str(offset / 1000), "-t", str(length / 1000),
"-i", str(source.resolve()), "-map", f"0:{audio_index}" if audio_index is not None else "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
"-af", f"aresample=16000,atrim=end_sample={length * 16},asetpts=PTS-STARTPTS",
"-c:a", "pcm_s16le", "-threads", "1", "-y", str(wav)]
@@ -127,6 +127,19 @@ def decoded_frames(wav, length):
return frames
def primary_audio_range(probe):
audio = next((stream for stream in probe.get("streams", []) if stream.get("codec_type") == "audio"), None)
if audio is None:
raise ValueError("No primary audio stream")
start = float(audio.get("start_time", 0))
if "duration" not in audio:
raise ValueError("Primary audio stream duration unavailable; proof requires explicit stream duration, not container fallback")
duration = float(audio["duration"])
if not math.isfinite(start) or not math.isfinite(duration) or start < 0 or not 0 < duration <= 600:
raise ValueError("Proof audio stream range exceeds bounds")
return math.ceil(start * 1000), math.ceil((start + duration) * 1000), audio["index"]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--whisper", type=Path, required=True)
@@ -161,30 +174,36 @@ def main():
raise ValueError(f"Frozen asset mismatch: {asset['id']}")
# Probe stdout is bounded by the same owned log, then parsed directly.
run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe",
"-show_entries", "format=duration", "-of", "json", str(source.resolve())],
"-show_entries", "stream=index,codec_type,duration,start_time:format=duration,start_time", "-of", "json", str(source.resolve())],
scratch, "probe", [], 30)
probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8"))
duration_s = float(probe["format"]["duration"])
if not math.isfinite(duration_s) or not 0 < duration_s <= 600:
raise ValueError("Proof duration exceeds ten-minute bound")
duration_ms = math.ceil(duration_s * 1000)
result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": duration_ms, "windows": []}
for offset in range(0, duration_ms, 30000):
length = min(30000, duration_ms - offset)
start_ms, end_ms, audio_index = primary_audio_range(probe)
result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": end_ms - start_ms,
"audio_start_ms": start_ms, "audio_end_ms": end_ms, "audio_stream_index": audio_index,
"actual_probe": probe, "windows": []}
for offset in range(start_ms, end_ms, 30000):
length = min(30000, end_ms - offset)
wav = scratch / "window.wav"
prefix = scratch / "recognized"
output = prefix.with_suffix(".json")
output.unlink(missing_ok=True)
decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav),
decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav, audio_index),
scratch, "decode", [(wav, 970000)], 60)
frames = decoded_frames(wav, length)
asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
"--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
"--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
raw = json.loads(output.read_text(encoding="utf-8"))
# Preserve genuine invalid output, never clamp it into valid evidence.
result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
result["windows"][-1]["decoded_frames"] = frames
try:
frames = decoded_frames(wav, length)
except ValueError as error:
with wave.open(str(wav), "rb") as audio:
actual_frames = audio.getnframes()
result["windows"].append({"offset_ms": offset, "duration_ms": length, "decode_ms": decode_ms,
"decoded_frames": actual_frames, "validation_error": str(error), "error_stage": "decode_coverage",
"segments": [], "asr_not_run": True})
else:
asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
"--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
"--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
raw = json.loads(output.read_text(encoding="utf-8"))
result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
result["windows"][-1]["decoded_frames"] = frames
args.output.parent.mkdir(parents=True, exist_ok=True)
checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture),
"model_sha256": digest, "worker_sha256": args.whisper_sha256,
@@ -202,7 +221,7 @@ def main():
"ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe),
"backend": "forced CPU", "threads": 1,
"harness_sha256": sha256(Path(__file__)),
"policy": "aresample16000-before-atrim-v2;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
"policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
"rows": rows}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")