karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Validate audio stream timelines and add genuine Google transcript diagnostics

Browse Source
This commit is contained in:
karl committed 2026-10-13 10:39:31 +02:00
1 parent 8ef874c609
commit 5eeba29c0d
7 files changed
+1074 -23
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/speech-eval/run.py
+40 -21
View File
@@ -108,10 +108,10 @@ def recognized_window(raw, offset, length, decode_ms, asr_ms):
return report
def decode_command(ffmpeg, source, offset, length, wav):
def decode_command(ffmpeg, source, offset, length, wav, audio_index=None):
return [str(ffmpeg.resolve()), "-nostdin", "-hide_banner", "-v", "error",
"-protocol_whitelist", "file,pipe", "-threads", "1", "-ss", str(offset / 1000), "-t", str(length / 1000),
"-i", str(source.resolve()), "-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
"-protocol_whitelist", "file,pipe", "-threads", "1", "-seek_timestamp", "1", "-ss", str(offset / 1000), "-t", str(length / 1000),
"-i", str(source.resolve()), "-map", f"0:{audio_index}" if audio_index is not None else "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
"-af", f"aresample=16000,atrim=end_sample={length * 16},asetpts=PTS-STARTPTS",
"-c:a", "pcm_s16le", "-threads", "1", "-y", str(wav)]
@@ -127,6 +127,19 @@ def decoded_frames(wav, length):
return frames
def primary_audio_range(probe):
audio = next((stream for stream in probe.get("streams", []) if stream.get("codec_type") == "audio"), None)
if audio is None:
raise ValueError("No primary audio stream")
start = float(audio.get("start_time", 0))
if "duration" not in audio:
raise ValueError("Primary audio stream duration unavailable; proof requires explicit stream duration, not container fallback")
duration = float(audio["duration"])
if not math.isfinite(start) or not math.isfinite(duration) or start < 0 or not 0 < duration <= 600:
raise ValueError("Proof audio stream range exceeds bounds")
return math.ceil(start * 1000), math.ceil((start + duration) * 1000), audio["index"]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--whisper", type=Path, required=True)
@@ -161,30 +174,36 @@ def main():
raise ValueError(f"Frozen asset mismatch: {asset['id']}")
# Probe stdout is bounded by the same owned log, then parsed directly.
run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe",
"-show_entries", "format=duration", "-of", "json", str(source.resolve())],
"-show_entries", "stream=index,codec_type,duration,start_time:format=duration,start_time", "-of", "json", str(source.resolve())],
scratch, "probe", [], 30)
probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8"))
duration_s = float(probe["format"]["duration"])
if not math.isfinite(duration_s) or not 0 < duration_s <= 600:
raise ValueError("Proof duration exceeds ten-minute bound")
duration_ms = math.ceil(duration_s * 1000)
result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": duration_ms, "windows": []}
for offset in range(0, duration_ms, 30000):
length = min(30000, duration_ms - offset)
start_ms, end_ms, audio_index = primary_audio_range(probe)
result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": end_ms - start_ms,
"audio_start_ms": start_ms, "audio_end_ms": end_ms, "audio_stream_index": audio_index,
"actual_probe": probe, "windows": []}
for offset in range(start_ms, end_ms, 30000):
length = min(30000, end_ms - offset)
wav = scratch / "window.wav"
prefix = scratch / "recognized"
output = prefix.with_suffix(".json")
output.unlink(missing_ok=True)
decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav),
decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav, audio_index),
scratch, "decode", [(wav, 970000)], 60)
frames = decoded_frames(wav, length)
asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
"--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
"--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
raw = json.loads(output.read_text(encoding="utf-8"))
# Preserve genuine invalid output, never clamp it into valid evidence.
result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
result["windows"][-1]["decoded_frames"] = frames
try:
frames = decoded_frames(wav, length)
except ValueError as error:
with wave.open(str(wav), "rb") as audio:
actual_frames = audio.getnframes()
result["windows"].append({"offset_ms": offset, "duration_ms": length, "decode_ms": decode_ms,
"decoded_frames": actual_frames, "validation_error": str(error), "error_stage": "decode_coverage",
"segments": [], "asr_not_run": True})
else:
asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
"--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
"--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
raw = json.loads(output.read_text(encoding="utf-8"))
result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
result["windows"][-1]["decoded_frames"] = frames
args.output.parent.mkdir(parents=True, exist_ok=True)
checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture),
"model_sha256": digest, "worker_sha256": args.whisper_sha256,
@@ -202,7 +221,7 @@ def main():
"ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe),
"backend": "forced CPU", "threads": 1,
"harness_sha256": sha256(Path(__file__)),
"policy": "aresample16000-before-atrim-v2;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
"policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
"rows": rows}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
tools/speech-eval/score.py
+8
View File
@@ -6,6 +6,13 @@ from pathlib import Path
from run import FIXTURE_SHA
def require_eligible(report):
if report.get("complete") is not True or report.get("quality_score_eligible") is not True:
raise ValueError("Cannot score incomplete runs or reduced/invalid competitor pools")
if report.get("excluded_windows") or report.get("excluded_window_count", 0):
raise ValueError("Cannot score rankings with excluded ASR windows")
def score(fixture, rankings):
queries = [query for query in fixture["queries"] if query["category"] == "video-speech"]
if len(queries) != 8:
@@ -41,6 +48,7 @@ def main():
args = parser.parse_args()
data = args.fixture.read_bytes()
rankings = json.loads(args.rankings.read_text(encoding="utf-8"))
require_eligible(rankings)
digest = hashlib.sha256(data).hexdigest()
if digest != FIXTURE_SHA or rankings["fixture_sha256"] != digest or not rankings.get("asr_report_sha256") or not rankings.get("google_worker_sha256"):
raise ValueError("Missing/mismatched actual ASR/Google provenance")
tools/speech-eval/test_contract.py
+40 -2
View File
@@ -6,11 +6,49 @@ import tempfile
import sys
import os
import wave
from run import segments, run_owned, recognized_window, decode_command, decoded_frames
from score import score
from run import segments, run_owned, recognized_window, decode_command, decoded_frames, primary_audio_range
from score import score, require_eligible
class ContractTests(unittest.TestCase):
def test_scoring_rejects_incomplete_or_reduced_pool(self):
for report in [{}, {"complete": False, "quality_score_eligible": True},
{"complete": True, "quality_score_eligible": False},
{"complete": True, "quality_score_eligible": True, "excluded_windows": [{}]}]:
with self.assertRaises(ValueError):
require_eligible(report)
require_eligible({"complete": True, "quality_score_eligible": True, "excluded_windows": []})
def test_primary_audio_duration_and_nonzero_start_ignore_longer_video(self):
probe = {"streams": [{"index": 0, "codec_type": "video", "duration": "90"},
{"index": 1, "codec_type": "audio", "start_time": "0.125", "duration": "31"}]}
self.assertEqual(primary_audio_range(probe), (125, 31125, 1))
@unittest.skipUnless(os.environ.get("SEARCHI_TEST_FFMPEG"), "Requires verified CPU FFmpeg")
def test_actual_video_longer_than_audio(self):
ffmpeg = Path(os.environ["SEARCHI_TEST_FFMPEG"])
ffprobe = ffmpeg.with_name("ffprobe.exe" if os.name == "nt" else "ffprobe")
with tempfile.TemporaryDirectory() as parent:
scratch = Path(parent)
source = scratch / "longer-video.mkv"
run_owned([str(ffmpeg), "-nostdin", "-hide_banner", "-v", "error", "-threads", "1",
"-f", "lavfi", "-i", "color=c=black:s=16x16:r=1:d=33",
"-itsoffset", "0.125", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=mono:d=31", "-c:v", "ffv1",
"-c:a", "pcm_s16le", "-threads", "1", "-output_ts_offset", "5", "-y", str(source)],
scratch, "mux", [(source, 4 * 1024 * 1024)], 30)
run_owned([str(ffprobe), "-v", "error", "-show_entries", "stream=index,codec_type,duration,start_time:stream_tags=DURATION:format=duration", "-of", "json", str(source)], scratch, "probe", [], 30)
probe = json.loads((scratch / "probe.log").read_text())
audio = next(row for row in probe["streams"] if row["codec_type"] == "audio")
# Matroska stores duration in tags; test conversion is explicit, not a format fallback.
if "duration" not in audio:
h, m, sec = audio["tags"]["DURATION"].split(":")
audio["duration"] = int(h) * 3600 + int(m) * 60 + float(sec) - float(audio["start_time"])
start, end, _ = primary_audio_range(probe)
self.assertEqual((start, end), (5125, 36125))
self.assertGreater(float(probe["format"]["duration"]), end / 1000)
wav = scratch / "tail.wav"
run_owned(decode_command(ffmpeg, source, 35125, 1000, wav, audio["index"]), scratch, "tail", [(wav, 970000)], 30)
self.assertGreaterEqual(decoded_frames(wav, 1000), 15984)
@unittest.skipUnless(os.environ.get("SEARCHI_TEST_FFMPEG"), "Requires verified CPU FFmpeg")
def test_actual_ffmpeg_full_window_and_partial_tail_sample_coverage(self):
ffmpeg = Path(os.environ["SEARCHI_TEST_FFMPEG"])