From 5eeba29c0de7601442f201217be70d8dfa7c17ca Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 10:39:31 +0200 Subject: [PATCH] Validate audio stream timelines and add genuine Google transcript diagnostics --- tools/speech-eval/run.py | 61 ++-- tools/speech-eval/score.py | 8 + tools/speech-eval/test_contract.py | 42 ++- tools/speech-rank/Cargo.lock | 486 +++++++++++++++++++++++++++++ tools/speech-rank/Cargo.toml | 12 + tools/speech-rank/README.md | 48 +++ tools/speech-rank/src/main.rs | 440 ++++++++++++++++++++++++++ 7 files changed, 1074 insertions(+), 23 deletions(-) create mode 100644 tools/speech-rank/Cargo.lock create mode 100644 tools/speech-rank/Cargo.toml create mode 100644 tools/speech-rank/README.md create mode 100644 tools/speech-rank/src/main.rs diff --git a/tools/speech-eval/run.py b/tools/speech-eval/run.py index 830b21e..ebd7057 100644 --- a/tools/speech-eval/run.py +++ b/tools/speech-eval/run.py @@ -108,10 +108,10 @@ def recognized_window(raw, offset, length, decode_ms, asr_ms): return report -def decode_command(ffmpeg, source, offset, length, wav): +def decode_command(ffmpeg, source, offset, length, wav, audio_index=None): return [str(ffmpeg.resolve()), "-nostdin", "-hide_banner", "-v", "error", - "-protocol_whitelist", "file,pipe", "-threads", "1", "-ss", str(offset / 1000), "-t", str(length / 1000), - "-i", str(source.resolve()), "-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000", + "-protocol_whitelist", "file,pipe", "-threads", "1", "-seek_timestamp", "1", "-ss", str(offset / 1000), "-t", str(length / 1000), + "-i", str(source.resolve()), "-map", f"0:{audio_index}" if audio_index is not None else "0:a:0", "-vn", "-ac", "1", "-ar", "16000", "-af", f"aresample=16000,atrim=end_sample={length * 16},asetpts=PTS-STARTPTS", "-c:a", "pcm_s16le", "-threads", "1", "-y", str(wav)] @@ -127,6 +127,19 @@ def decoded_frames(wav, length): return frames +def primary_audio_range(probe): + audio = next((stream for stream in probe.get("streams", []) if stream.get("codec_type") == "audio"), None) + if audio is None: + raise ValueError("No primary audio stream") + start = float(audio.get("start_time", 0)) + if "duration" not in audio: + raise ValueError("Primary audio stream duration unavailable; proof requires explicit stream duration, not container fallback") + duration = float(audio["duration"]) + if not math.isfinite(start) or not math.isfinite(duration) or start < 0 or not 0 < duration <= 600: + raise ValueError("Proof audio stream range exceeds bounds") + return math.ceil(start * 1000), math.ceil((start + duration) * 1000), audio["index"] + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--whisper", type=Path, required=True) @@ -161,30 +174,36 @@ def main(): raise ValueError(f"Frozen asset mismatch: {asset['id']}") # Probe stdout is bounded by the same owned log, then parsed directly. run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe", - "-show_entries", "format=duration", "-of", "json", str(source.resolve())], + "-show_entries", "stream=index,codec_type,duration,start_time:format=duration,start_time", "-of", "json", str(source.resolve())], scratch, "probe", [], 30) probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8")) - duration_s = float(probe["format"]["duration"]) - if not math.isfinite(duration_s) or not 0 < duration_s <= 600: - raise ValueError("Proof duration exceeds ten-minute bound") - duration_ms = math.ceil(duration_s * 1000) - result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": duration_ms, "windows": []} - for offset in range(0, duration_ms, 30000): - length = min(30000, duration_ms - offset) + start_ms, end_ms, audio_index = primary_audio_range(probe) + result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": end_ms - start_ms, + "audio_start_ms": start_ms, "audio_end_ms": end_ms, "audio_stream_index": audio_index, + "actual_probe": probe, "windows": []} + for offset in range(start_ms, end_ms, 30000): + length = min(30000, end_ms - offset) wav = scratch / "window.wav" prefix = scratch / "recognized" output = prefix.with_suffix(".json") output.unlink(missing_ok=True) - decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav), + decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav, audio_index), scratch, "decode", [(wav, 970000)], 60) - frames = decoded_frames(wav, length) - asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1", - "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav), - "--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600) - raw = json.loads(output.read_text(encoding="utf-8")) - # Preserve genuine invalid output, never clamp it into valid evidence. - result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms)) - result["windows"][-1]["decoded_frames"] = frames + try: + frames = decoded_frames(wav, length) + except ValueError as error: + with wave.open(str(wav), "rb") as audio: + actual_frames = audio.getnframes() + result["windows"].append({"offset_ms": offset, "duration_ms": length, "decode_ms": decode_ms, + "decoded_frames": actual_frames, "validation_error": str(error), "error_stage": "decode_coverage", + "segments": [], "asr_not_run": True}) + else: + asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1", + "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav), + "--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600) + raw = json.loads(output.read_text(encoding="utf-8")) + result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms)) + result["windows"][-1]["decoded_frames"] = frames args.output.parent.mkdir(parents=True, exist_ok=True) checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture), "model_sha256": digest, "worker_sha256": args.whisper_sha256, @@ -202,7 +221,7 @@ def main(): "ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe), "backend": "forced CPU", "threads": 1, "harness_sha256": sha256(Path(__file__)), - "policy": "aresample16000-before-atrim-v2;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times", + "policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times", "rows": rows} args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") diff --git a/tools/speech-eval/score.py b/tools/speech-eval/score.py index ba93d0f..db79c16 100644 --- a/tools/speech-eval/score.py +++ b/tools/speech-eval/score.py @@ -6,6 +6,13 @@ from pathlib import Path from run import FIXTURE_SHA +def require_eligible(report): + if report.get("complete") is not True or report.get("quality_score_eligible") is not True: + raise ValueError("Cannot score incomplete runs or reduced/invalid competitor pools") + if report.get("excluded_windows") or report.get("excluded_window_count", 0): + raise ValueError("Cannot score rankings with excluded ASR windows") + + def score(fixture, rankings): queries = [query for query in fixture["queries"] if query["category"] == "video-speech"] if len(queries) != 8: @@ -41,6 +48,7 @@ def main(): args = parser.parse_args() data = args.fixture.read_bytes() rankings = json.loads(args.rankings.read_text(encoding="utf-8")) + require_eligible(rankings) digest = hashlib.sha256(data).hexdigest() if digest != FIXTURE_SHA or rankings["fixture_sha256"] != digest or not rankings.get("asr_report_sha256") or not rankings.get("google_worker_sha256"): raise ValueError("Missing/mismatched actual ASR/Google provenance") diff --git a/tools/speech-eval/test_contract.py b/tools/speech-eval/test_contract.py index 2952550..5954172 100644 --- a/tools/speech-eval/test_contract.py +++ b/tools/speech-eval/test_contract.py @@ -6,11 +6,49 @@ import tempfile import sys import os import wave -from run import segments, run_owned, recognized_window, decode_command, decoded_frames -from score import score +from run import segments, run_owned, recognized_window, decode_command, decoded_frames, primary_audio_range +from score import score, require_eligible class ContractTests(unittest.TestCase): + def test_scoring_rejects_incomplete_or_reduced_pool(self): + for report in [{}, {"complete": False, "quality_score_eligible": True}, + {"complete": True, "quality_score_eligible": False}, + {"complete": True, "quality_score_eligible": True, "excluded_windows": [{}]}]: + with self.assertRaises(ValueError): + require_eligible(report) + require_eligible({"complete": True, "quality_score_eligible": True, "excluded_windows": []}) + def test_primary_audio_duration_and_nonzero_start_ignore_longer_video(self): + probe = {"streams": [{"index": 0, "codec_type": "video", "duration": "90"}, + {"index": 1, "codec_type": "audio", "start_time": "0.125", "duration": "31"}]} + self.assertEqual(primary_audio_range(probe), (125, 31125, 1)) + + @unittest.skipUnless(os.environ.get("SEARCHI_TEST_FFMPEG"), "Requires verified CPU FFmpeg") + def test_actual_video_longer_than_audio(self): + ffmpeg = Path(os.environ["SEARCHI_TEST_FFMPEG"]) + ffprobe = ffmpeg.with_name("ffprobe.exe" if os.name == "nt" else "ffprobe") + with tempfile.TemporaryDirectory() as parent: + scratch = Path(parent) + source = scratch / "longer-video.mkv" + run_owned([str(ffmpeg), "-nostdin", "-hide_banner", "-v", "error", "-threads", "1", + "-f", "lavfi", "-i", "color=c=black:s=16x16:r=1:d=33", + "-itsoffset", "0.125", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=mono:d=31", "-c:v", "ffv1", + "-c:a", "pcm_s16le", "-threads", "1", "-output_ts_offset", "5", "-y", str(source)], + scratch, "mux", [(source, 4 * 1024 * 1024)], 30) + run_owned([str(ffprobe), "-v", "error", "-show_entries", "stream=index,codec_type,duration,start_time:stream_tags=DURATION:format=duration", "-of", "json", str(source)], scratch, "probe", [], 30) + probe = json.loads((scratch / "probe.log").read_text()) + audio = next(row for row in probe["streams"] if row["codec_type"] == "audio") + # Matroska stores duration in tags; test conversion is explicit, not a format fallback. + if "duration" not in audio: + h, m, sec = audio["tags"]["DURATION"].split(":") + audio["duration"] = int(h) * 3600 + int(m) * 60 + float(sec) - float(audio["start_time"]) + start, end, _ = primary_audio_range(probe) + self.assertEqual((start, end), (5125, 36125)) + self.assertGreater(float(probe["format"]["duration"]), end / 1000) + wav = scratch / "tail.wav" + run_owned(decode_command(ffmpeg, source, 35125, 1000, wav, audio["index"]), scratch, "tail", [(wav, 970000)], 30) + self.assertGreaterEqual(decoded_frames(wav, 1000), 15984) + @unittest.skipUnless(os.environ.get("SEARCHI_TEST_FFMPEG"), "Requires verified CPU FFmpeg") def test_actual_ffmpeg_full_window_and_partial_tail_sample_coverage(self): ffmpeg = Path(os.environ["SEARCHI_TEST_FFMPEG"]) diff --git a/tools/speech-rank/Cargo.lock b/tools/speech-rank/Cargo.lock new file mode 100644 index 0000000..9d8fe94 --- /dev/null +++ b/tools/speech-rank/Cargo.lock @@ -0,0 +1,486 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "image-webp", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "searchi-semantic" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "image", + "jpeg-decoder", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "searchi-speech-rank" +version = "0.1.0" +dependencies = [ + "anyhow", + "searchi-semantic", + "serde_json", + "sha2", + "tempfile", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/tools/speech-rank/Cargo.toml b/tools/speech-rank/Cargo.toml new file mode 100644 index 0000000..bd35936 --- /dev/null +++ b/tools/speech-rank/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "searchi-speech-rank" +version = "0.1.0" +edition = "2021" +publish = false +[workspace] +[dependencies] +anyhow = "1" +serde_json = "1" +sha2 = "0.10" +searchi-semantic = { path = "../../crates/searchi-semantic" } +tempfile = "3" diff --git a/tools/speech-rank/README.md b/tools/speech-rank/README.md new file mode 100644 index 0000000..ccd6b6e --- /dev/null +++ b/tools/speech-rank/README.md @@ -0,0 +1,48 @@ +# Genuine Google transcript ranking diagnostic + +This internal experiment ranks **actual recognized ASR text** with the shared +Google EmbeddingGemma 2 adapter. It is separate from production ANN/hybrid search +and does not add speech recognition to the application. + +All valid segments from the NASA video and four music/sound competitors enter +the same candidate pool. Original source filenames are the only document titles; +recognized text is unchanged. Frozen relevance intervals never select, rewrite, +filter or rank candidates. General Google search queries use the normal 512-token +query role; document encoding uses 2048 tokens. The document process is dropped +before opening the query process. Both use forced CPU and one thread. + +Long segments use the actual Google tokenizer's source-preserving windows. +Every vector's original segment and UTF-8 byte range are recorded. Exact cosine +scores are computed from genuine Google vectors and aggregated by maximum over +token windows belonging to the same original ASR segment. All vector and segment +ranks are retained; at most the first 100 segment hits per query are exported for +the existing scorer. Ties use stable source order. No confidence threshold or +oracle matching is applied. + +```powershell +cargo test --manifest-path tools/speech-rank/Cargo.toml --locked +cargo build --release --manifest-path tools/speech-rank/Cargo.toml --locked +$env:SEARCHI_FORCE_CPU = '1' +tools/speech-rank/target/release/searchi-speech-rank.exe .searchi/speech/base-results.json fixtures/media/fixture.json C:/path/to/verified-models C:/path/to/searchi-gemma-worker.exe .searchi/speech/base-google-ranks.json +python tools/speech-eval/score.py --rankings .searchi/speech/base-google-ranks.json --output .searchi/speech/base-google-score.json +``` + +Run only after the shared inference CPU slot is released. The output directory +must exist, and the output filename must be new. `create_new` reserves evidence +before costly work; atomic JSON checkpoints retain incomplete phases. Inputs +are bounded, source/model hashes verified, and recognized segment timestamps +reconstructed from actual Whisper JSON to reject stale normalized text. Reports +from truncated decoding or an older policy are rejected. The artifact records +the actual ASR bytes, worker/runner/model hashes and process/timing information. + +Invalid ASR windows are explicitly marked ineligible with reasons, never quietly +discarded. A report with any excluded window has `quality_score_eligible: false`: +do not publish a quality score from a reduced competitor pool. Valid hallucinated +music or sound text remains eligible and must compete. `complete` reports prove +the diagnostic finished, not speech accuracy. + +Exported `start_ms`/`end_ms` refer to the full original media timeline. They are +ASR segment approximations, potentially affected by 30-second recognition +boundaries; they are **not word alignment**. A token window retains its containing +segment's same interval. The existing eight frozen NASA queries constitute a tiny +diagnostic, not held-out calibration, general media quality or production proof. diff --git a/tools/speech-rank/src/main.rs b/tools/speech-rank/src/main.rs new file mode 100644 index 0000000..54909e9 --- /dev/null +++ b/tools/speech-rank/src/main.rs @@ -0,0 +1,440 @@ +//! Genuine Google retrieval over recognized ASR text; labels never select segments. +use anyhow::{ensure, Context, Result}; +use searchi_semantic::{Device, GemmaEmbedder, GEMMA_DIMENSIONS}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::{Read, Write}, + path::{Path, PathBuf}, + time::Instant, +}; + +const FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393"; +const ASSETS: [&str; 5] = ["clip04", "music01", "music02", "sound01", "sound02"]; +const POLICY: &str = "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times"; + +fn digest(path: &Path) -> Result { + let mut file = fs::File::open(path)?; + let mut hash = Sha256::new(); + let mut buffer = [0; 65536]; + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + Ok(format!("{:x}", hash.finalize())) +} +fn number(value: &Value) -> Result { + value + .as_u64() + .context("Expected unsigned integer milliseconds") +} +fn checkpoint(path: &Path, report: &Value) -> Result<()> { + let parent = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let mut temporary = tempfile::NamedTempFile::new_in(parent)?; + temporary.write_all(&serde_json::to_vec_pretty(report)?)?; + temporary.as_file().sync_all()?; + temporary.persist(path)?; + Ok(()) +} + +/// Reconstruct absolute segment coordinates from actual whisper output, never labels. +fn parsed_segments(raw: &Value, offset: u64, duration: u64) -> Result> { + let rows = raw["transcription"] + .as_array() + .context("Missing actual transcription")?; + ensure!(rows.len() <= 1000, "Oversized transcription"); + let mut segments = Vec::new(); + let mut previous = 0; + for row in rows { + let start = number(&row["offsets"]["from"])?; + let end = number(&row["offsets"]["to"])?; + let text = row["text"].as_str().context("Invalid recognized text")?; + ensure!( + start >= previous && start <= end && end <= duration && text.len() <= 4096, + "Invalid actual ASR segment" + ); + previous = start; + if !text.trim().is_empty() { + segments.push(json!({"start_ms":offset.checked_add(start).context("Timestamp overflow")?,"end_ms":offset.checked_add(end).context("Timestamp overflow")?,"text":text})); + } + } + Ok(segments) +} + +fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec, Vec)> { + ensure!( + asr["version"] == 1 && asr["upstream_commit"] == "d1be6fde11ac6e0407606b4e42fe72d34add8037", + "Unknown ASR implementation" + ); + ensure!( + matches!( + (asr["candidate"].as_str(), asr["model_sha256"].as_str()), + ( + Some("base"), + Some("60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe") + ) | ( + Some("small"), + Some("1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b") + ) + ), + "Unknown ASR model" + ); + ensure!( + asr["complete"] == true + && asr["fixture_sha256"] == FIXTURE_SHA + && asr["policy"] == POLICY + && asr["backend"] == "forced CPU" + && asr["threads"] == 1, + "Incomplete, stale, or unrecognized ASR report" + ); + for field in [ + "model_sha256", + "worker_sha256", + "harness_sha256", + "ffmpeg_sha256", + "ffprobe_sha256", + ] { + ensure!(asr[field].as_str().is_some_and(|hash| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit())), "Missing ASR provenance"); + } + let rows = asr["rows"].as_array().context("Missing ASR rows")?; + ensure!(rows.len() == 5, "All five competing assets required"); + let mut seen = BTreeSet::new(); + let mut segments = Vec::new(); + let mut excluded = Vec::new(); + for row in rows { + let asset_id = row["asset"].as_str().context("Missing asset ID")?; + ensure!( + ASSETS.contains(&asset_id) && seen.insert(asset_id), + "Unknown/duplicate asset" + ); + let asset = fixture["assets"] + .as_array() + .context("Missing fixture assets")? + .iter() + .find(|asset| asset["id"] == asset_id) + .context("Unknown source")?; + ensure!( + row["source_sha256"] == asset["sha256"], + "Stale source provenance" + ); + let start = number(&row["audio_start_ms"])?; + let end = number(&row["audio_end_ms"])?; + ensure!( + start < end && end - start <= 600000 && number(&row["duration_ms"])? == end - start, + "Unbounded audio range" + ); + let windows = row["windows"].as_array().context("Missing windows")?; + ensure!(windows.len() <= 20, "Oversized window list"); + let mut next = start; + for (window_index, window) in windows.iter().enumerate() { + let offset = number(&window["offset_ms"])?; + let duration = number(&window["duration_ms"])?; + ensure!( + offset == next && duration == (end - offset).min(30000) && duration > 0, + "Missing, overlapping or truncated source windows" + ); + next = offset.checked_add(duration).context("Window overflow")?; + if let Some(error) = window.get("validation_error") { + ensure!( + error.as_str().is_some_and(|error| !error.is_empty()), + "Invalid exclusion reason" + ); + if window["error_stage"] == "decode_coverage" { + let frames = number(&window["decoded_frames"])?; + ensure!( + window["asr_not_run"] == true + && (frames > duration * 16 + || frames.saturating_add(16) < duration * 16), + "False decode exclusion" + ); + } else { + ensure!( + parsed_segments(&window["actual_whisper_json"], offset, duration).is_err(), + "Valid recognized text must not be excluded" + ); + } + excluded.push(json!({"asset":asset_id,"window_index":window_index,"offset_ms":offset,"duration_ms":duration,"eligible":false,"reason":error})); + continue; + } + let frames = number(&window["decoded_frames"])?; + ensure!( + frames <= duration * 16 && frames + 16 >= duration * 16, + "Stale/truncated decoded audio coverage" + ); + let actual = parsed_segments(&window["actual_whisper_json"], offset, duration)?; + ensure!( + window["segments"] == json!(actual), + "Normalized ASR segments differ from actual output" + ); + let title = Path::new(asset["path"].as_str().context("Missing source filename")?) + .file_name() + .context("Missing filename")? + .to_string_lossy(); + for (segment_index, segment) in actual.into_iter().enumerate() { + segments.push(json!({"asset":asset_id,"window_index":window_index,"segment_index":segment_index,"source_window_offset_ms":offset,"source_window_duration_ms":duration,"start_ms":segment["start_ms"],"end_ms":segment["end_ms"],"text":segment["text"],"title":title})); + ensure!(segments.len() <= 5000, "Too many recognized segments"); + } + } + ensure!(next == end, "Incomplete source-window coverage"); + } + ensure!( + asr["quality_failed"] == !excluded.is_empty(), + "ASR validation status inconsistent" + ); + ensure!( + asr["validation_failures"] + .as_array() + .context("Missing validation failures")? + .len() + == excluded.len(), + "ASR validation failure count inconsistent" + ); + Ok((segments, excluded)) +} + +fn cosine(a: &[f32], b: &[f32]) -> Result { + ensure!( + a.len() == GEMMA_DIMENSIONS && b.len() == GEMMA_DIMENSIONS, + "Wrong vector dimensions" + ); + let (mut dot, mut aa, mut bb) = (0.0f64, 0.0f64, 0.0f64); + for (&a, &b) in a.iter().zip(b) { + ensure!(a.is_finite() && b.is_finite(), "Nonfinite vector"); + let (a, b) = (f64::from(a), f64::from(b)); + dot += a * b; + aa += a * a; + bb += b * b; + } + ensure!(aa > 0.0 && bb > 0.0, "Zero vector"); + Ok(dot / (aa * bb).sqrt()) +} + +fn main() -> Result<()> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + ensure!( + args.len() == 5, + "Supply ASR_REPORT MEDIA_FIXTURE MODEL_DIR GOOGLE_WORKER NEW_OUTPUT_JSON" + ); + ensure!( + std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"), + "CPU only" + ); + let asr_path = PathBuf::from(&args[0]); + let fixture_path = PathBuf::from(&args[1]); + let models = PathBuf::from(&args[2]); + let worker = PathBuf::from(&args[3]); + let output = PathBuf::from(&args[4]); + fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&output) + .context("Cannot reserve new output")?; + let mut report = + json!({"complete":false,"phase":"verification","rankings":{},"raw_queries":[]}); + checkpoint(&output, &report)?; + ensure!( + fs::metadata(&asr_path)?.len() <= 16 * 1024 * 1024, + "Oversized ASR report" + ); + let fixture_bytes = fs::read(&fixture_path)?; + ensure!( + format!("{:x}", Sha256::digest(&fixture_bytes)) == FIXTURE_SHA, + "Frozen media fixture changed" + ); + let fixture: Value = serde_json::from_slice(&fixture_bytes)?; + let asr_bytes = fs::read(&asr_path)?; + let asr_hash = format!("{:x}", Sha256::digest(&asr_bytes)); + let asr: Value = serde_json::from_slice(&asr_bytes)?; + let (segments, excluded) = eligible(&asr, &fixture)?; + ensure!(!segments.is_empty(), "No eligible recognized text"); + for asset in fixture["assets"] + .as_array() + .context("Missing assets")? + .iter() + .filter(|asset| ASSETS.contains(&asset["id"].as_str().unwrap_or_default())) + { + let source = fixture_path + .parent() + .context("Missing fixture parent")? + .join(asset["path"].as_str().context("Missing asset path")?); + ensure!( + fs::metadata(&source)?.len() == number(&asset["bytes"])? + && digest(&source)? == asset["sha256"], + "Frozen media source changed" + ); + } + let manifest_path = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../embeddinggemma2/model-manifest.json"); + let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?; + for asset in manifest["files"] + .as_array() + .context("Missing model files")? + { + let path = models.join(asset["path"].as_str().context("Missing model path")?); + ensure!( + fs::metadata(&path)?.len() == number(&asset["bytes"])? + && digest(&path)? == asset["sha256"], + "Google model bytes changed" + ); + } + report["fixture_sha256"] = json!(FIXTURE_SHA); + report["asr_report_sha256"] = json!(asr_hash); + report["asr_provenance"] = asr.clone(); + report["google_worker_sha256"] = json!(digest(&worker)?); + report["runner_sha256"] = json!(digest(&std::env::current_exe()?)?); + report["google_model_manifest_sha256"] = json!(digest(&manifest_path)?); + report["google_model"] = manifest; + report["segments"] = json!(segments); + report["excluded_windows"] = json!(excluded); + report["excluded_window_count"] = json!(excluded.len()); + report["quality_score_eligible"] = json!(excluded.is_empty()); + report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration."); + report["phase"] = json!("document_encoding"); + checkpoint(&output, &report)?; + let start = Instant::now(); + let mut model = GemmaEmbedder::load_with_device(&models, &worker, 1, Device::Cpu)?; + report["document_worker_info"] = json!(model.info); + let mut vectors = Vec::new(); + let mut mappings = Vec::new(); + for (segment_id, segment) in segments.iter().enumerate() { + let text = segment["text"].as_str().context("Missing text")?; + let title = segment["title"].as_str().context("Missing title")?; + for window in model.split_windows(text, title)? { + let mut encoded = model.embed_documents(&[&window.text], title)?; + ensure!(encoded.len() == 1, "Missing document embedding"); + vectors.push(encoded.remove(0)); + mappings.push(json!({"segment_id":segment_id,"text_start_byte":window.start_byte,"text_end_byte":window.end_byte})); + ensure!(vectors.len() <= 10000, "Too many token windows"); + } + } + report["document_encoding_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0); + report["vector_source_mapping"] = json!(mappings); + drop(model); + let start = Instant::now(); + let mut query_model = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?; + report["query_worker_info"] = json!(query_model.info); + report["query_startup_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0); + let queries: Vec<_> = fixture["queries"] + .as_array() + .context("Missing fixture queries")? + .iter() + .filter(|query| query["category"] == "video-speech") + .collect(); + ensure!( + queries.len() == 8, + "Exactly eight frozen speech queries required" + ); + report["phase"] = json!("query_ranking"); + checkpoint(&output, &report)?; + for query in queries { + let start = Instant::now(); + let vector = + query_model.embed_query(query["text"].as_str().context("Missing query text")?)?; + let mut raw = Vec::new(); + let mut best: BTreeMap = BTreeMap::new(); + for (id, document) in vectors.iter().enumerate() { + let score = cosine(&vector, document)?; + let segment_id = number(&mappings[id]["segment_id"])? as usize; + best.entry(segment_id) + .and_modify(|old| *old = old.max(score)) + .or_insert(score); + raw.push(json!({"vector_id":id,"segment_id":segment_id,"cosine":score})); + } + raw.sort_by(|a, b| { + b["cosine"] + .as_f64() + .unwrap() + .total_cmp(&a["cosine"].as_f64().unwrap()) + .then_with(|| a["vector_id"].as_u64().cmp(&b["vector_id"].as_u64())) + }); + let mut ranks: Vec<_> = best.into_iter().collect(); + ranks.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0))); + let all: Vec<_> = ranks.into_iter().map(|(id,score)| json!({"asset":segments[id]["asset"],"start_ms":segments[id]["start_ms"],"end_ms":segments[id]["end_ms"],"segment_id":id,"cosine":score})).collect(); + report["rankings"][query["id"].as_str().context("Missing query ID")?] = + json!(all.iter().take(100).collect::>()); + report["raw_queries"].as_array_mut().context("Missing output queries")?.push(json!({"id":query["id"],"text":query["text"],"elapsed_ms":start.elapsed().as_secs_f64()*1000.0,"all_segment_ranks":all,"all_vector_ranks":raw})); + checkpoint(&output, &report)?; + } + ensure!( + report["asr_report_sha256"] == digest(&asr_path)?, + "ASR report changed during ranking" + ); + ensure!( + digest(&fixture_path)? == FIXTURE_SHA + && report["google_worker_sha256"] == digest(&worker)? + && report["google_model_manifest_sha256"] == digest(&manifest_path)?, + "Fixture or runtime provenance changed during ranking" + ); + report["complete"] = json!(true); + report["phase"] = json!("complete"); + checkpoint(&output, &report)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn timestamps_are_absolute_milliseconds_and_invalid_tail_rejected() { + let raw = + json!({"transcription":[{"offsets":{"from":100,"to":2000},"text":" untouched text "}]}); + assert_eq!( + parsed_segments(&raw, 30000, 3000).unwrap(), + vec![json!({"start_ms":30100,"end_ms":32000,"text":" untouched text "})] + ); + assert!(parsed_segments(&raw, 30000, 1000).is_err()); + assert!(parsed_segments(&raw, u64::MAX, 3000).is_err()); + assert!(parsed_segments( + &json!({"transcription":[{"offsets":{"from":1.0,"to":2},"text":"x"}]}), + 0, + 30 + ) + .is_err()); + } + #[test] + fn incomplete_and_stale_asr_are_rejected_before_inference() { + assert!(eligible(&json!({"complete":false}), &json!({})).is_err()); + assert!(eligible( + &json!({"complete":true,"fixture_sha256":"stale"}), + &json!({}) + ) + .is_err()); + } + + #[test] + fn source_identity_coverage_and_recognized_text_cannot_be_forged() { + let hash = "a".repeat(64); + let assets: Vec<_> = ASSETS + .iter() + .map(|id| json!({"id":id,"sha256":hash,"path":format!("{id}.mp3")})) + .collect(); + let raw = json!({"transcription":[{"offsets":{"from":10,"to":900},"text":"actual unit-test text"}]}); + let rows:Vec<_>=ASSETS.iter().map(|id|json!({"asset":id,"source_sha256":hash,"audio_start_ms":1000,"audio_end_ms":2000,"duration_ms":1000,"windows":[{"offset_ms":1000,"duration_ms":1000,"decoded_frames":16000,"actual_whisper_json":raw,"segments":parsed_segments(&raw,1000,1000).unwrap()}]})).collect(); + let report = json!({"version":1,"complete":true,"fixture_sha256":FIXTURE_SHA,"policy":POLICY,"backend":"forced CPU","threads":1,"upstream_commit":"d1be6fde11ac6e0407606b4e42fe72d34add8037","candidate":"base","model_sha256":"60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe","worker_sha256":hash,"harness_sha256":hash,"ffmpeg_sha256":hash,"ffprobe_sha256":hash,"quality_failed":false,"validation_failures":[],"rows":rows}); + let fixture = json!({"assets":assets}); + assert_eq!(eligible(&report, &fixture).unwrap().0.len(), 5); + let mut changed = report.clone(); + changed["rows"][0]["asset"] = json!("unknown"); + assert!(eligible(&changed, &fixture).is_err()); + let mut changed = report.clone(); + changed["rows"][0]["source_sha256"] = json!("stale"); + assert!(eligible(&changed, &fixture).is_err()); + let mut changed = report.clone(); + changed["rows"][0]["windows"][0]["segments"][0]["start_ms"] = json!(10); + assert!(eligible(&changed, &fixture).is_err()); + let mut changed = report.clone(); + changed["rows"][0]["windows"][0]["decoded_frames"] = json!(8000); + assert!(eligible(&changed, &fixture).is_err()); + let mut changed = report; + changed["rows"][0]["windows"][0]["validation_error"] = json!("hide a valid competitor"); + assert!(eligible(&changed, &fixture).is_err()); + } +}