Validate audio stream timelines and add genuine Google transcript diagnostics
This commit is contained in:
karl committed
1 parent
8ef874c609
commit
5eeba29c0d
7 files changed
+1074
-23
No files matched your search
+40
-21
@@ -108,10 +108,10 @@ def recognized_window(raw, offset, length, decode_ms, asr_ms):
|
||||
return report
|
||||
|
||||
|
||||
def decode_command(ffmpeg, source, offset, length, wav):
|
||||
def decode_command(ffmpeg, source, offset, length, wav, audio_index=None):
|
||||
return [str(ffmpeg.resolve()), "-nostdin", "-hide_banner", "-v", "error",
|
||||
"-protocol_whitelist", "file,pipe", "-threads", "1", "-ss", str(offset / 1000), "-t", str(length / 1000),
|
||||
"-i", str(source.resolve()), "-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
|
||||
"-protocol_whitelist", "file,pipe", "-threads", "1", "-seek_timestamp", "1", "-ss", str(offset / 1000), "-t", str(length / 1000),
|
||||
"-i", str(source.resolve()), "-map", f"0:{audio_index}" if audio_index is not None else "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
|
||||
"-af", f"aresample=16000,atrim=end_sample={length * 16},asetpts=PTS-STARTPTS",
|
||||
"-c:a", "pcm_s16le", "-threads", "1", "-y", str(wav)]
|
||||
|
||||
@@ -127,6 +127,19 @@ def decoded_frames(wav, length):
|
||||
return frames
|
||||
|
||||
|
||||
def primary_audio_range(probe):
|
||||
audio = next((stream for stream in probe.get("streams", []) if stream.get("codec_type") == "audio"), None)
|
||||
if audio is None:
|
||||
raise ValueError("No primary audio stream")
|
||||
start = float(audio.get("start_time", 0))
|
||||
if "duration" not in audio:
|
||||
raise ValueError("Primary audio stream duration unavailable; proof requires explicit stream duration, not container fallback")
|
||||
duration = float(audio["duration"])
|
||||
if not math.isfinite(start) or not math.isfinite(duration) or start < 0 or not 0 < duration <= 600:
|
||||
raise ValueError("Proof audio stream range exceeds bounds")
|
||||
return math.ceil(start * 1000), math.ceil((start + duration) * 1000), audio["index"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--whisper", type=Path, required=True)
|
||||
@@ -161,30 +174,36 @@ def main():
|
||||
raise ValueError(f"Frozen asset mismatch: {asset['id']}")
|
||||
# Probe stdout is bounded by the same owned log, then parsed directly.
|
||||
run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe",
|
||||
"-show_entries", "format=duration", "-of", "json", str(source.resolve())],
|
||||
"-show_entries", "stream=index,codec_type,duration,start_time:format=duration,start_time", "-of", "json", str(source.resolve())],
|
||||
scratch, "probe", [], 30)
|
||||
probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8"))
|
||||
duration_s = float(probe["format"]["duration"])
|
||||
if not math.isfinite(duration_s) or not 0 < duration_s <= 600:
|
||||
raise ValueError("Proof duration exceeds ten-minute bound")
|
||||
duration_ms = math.ceil(duration_s * 1000)
|
||||
result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": duration_ms, "windows": []}
|
||||
for offset in range(0, duration_ms, 30000):
|
||||
length = min(30000, duration_ms - offset)
|
||||
start_ms, end_ms, audio_index = primary_audio_range(probe)
|
||||
result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": end_ms - start_ms,
|
||||
"audio_start_ms": start_ms, "audio_end_ms": end_ms, "audio_stream_index": audio_index,
|
||||
"actual_probe": probe, "windows": []}
|
||||
for offset in range(start_ms, end_ms, 30000):
|
||||
length = min(30000, end_ms - offset)
|
||||
wav = scratch / "window.wav"
|
||||
prefix = scratch / "recognized"
|
||||
output = prefix.with_suffix(".json")
|
||||
output.unlink(missing_ok=True)
|
||||
decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav),
|
||||
decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav, audio_index),
|
||||
scratch, "decode", [(wav, 970000)], 60)
|
||||
frames = decoded_frames(wav, length)
|
||||
asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
|
||||
"--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
|
||||
"--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
|
||||
raw = json.loads(output.read_text(encoding="utf-8"))
|
||||
# Preserve genuine invalid output, never clamp it into valid evidence.
|
||||
result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
|
||||
result["windows"][-1]["decoded_frames"] = frames
|
||||
try:
|
||||
frames = decoded_frames(wav, length)
|
||||
except ValueError as error:
|
||||
with wave.open(str(wav), "rb") as audio:
|
||||
actual_frames = audio.getnframes()
|
||||
result["windows"].append({"offset_ms": offset, "duration_ms": length, "decode_ms": decode_ms,
|
||||
"decoded_frames": actual_frames, "validation_error": str(error), "error_stage": "decode_coverage",
|
||||
"segments": [], "asr_not_run": True})
|
||||
else:
|
||||
asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
|
||||
"--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
|
||||
"--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
|
||||
raw = json.loads(output.read_text(encoding="utf-8"))
|
||||
result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
|
||||
result["windows"][-1]["decoded_frames"] = frames
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture),
|
||||
"model_sha256": digest, "worker_sha256": args.whisper_sha256,
|
||||
@@ -202,7 +221,7 @@ def main():
|
||||
"ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe),
|
||||
"backend": "forced CPU", "threads": 1,
|
||||
"harness_sha256": sha256(Path(__file__)),
|
||||
"policy": "aresample16000-before-atrim-v2;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
|
||||
"policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
|
||||
"rows": rows}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
|
||||
@@ -6,6 +6,13 @@ from pathlib import Path
|
||||
from run import FIXTURE_SHA
|
||||
|
||||
|
||||
def require_eligible(report):
|
||||
if report.get("complete") is not True or report.get("quality_score_eligible") is not True:
|
||||
raise ValueError("Cannot score incomplete runs or reduced/invalid competitor pools")
|
||||
if report.get("excluded_windows") or report.get("excluded_window_count", 0):
|
||||
raise ValueError("Cannot score rankings with excluded ASR windows")
|
||||
|
||||
|
||||
def score(fixture, rankings):
|
||||
queries = [query for query in fixture["queries"] if query["category"] == "video-speech"]
|
||||
if len(queries) != 8:
|
||||
@@ -41,6 +48,7 @@ def main():
|
||||
args = parser.parse_args()
|
||||
data = args.fixture.read_bytes()
|
||||
rankings = json.loads(args.rankings.read_text(encoding="utf-8"))
|
||||
require_eligible(rankings)
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
if digest != FIXTURE_SHA or rankings["fixture_sha256"] != digest or not rankings.get("asr_report_sha256") or not rankings.get("google_worker_sha256"):
|
||||
raise ValueError("Missing/mismatched actual ASR/Google provenance")
|
||||
|
||||
@@ -6,11 +6,49 @@ import tempfile
|
||||
import sys
|
||||
import os
|
||||
import wave
|
||||
from run import segments, run_owned, recognized_window, decode_command, decoded_frames
|
||||
from score import score
|
||||
from run import segments, run_owned, recognized_window, decode_command, decoded_frames, primary_audio_range
|
||||
from score import score, require_eligible
|
||||
|
||||
|
||||
class ContractTests(unittest.TestCase):
|
||||
def test_scoring_rejects_incomplete_or_reduced_pool(self):
|
||||
for report in [{}, {"complete": False, "quality_score_eligible": True},
|
||||
{"complete": True, "quality_score_eligible": False},
|
||||
{"complete": True, "quality_score_eligible": True, "excluded_windows": [{}]}]:
|
||||
with self.assertRaises(ValueError):
|
||||
require_eligible(report)
|
||||
require_eligible({"complete": True, "quality_score_eligible": True, "excluded_windows": []})
|
||||
def test_primary_audio_duration_and_nonzero_start_ignore_longer_video(self):
|
||||
probe = {"streams": [{"index": 0, "codec_type": "video", "duration": "90"},
|
||||
{"index": 1, "codec_type": "audio", "start_time": "0.125", "duration": "31"}]}
|
||||
self.assertEqual(primary_audio_range(probe), (125, 31125, 1))
|
||||
|
||||
@unittest.skipUnless(os.environ.get("SEARCHI_TEST_FFMPEG"), "Requires verified CPU FFmpeg")
|
||||
def test_actual_video_longer_than_audio(self):
|
||||
ffmpeg = Path(os.environ["SEARCHI_TEST_FFMPEG"])
|
||||
ffprobe = ffmpeg.with_name("ffprobe.exe" if os.name == "nt" else "ffprobe")
|
||||
with tempfile.TemporaryDirectory() as parent:
|
||||
scratch = Path(parent)
|
||||
source = scratch / "longer-video.mkv"
|
||||
run_owned([str(ffmpeg), "-nostdin", "-hide_banner", "-v", "error", "-threads", "1",
|
||||
"-f", "lavfi", "-i", "color=c=black:s=16x16:r=1:d=33",
|
||||
"-itsoffset", "0.125", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=mono:d=31", "-c:v", "ffv1",
|
||||
"-c:a", "pcm_s16le", "-threads", "1", "-output_ts_offset", "5", "-y", str(source)],
|
||||
scratch, "mux", [(source, 4 * 1024 * 1024)], 30)
|
||||
run_owned([str(ffprobe), "-v", "error", "-show_entries", "stream=index,codec_type,duration,start_time:stream_tags=DURATION:format=duration", "-of", "json", str(source)], scratch, "probe", [], 30)
|
||||
probe = json.loads((scratch / "probe.log").read_text())
|
||||
audio = next(row for row in probe["streams"] if row["codec_type"] == "audio")
|
||||
# Matroska stores duration in tags; test conversion is explicit, not a format fallback.
|
||||
if "duration" not in audio:
|
||||
h, m, sec = audio["tags"]["DURATION"].split(":")
|
||||
audio["duration"] = int(h) * 3600 + int(m) * 60 + float(sec) - float(audio["start_time"])
|
||||
start, end, _ = primary_audio_range(probe)
|
||||
self.assertEqual((start, end), (5125, 36125))
|
||||
self.assertGreater(float(probe["format"]["duration"]), end / 1000)
|
||||
wav = scratch / "tail.wav"
|
||||
run_owned(decode_command(ffmpeg, source, 35125, 1000, wav, audio["index"]), scratch, "tail", [(wav, 970000)], 30)
|
||||
self.assertGreaterEqual(decoded_frames(wav, 1000), 15984)
|
||||
|
||||
@unittest.skipUnless(os.environ.get("SEARCHI_TEST_FFMPEG"), "Requires verified CPU FFmpeg")
|
||||
def test_actual_ffmpeg_full_window_and_partial_tail_sample_coverage(self):
|
||||
ffmpeg = Path(os.environ["SEARCHI_TEST_FFMPEG"])
|
||||
|
||||
Generated
+486
@@ -0,0 +1,486 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.25.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder-lite"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
||||
|
||||
[[package]]
|
||||
name = "fdeflate"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
|
||||
dependencies = [
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide 0.9.1",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image"
|
||||
version = "0.25.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder-lite",
|
||||
"image-webp",
|
||||
"moxcms",
|
||||
"num-traits",
|
||||
"png",
|
||||
"zune-core",
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image-webp"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
|
||||
dependencies = [
|
||||
"byteorder-lite",
|
||||
"quick-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jpeg-decoder"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.190"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "moxcms"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"pxfm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "png"
|
||||
version = "0.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crc32fast",
|
||||
"fdeflate",
|
||||
"flate2",
|
||||
"miniz_oxide 0.8.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pxfm"
|
||||
version = "0.1.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "searchi-semantic"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
"image",
|
||||
"jpeg-decoder",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "searchi-speech-rank"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"searchi-semantic",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.154"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zune-core"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b"
|
||||
|
||||
[[package]]
|
||||
name = "zune-jpeg"
|
||||
version = "0.5.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
|
||||
dependencies = [
|
||||
"zune-core",
|
||||
]
|
||||
@@ -0,0 +1,12 @@
|
||||
[package]
|
||||
name = "searchi-speech-rank"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
publish = false
|
||||
[workspace]
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
serde_json = "1"
|
||||
sha2 = "0.10"
|
||||
searchi-semantic = { path = "../../crates/searchi-semantic" }
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,48 @@
|
||||
# Genuine Google transcript ranking diagnostic
|
||||
|
||||
This internal experiment ranks **actual recognized ASR text** with the shared
|
||||
Google EmbeddingGemma 2 adapter. It is separate from production ANN/hybrid search
|
||||
and does not add speech recognition to the application.
|
||||
|
||||
All valid segments from the NASA video and four music/sound competitors enter
|
||||
the same candidate pool. Original source filenames are the only document titles;
|
||||
recognized text is unchanged. Frozen relevance intervals never select, rewrite,
|
||||
filter or rank candidates. General Google search queries use the normal 512-token
|
||||
query role; document encoding uses 2048 tokens. The document process is dropped
|
||||
before opening the query process. Both use forced CPU and one thread.
|
||||
|
||||
Long segments use the actual Google tokenizer's source-preserving windows.
|
||||
Every vector's original segment and UTF-8 byte range are recorded. Exact cosine
|
||||
scores are computed from genuine Google vectors and aggregated by maximum over
|
||||
token windows belonging to the same original ASR segment. All vector and segment
|
||||
ranks are retained; at most the first 100 segment hits per query are exported for
|
||||
the existing scorer. Ties use stable source order. No confidence threshold or
|
||||
oracle matching is applied.
|
||||
|
||||
```powershell
|
||||
cargo test --manifest-path tools/speech-rank/Cargo.toml --locked
|
||||
cargo build --release --manifest-path tools/speech-rank/Cargo.toml --locked
|
||||
$env:SEARCHI_FORCE_CPU = '1'
|
||||
tools/speech-rank/target/release/searchi-speech-rank.exe .searchi/speech/base-results.json fixtures/media/fixture.json C:/path/to/verified-models C:/path/to/searchi-gemma-worker.exe .searchi/speech/base-google-ranks.json
|
||||
python tools/speech-eval/score.py --rankings .searchi/speech/base-google-ranks.json --output .searchi/speech/base-google-score.json
|
||||
```
|
||||
|
||||
Run only after the shared inference CPU slot is released. The output directory
|
||||
must exist, and the output filename must be new. `create_new` reserves evidence
|
||||
before costly work; atomic JSON checkpoints retain incomplete phases. Inputs
|
||||
are bounded, source/model hashes verified, and recognized segment timestamps
|
||||
reconstructed from actual Whisper JSON to reject stale normalized text. Reports
|
||||
from truncated decoding or an older policy are rejected. The artifact records
|
||||
the actual ASR bytes, worker/runner/model hashes and process/timing information.
|
||||
|
||||
Invalid ASR windows are explicitly marked ineligible with reasons, never quietly
|
||||
discarded. A report with any excluded window has `quality_score_eligible: false`:
|
||||
do not publish a quality score from a reduced competitor pool. Valid hallucinated
|
||||
music or sound text remains eligible and must compete. `complete` reports prove
|
||||
the diagnostic finished, not speech accuracy.
|
||||
|
||||
Exported `start_ms`/`end_ms` refer to the full original media timeline. They are
|
||||
ASR segment approximations, potentially affected by 30-second recognition
|
||||
boundaries; they are **not word alignment**. A token window retains its containing
|
||||
segment's same interval. The existing eight frozen NASA queries constitute a tiny
|
||||
diagnostic, not held-out calibration, general media quality or production proof.
|
||||
@@ -0,0 +1,440 @@
|
||||
//! Genuine Google retrieval over recognized ASR text; labels never select segments.
|
||||
use anyhow::{ensure, Context, Result};
|
||||
use searchi_semantic::{Device, GemmaEmbedder, GEMMA_DIMENSIONS};
|
||||
use serde_json::{json, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fs,
|
||||
io::{Read, Write},
|
||||
path::{Path, PathBuf},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393";
|
||||
const ASSETS: [&str; 5] = ["clip04", "music01", "music02", "sound01", "sound02"];
|
||||
const POLICY: &str = "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times";
|
||||
|
||||
fn digest(path: &Path) -> Result<String> {
|
||||
let mut file = fs::File::open(path)?;
|
||||
let mut hash = Sha256::new();
|
||||
let mut buffer = [0; 65536];
|
||||
loop {
|
||||
let n = file.read(&mut buffer)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hash.update(&buffer[..n]);
|
||||
}
|
||||
Ok(format!("{:x}", hash.finalize()))
|
||||
}
|
||||
fn number(value: &Value) -> Result<u64> {
|
||||
value
|
||||
.as_u64()
|
||||
.context("Expected unsigned integer milliseconds")
|
||||
}
|
||||
fn checkpoint(path: &Path, report: &Value) -> Result<()> {
|
||||
let parent = path
|
||||
.parent()
|
||||
.filter(|p| !p.as_os_str().is_empty())
|
||||
.unwrap_or(Path::new("."));
|
||||
let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
|
||||
temporary.write_all(&serde_json::to_vec_pretty(report)?)?;
|
||||
temporary.as_file().sync_all()?;
|
||||
temporary.persist(path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reconstruct absolute segment coordinates from actual whisper output, never labels.
|
||||
fn parsed_segments(raw: &Value, offset: u64, duration: u64) -> Result<Vec<Value>> {
|
||||
let rows = raw["transcription"]
|
||||
.as_array()
|
||||
.context("Missing actual transcription")?;
|
||||
ensure!(rows.len() <= 1000, "Oversized transcription");
|
||||
let mut segments = Vec::new();
|
||||
let mut previous = 0;
|
||||
for row in rows {
|
||||
let start = number(&row["offsets"]["from"])?;
|
||||
let end = number(&row["offsets"]["to"])?;
|
||||
let text = row["text"].as_str().context("Invalid recognized text")?;
|
||||
ensure!(
|
||||
start >= previous && start <= end && end <= duration && text.len() <= 4096,
|
||||
"Invalid actual ASR segment"
|
||||
);
|
||||
previous = start;
|
||||
if !text.trim().is_empty() {
|
||||
segments.push(json!({"start_ms":offset.checked_add(start).context("Timestamp overflow")?,"end_ms":offset.checked_add(end).context("Timestamp overflow")?,"text":text}));
|
||||
}
|
||||
}
|
||||
Ok(segments)
|
||||
}
|
||||
|
||||
fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
|
||||
ensure!(
|
||||
asr["version"] == 1 && asr["upstream_commit"] == "d1be6fde11ac6e0407606b4e42fe72d34add8037",
|
||||
"Unknown ASR implementation"
|
||||
);
|
||||
ensure!(
|
||||
matches!(
|
||||
(asr["candidate"].as_str(), asr["model_sha256"].as_str()),
|
||||
(
|
||||
Some("base"),
|
||||
Some("60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe")
|
||||
) | (
|
||||
Some("small"),
|
||||
Some("1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b")
|
||||
)
|
||||
),
|
||||
"Unknown ASR model"
|
||||
);
|
||||
ensure!(
|
||||
asr["complete"] == true
|
||||
&& asr["fixture_sha256"] == FIXTURE_SHA
|
||||
&& asr["policy"] == POLICY
|
||||
&& asr["backend"] == "forced CPU"
|
||||
&& asr["threads"] == 1,
|
||||
"Incomplete, stale, or unrecognized ASR report"
|
||||
);
|
||||
for field in [
|
||||
"model_sha256",
|
||||
"worker_sha256",
|
||||
"harness_sha256",
|
||||
"ffmpeg_sha256",
|
||||
"ffprobe_sha256",
|
||||
] {
|
||||
ensure!(asr[field].as_str().is_some_and(|hash| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit())), "Missing ASR provenance");
|
||||
}
|
||||
let rows = asr["rows"].as_array().context("Missing ASR rows")?;
|
||||
ensure!(rows.len() == 5, "All five competing assets required");
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut segments = Vec::new();
|
||||
let mut excluded = Vec::new();
|
||||
for row in rows {
|
||||
let asset_id = row["asset"].as_str().context("Missing asset ID")?;
|
||||
ensure!(
|
||||
ASSETS.contains(&asset_id) && seen.insert(asset_id),
|
||||
"Unknown/duplicate asset"
|
||||
);
|
||||
let asset = fixture["assets"]
|
||||
.as_array()
|
||||
.context("Missing fixture assets")?
|
||||
.iter()
|
||||
.find(|asset| asset["id"] == asset_id)
|
||||
.context("Unknown source")?;
|
||||
ensure!(
|
||||
row["source_sha256"] == asset["sha256"],
|
||||
"Stale source provenance"
|
||||
);
|
||||
let start = number(&row["audio_start_ms"])?;
|
||||
let end = number(&row["audio_end_ms"])?;
|
||||
ensure!(
|
||||
start < end && end - start <= 600000 && number(&row["duration_ms"])? == end - start,
|
||||
"Unbounded audio range"
|
||||
);
|
||||
let windows = row["windows"].as_array().context("Missing windows")?;
|
||||
ensure!(windows.len() <= 20, "Oversized window list");
|
||||
let mut next = start;
|
||||
for (window_index, window) in windows.iter().enumerate() {
|
||||
let offset = number(&window["offset_ms"])?;
|
||||
let duration = number(&window["duration_ms"])?;
|
||||
ensure!(
|
||||
offset == next && duration == (end - offset).min(30000) && duration > 0,
|
||||
"Missing, overlapping or truncated source windows"
|
||||
);
|
||||
next = offset.checked_add(duration).context("Window overflow")?;
|
||||
if let Some(error) = window.get("validation_error") {
|
||||
ensure!(
|
||||
error.as_str().is_some_and(|error| !error.is_empty()),
|
||||
"Invalid exclusion reason"
|
||||
);
|
||||
if window["error_stage"] == "decode_coverage" {
|
||||
let frames = number(&window["decoded_frames"])?;
|
||||
ensure!(
|
||||
window["asr_not_run"] == true
|
||||
&& (frames > duration * 16
|
||||
|| frames.saturating_add(16) < duration * 16),
|
||||
"False decode exclusion"
|
||||
);
|
||||
} else {
|
||||
ensure!(
|
||||
parsed_segments(&window["actual_whisper_json"], offset, duration).is_err(),
|
||||
"Valid recognized text must not be excluded"
|
||||
);
|
||||
}
|
||||
excluded.push(json!({"asset":asset_id,"window_index":window_index,"offset_ms":offset,"duration_ms":duration,"eligible":false,"reason":error}));
|
||||
continue;
|
||||
}
|
||||
let frames = number(&window["decoded_frames"])?;
|
||||
ensure!(
|
||||
frames <= duration * 16 && frames + 16 >= duration * 16,
|
||||
"Stale/truncated decoded audio coverage"
|
||||
);
|
||||
let actual = parsed_segments(&window["actual_whisper_json"], offset, duration)?;
|
||||
ensure!(
|
||||
window["segments"] == json!(actual),
|
||||
"Normalized ASR segments differ from actual output"
|
||||
);
|
||||
let title = Path::new(asset["path"].as_str().context("Missing source filename")?)
|
||||
.file_name()
|
||||
.context("Missing filename")?
|
||||
.to_string_lossy();
|
||||
for (segment_index, segment) in actual.into_iter().enumerate() {
|
||||
segments.push(json!({"asset":asset_id,"window_index":window_index,"segment_index":segment_index,"source_window_offset_ms":offset,"source_window_duration_ms":duration,"start_ms":segment["start_ms"],"end_ms":segment["end_ms"],"text":segment["text"],"title":title}));
|
||||
ensure!(segments.len() <= 5000, "Too many recognized segments");
|
||||
}
|
||||
}
|
||||
ensure!(next == end, "Incomplete source-window coverage");
|
||||
}
|
||||
ensure!(
|
||||
asr["quality_failed"] == !excluded.is_empty(),
|
||||
"ASR validation status inconsistent"
|
||||
);
|
||||
ensure!(
|
||||
asr["validation_failures"]
|
||||
.as_array()
|
||||
.context("Missing validation failures")?
|
||||
.len()
|
||||
== excluded.len(),
|
||||
"ASR validation failure count inconsistent"
|
||||
);
|
||||
Ok((segments, excluded))
|
||||
}
|
||||
|
||||
fn cosine(a: &[f32], b: &[f32]) -> Result<f64> {
|
||||
ensure!(
|
||||
a.len() == GEMMA_DIMENSIONS && b.len() == GEMMA_DIMENSIONS,
|
||||
"Wrong vector dimensions"
|
||||
);
|
||||
let (mut dot, mut aa, mut bb) = (0.0f64, 0.0f64, 0.0f64);
|
||||
for (&a, &b) in a.iter().zip(b) {
|
||||
ensure!(a.is_finite() && b.is_finite(), "Nonfinite vector");
|
||||
let (a, b) = (f64::from(a), f64::from(b));
|
||||
dot += a * b;
|
||||
aa += a * a;
|
||||
bb += b * b;
|
||||
}
|
||||
ensure!(aa > 0.0 && bb > 0.0, "Zero vector");
|
||||
Ok(dot / (aa * bb).sqrt())
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
||||
ensure!(
|
||||
args.len() == 5,
|
||||
"Supply ASR_REPORT MEDIA_FIXTURE MODEL_DIR GOOGLE_WORKER NEW_OUTPUT_JSON"
|
||||
);
|
||||
ensure!(
|
||||
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
|
||||
"CPU only"
|
||||
);
|
||||
let asr_path = PathBuf::from(&args[0]);
|
||||
let fixture_path = PathBuf::from(&args[1]);
|
||||
let models = PathBuf::from(&args[2]);
|
||||
let worker = PathBuf::from(&args[3]);
|
||||
let output = PathBuf::from(&args[4]);
|
||||
fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&output)
|
||||
.context("Cannot reserve new output")?;
|
||||
let mut report =
|
||||
json!({"complete":false,"phase":"verification","rankings":{},"raw_queries":[]});
|
||||
checkpoint(&output, &report)?;
|
||||
ensure!(
|
||||
fs::metadata(&asr_path)?.len() <= 16 * 1024 * 1024,
|
||||
"Oversized ASR report"
|
||||
);
|
||||
let fixture_bytes = fs::read(&fixture_path)?;
|
||||
ensure!(
|
||||
format!("{:x}", Sha256::digest(&fixture_bytes)) == FIXTURE_SHA,
|
||||
"Frozen media fixture changed"
|
||||
);
|
||||
let fixture: Value = serde_json::from_slice(&fixture_bytes)?;
|
||||
let asr_bytes = fs::read(&asr_path)?;
|
||||
let asr_hash = format!("{:x}", Sha256::digest(&asr_bytes));
|
||||
let asr: Value = serde_json::from_slice(&asr_bytes)?;
|
||||
let (segments, excluded) = eligible(&asr, &fixture)?;
|
||||
ensure!(!segments.is_empty(), "No eligible recognized text");
|
||||
for asset in fixture["assets"]
|
||||
.as_array()
|
||||
.context("Missing assets")?
|
||||
.iter()
|
||||
.filter(|asset| ASSETS.contains(&asset["id"].as_str().unwrap_or_default()))
|
||||
{
|
||||
let source = fixture_path
|
||||
.parent()
|
||||
.context("Missing fixture parent")?
|
||||
.join(asset["path"].as_str().context("Missing asset path")?);
|
||||
ensure!(
|
||||
fs::metadata(&source)?.len() == number(&asset["bytes"])?
|
||||
&& digest(&source)? == asset["sha256"],
|
||||
"Frozen media source changed"
|
||||
);
|
||||
}
|
||||
let manifest_path =
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../embeddinggemma2/model-manifest.json");
|
||||
let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?;
|
||||
for asset in manifest["files"]
|
||||
.as_array()
|
||||
.context("Missing model files")?
|
||||
{
|
||||
let path = models.join(asset["path"].as_str().context("Missing model path")?);
|
||||
ensure!(
|
||||
fs::metadata(&path)?.len() == number(&asset["bytes"])?
|
||||
&& digest(&path)? == asset["sha256"],
|
||||
"Google model bytes changed"
|
||||
);
|
||||
}
|
||||
report["fixture_sha256"] = json!(FIXTURE_SHA);
|
||||
report["asr_report_sha256"] = json!(asr_hash);
|
||||
report["asr_provenance"] = asr.clone();
|
||||
report["google_worker_sha256"] = json!(digest(&worker)?);
|
||||
report["runner_sha256"] = json!(digest(&std::env::current_exe()?)?);
|
||||
report["google_model_manifest_sha256"] = json!(digest(&manifest_path)?);
|
||||
report["google_model"] = manifest;
|
||||
report["segments"] = json!(segments);
|
||||
report["excluded_windows"] = json!(excluded);
|
||||
report["excluded_window_count"] = json!(excluded.len());
|
||||
report["quality_score_eligible"] = json!(excluded.is_empty());
|
||||
report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration.");
|
||||
report["phase"] = json!("document_encoding");
|
||||
checkpoint(&output, &report)?;
|
||||
let start = Instant::now();
|
||||
let mut model = GemmaEmbedder::load_with_device(&models, &worker, 1, Device::Cpu)?;
|
||||
report["document_worker_info"] = json!(model.info);
|
||||
let mut vectors = Vec::new();
|
||||
let mut mappings = Vec::new();
|
||||
for (segment_id, segment) in segments.iter().enumerate() {
|
||||
let text = segment["text"].as_str().context("Missing text")?;
|
||||
let title = segment["title"].as_str().context("Missing title")?;
|
||||
for window in model.split_windows(text, title)? {
|
||||
let mut encoded = model.embed_documents(&[&window.text], title)?;
|
||||
ensure!(encoded.len() == 1, "Missing document embedding");
|
||||
vectors.push(encoded.remove(0));
|
||||
mappings.push(json!({"segment_id":segment_id,"text_start_byte":window.start_byte,"text_end_byte":window.end_byte}));
|
||||
ensure!(vectors.len() <= 10000, "Too many token windows");
|
||||
}
|
||||
}
|
||||
report["document_encoding_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0);
|
||||
report["vector_source_mapping"] = json!(mappings);
|
||||
drop(model);
|
||||
let start = Instant::now();
|
||||
let mut query_model = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?;
|
||||
report["query_worker_info"] = json!(query_model.info);
|
||||
report["query_startup_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0);
|
||||
let queries: Vec<_> = fixture["queries"]
|
||||
.as_array()
|
||||
.context("Missing fixture queries")?
|
||||
.iter()
|
||||
.filter(|query| query["category"] == "video-speech")
|
||||
.collect();
|
||||
ensure!(
|
||||
queries.len() == 8,
|
||||
"Exactly eight frozen speech queries required"
|
||||
);
|
||||
report["phase"] = json!("query_ranking");
|
||||
checkpoint(&output, &report)?;
|
||||
for query in queries {
|
||||
let start = Instant::now();
|
||||
let vector =
|
||||
query_model.embed_query(query["text"].as_str().context("Missing query text")?)?;
|
||||
let mut raw = Vec::new();
|
||||
let mut best: BTreeMap<usize, f64> = BTreeMap::new();
|
||||
for (id, document) in vectors.iter().enumerate() {
|
||||
let score = cosine(&vector, document)?;
|
||||
let segment_id = number(&mappings[id]["segment_id"])? as usize;
|
||||
best.entry(segment_id)
|
||||
.and_modify(|old| *old = old.max(score))
|
||||
.or_insert(score);
|
||||
raw.push(json!({"vector_id":id,"segment_id":segment_id,"cosine":score}));
|
||||
}
|
||||
raw.sort_by(|a, b| {
|
||||
b["cosine"]
|
||||
.as_f64()
|
||||
.unwrap()
|
||||
.total_cmp(&a["cosine"].as_f64().unwrap())
|
||||
.then_with(|| a["vector_id"].as_u64().cmp(&b["vector_id"].as_u64()))
|
||||
});
|
||||
let mut ranks: Vec<_> = best.into_iter().collect();
|
||||
ranks.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
|
||||
let all: Vec<_> = ranks.into_iter().map(|(id,score)| json!({"asset":segments[id]["asset"],"start_ms":segments[id]["start_ms"],"end_ms":segments[id]["end_ms"],"segment_id":id,"cosine":score})).collect();
|
||||
report["rankings"][query["id"].as_str().context("Missing query ID")?] =
|
||||
json!(all.iter().take(100).collect::<Vec<_>>());
|
||||
report["raw_queries"].as_array_mut().context("Missing output queries")?.push(json!({"id":query["id"],"text":query["text"],"elapsed_ms":start.elapsed().as_secs_f64()*1000.0,"all_segment_ranks":all,"all_vector_ranks":raw}));
|
||||
checkpoint(&output, &report)?;
|
||||
}
|
||||
ensure!(
|
||||
report["asr_report_sha256"] == digest(&asr_path)?,
|
||||
"ASR report changed during ranking"
|
||||
);
|
||||
ensure!(
|
||||
digest(&fixture_path)? == FIXTURE_SHA
|
||||
&& report["google_worker_sha256"] == digest(&worker)?
|
||||
&& report["google_model_manifest_sha256"] == digest(&manifest_path)?,
|
||||
"Fixture or runtime provenance changed during ranking"
|
||||
);
|
||||
report["complete"] = json!(true);
|
||||
report["phase"] = json!("complete");
|
||||
checkpoint(&output, &report)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn timestamps_are_absolute_milliseconds_and_invalid_tail_rejected() {
|
||||
let raw =
|
||||
json!({"transcription":[{"offsets":{"from":100,"to":2000},"text":" untouched text "}]});
|
||||
assert_eq!(
|
||||
parsed_segments(&raw, 30000, 3000).unwrap(),
|
||||
vec![json!({"start_ms":30100,"end_ms":32000,"text":" untouched text "})]
|
||||
);
|
||||
assert!(parsed_segments(&raw, 30000, 1000).is_err());
|
||||
assert!(parsed_segments(&raw, u64::MAX, 3000).is_err());
|
||||
assert!(parsed_segments(
|
||||
&json!({"transcription":[{"offsets":{"from":1.0,"to":2},"text":"x"}]}),
|
||||
0,
|
||||
30
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
#[test]
|
||||
fn incomplete_and_stale_asr_are_rejected_before_inference() {
|
||||
assert!(eligible(&json!({"complete":false}), &json!({})).is_err());
|
||||
assert!(eligible(
|
||||
&json!({"complete":true,"fixture_sha256":"stale"}),
|
||||
&json!({})
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_identity_coverage_and_recognized_text_cannot_be_forged() {
|
||||
let hash = "a".repeat(64);
|
||||
let assets: Vec<_> = ASSETS
|
||||
.iter()
|
||||
.map(|id| json!({"id":id,"sha256":hash,"path":format!("{id}.mp3")}))
|
||||
.collect();
|
||||
let raw = json!({"transcription":[{"offsets":{"from":10,"to":900},"text":"actual unit-test text"}]});
|
||||
let rows:Vec<_>=ASSETS.iter().map(|id|json!({"asset":id,"source_sha256":hash,"audio_start_ms":1000,"audio_end_ms":2000,"duration_ms":1000,"windows":[{"offset_ms":1000,"duration_ms":1000,"decoded_frames":16000,"actual_whisper_json":raw,"segments":parsed_segments(&raw,1000,1000).unwrap()}]})).collect();
|
||||
let report = json!({"version":1,"complete":true,"fixture_sha256":FIXTURE_SHA,"policy":POLICY,"backend":"forced CPU","threads":1,"upstream_commit":"d1be6fde11ac6e0407606b4e42fe72d34add8037","candidate":"base","model_sha256":"60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe","worker_sha256":hash,"harness_sha256":hash,"ffmpeg_sha256":hash,"ffprobe_sha256":hash,"quality_failed":false,"validation_failures":[],"rows":rows});
|
||||
let fixture = json!({"assets":assets});
|
||||
assert_eq!(eligible(&report, &fixture).unwrap().0.len(), 5);
|
||||
let mut changed = report.clone();
|
||||
changed["rows"][0]["asset"] = json!("unknown");
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
let mut changed = report.clone();
|
||||
changed["rows"][0]["source_sha256"] = json!("stale");
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
let mut changed = report.clone();
|
||||
changed["rows"][0]["windows"][0]["segments"][0]["start_ms"] = json!(10);
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
let mut changed = report.clone();
|
||||
changed["rows"][0]["windows"][0]["decoded_frames"] = json!(8000);
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
let mut changed = report;
|
||||
changed["rows"][0]["windows"][0]["validation_error"] = json!("hide a valid competitor");
|
||||
assert!(eligible(&changed, &fixture).is_err());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user