Validate audio stream timelines and add genuine Google transcript diagnostics
This commit is contained in:
karl committed
1 parent
8ef874c609
commit
5eeba29c0d
7 files changed
+1067
-16
No files matched your search
+33
-14
@@ -108,10 +108,10 @@ def recognized_window(raw, offset, length, decode_ms, asr_ms):
|
|||||||
return report
|
return report
|
||||||
|
|
||||||
|
|
||||||
def decode_command(ffmpeg, source, offset, length, wav):
|
def decode_command(ffmpeg, source, offset, length, wav, audio_index=None):
|
||||||
return [str(ffmpeg.resolve()), "-nostdin", "-hide_banner", "-v", "error",
|
return [str(ffmpeg.resolve()), "-nostdin", "-hide_banner", "-v", "error",
|
||||||
"-protocol_whitelist", "file,pipe", "-threads", "1", "-ss", str(offset / 1000), "-t", str(length / 1000),
|
"-protocol_whitelist", "file,pipe", "-threads", "1", "-seek_timestamp", "1", "-ss", str(offset / 1000), "-t", str(length / 1000),
|
||||||
"-i", str(source.resolve()), "-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
|
"-i", str(source.resolve()), "-map", f"0:{audio_index}" if audio_index is not None else "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
|
||||||
"-af", f"aresample=16000,atrim=end_sample={length * 16},asetpts=PTS-STARTPTS",
|
"-af", f"aresample=16000,atrim=end_sample={length * 16},asetpts=PTS-STARTPTS",
|
||||||
"-c:a", "pcm_s16le", "-threads", "1", "-y", str(wav)]
|
"-c:a", "pcm_s16le", "-threads", "1", "-y", str(wav)]
|
||||||
|
|
||||||
@@ -127,6 +127,19 @@ def decoded_frames(wav, length):
|
|||||||
return frames
|
return frames
|
||||||
|
|
||||||
|
|
||||||
|
def primary_audio_range(probe):
|
||||||
|
audio = next((stream for stream in probe.get("streams", []) if stream.get("codec_type") == "audio"), None)
|
||||||
|
if audio is None:
|
||||||
|
raise ValueError("No primary audio stream")
|
||||||
|
start = float(audio.get("start_time", 0))
|
||||||
|
if "duration" not in audio:
|
||||||
|
raise ValueError("Primary audio stream duration unavailable; proof requires explicit stream duration, not container fallback")
|
||||||
|
duration = float(audio["duration"])
|
||||||
|
if not math.isfinite(start) or not math.isfinite(duration) or start < 0 or not 0 < duration <= 600:
|
||||||
|
raise ValueError("Proof audio stream range exceeds bounds")
|
||||||
|
return math.ceil(start * 1000), math.ceil((start + duration) * 1000), audio["index"]
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument("--whisper", type=Path, required=True)
|
parser.add_argument("--whisper", type=Path, required=True)
|
||||||
@@ -161,28 +174,34 @@ def main():
|
|||||||
raise ValueError(f"Frozen asset mismatch: {asset['id']}")
|
raise ValueError(f"Frozen asset mismatch: {asset['id']}")
|
||||||
# Probe stdout is bounded by the same owned log, then parsed directly.
|
# Probe stdout is bounded by the same owned log, then parsed directly.
|
||||||
run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe",
|
run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe",
|
||||||
"-show_entries", "format=duration", "-of", "json", str(source.resolve())],
|
"-show_entries", "stream=index,codec_type,duration,start_time:format=duration,start_time", "-of", "json", str(source.resolve())],
|
||||||
scratch, "probe", [], 30)
|
scratch, "probe", [], 30)
|
||||||
probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8"))
|
probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8"))
|
||||||
duration_s = float(probe["format"]["duration"])
|
start_ms, end_ms, audio_index = primary_audio_range(probe)
|
||||||
if not math.isfinite(duration_s) or not 0 < duration_s <= 600:
|
result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": end_ms - start_ms,
|
||||||
raise ValueError("Proof duration exceeds ten-minute bound")
|
"audio_start_ms": start_ms, "audio_end_ms": end_ms, "audio_stream_index": audio_index,
|
||||||
duration_ms = math.ceil(duration_s * 1000)
|
"actual_probe": probe, "windows": []}
|
||||||
result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": duration_ms, "windows": []}
|
for offset in range(start_ms, end_ms, 30000):
|
||||||
for offset in range(0, duration_ms, 30000):
|
length = min(30000, end_ms - offset)
|
||||||
length = min(30000, duration_ms - offset)
|
|
||||||
wav = scratch / "window.wav"
|
wav = scratch / "window.wav"
|
||||||
prefix = scratch / "recognized"
|
prefix = scratch / "recognized"
|
||||||
output = prefix.with_suffix(".json")
|
output = prefix.with_suffix(".json")
|
||||||
output.unlink(missing_ok=True)
|
output.unlink(missing_ok=True)
|
||||||
decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav),
|
decode_ms = run_owned(decode_command(args.ffmpeg, source, offset, length, wav, audio_index),
|
||||||
scratch, "decode", [(wav, 970000)], 60)
|
scratch, "decode", [(wav, 970000)], 60)
|
||||||
|
try:
|
||||||
frames = decoded_frames(wav, length)
|
frames = decoded_frames(wav, length)
|
||||||
|
except ValueError as error:
|
||||||
|
with wave.open(str(wav), "rb") as audio:
|
||||||
|
actual_frames = audio.getnframes()
|
||||||
|
result["windows"].append({"offset_ms": offset, "duration_ms": length, "decode_ms": decode_ms,
|
||||||
|
"decoded_frames": actual_frames, "validation_error": str(error), "error_stage": "decode_coverage",
|
||||||
|
"segments": [], "asr_not_run": True})
|
||||||
|
else:
|
||||||
asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
|
asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
|
||||||
"--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
|
"--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
|
||||||
"--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
|
"--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
|
||||||
raw = json.loads(output.read_text(encoding="utf-8"))
|
raw = json.loads(output.read_text(encoding="utf-8"))
|
||||||
# Preserve genuine invalid output, never clamp it into valid evidence.
|
|
||||||
result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
|
result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms))
|
||||||
result["windows"][-1]["decoded_frames"] = frames
|
result["windows"][-1]["decoded_frames"] = frames
|
||||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -202,7 +221,7 @@ def main():
|
|||||||
"ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe),
|
"ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe),
|
||||||
"backend": "forced CPU", "threads": 1,
|
"backend": "forced CPU", "threads": 1,
|
||||||
"harness_sha256": sha256(Path(__file__)),
|
"harness_sha256": sha256(Path(__file__)),
|
||||||
"policy": "aresample16000-before-atrim-v2;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
|
"policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
|
||||||
"rows": rows}
|
"rows": rows}
|
||||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||||
|
|||||||
@@ -6,6 +6,13 @@ from pathlib import Path
|
|||||||
from run import FIXTURE_SHA
|
from run import FIXTURE_SHA
|
||||||
|
|
||||||
|
|
||||||
|
def require_eligible(report):
|
||||||
|
if report.get("complete") is not True or report.get("quality_score_eligible") is not True:
|
||||||
|
raise ValueError("Cannot score incomplete runs or reduced/invalid competitor pools")
|
||||||
|
if report.get("excluded_windows") or report.get("excluded_window_count", 0):
|
||||||
|
raise ValueError("Cannot score rankings with excluded ASR windows")
|
||||||
|
|
||||||
|
|
||||||
def score(fixture, rankings):
|
def score(fixture, rankings):
|
||||||
queries = [query for query in fixture["queries"] if query["category"] == "video-speech"]
|
queries = [query for query in fixture["queries"] if query["category"] == "video-speech"]
|
||||||
if len(queries) != 8:
|
if len(queries) != 8:
|
||||||
@@ -41,6 +48,7 @@ def main():
|
|||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
data = args.fixture.read_bytes()
|
data = args.fixture.read_bytes()
|
||||||
rankings = json.loads(args.rankings.read_text(encoding="utf-8"))
|
rankings = json.loads(args.rankings.read_text(encoding="utf-8"))
|
||||||
|
require_eligible(rankings)
|
||||||
digest = hashlib.sha256(data).hexdigest()
|
digest = hashlib.sha256(data).hexdigest()
|
||||||
if digest != FIXTURE_SHA or rankings["fixture_sha256"] != digest or not rankings.get("asr_report_sha256") or not rankings.get("google_worker_sha256"):
|
if digest != FIXTURE_SHA or rankings["fixture_sha256"] != digest or not rankings.get("asr_report_sha256") or not rankings.get("google_worker_sha256"):
|
||||||
raise ValueError("Missing/mismatched actual ASR/Google provenance")
|
raise ValueError("Missing/mismatched actual ASR/Google provenance")
|
||||||
|
|||||||
@@ -6,11 +6,49 @@ import tempfile
|
|||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
import wave
|
import wave
|
||||||
from run import segments, run_owned, recognized_window, decode_command, decoded_frames
|
from run import segments, run_owned, recognized_window, decode_command, decoded_frames, primary_audio_range
|
||||||
from score import score
|
from score import score, require_eligible
|
||||||
|
|
||||||
|
|
||||||
class ContractTests(unittest.TestCase):
|
class ContractTests(unittest.TestCase):
|
||||||
|
def test_scoring_rejects_incomplete_or_reduced_pool(self):
|
||||||
|
for report in [{}, {"complete": False, "quality_score_eligible": True},
|
||||||
|
{"complete": True, "quality_score_eligible": False},
|
||||||
|
{"complete": True, "quality_score_eligible": True, "excluded_windows": [{}]}]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
require_eligible(report)
|
||||||
|
require_eligible({"complete": True, "quality_score_eligible": True, "excluded_windows": []})
|
||||||
|
def test_primary_audio_duration_and_nonzero_start_ignore_longer_video(self):
|
||||||
|
probe = {"streams": [{"index": 0, "codec_type": "video", "duration": "90"},
|
||||||
|
{"index": 1, "codec_type": "audio", "start_time": "0.125", "duration": "31"}]}
|
||||||
|
self.assertEqual(primary_audio_range(probe), (125, 31125, 1))
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get("SEARCHI_TEST_FFMPEG"), "Requires verified CPU FFmpeg")
|
||||||
|
def test_actual_video_longer_than_audio(self):
|
||||||
|
ffmpeg = Path(os.environ["SEARCHI_TEST_FFMPEG"])
|
||||||
|
ffprobe = ffmpeg.with_name("ffprobe.exe" if os.name == "nt" else "ffprobe")
|
||||||
|
with tempfile.TemporaryDirectory() as parent:
|
||||||
|
scratch = Path(parent)
|
||||||
|
source = scratch / "longer-video.mkv"
|
||||||
|
run_owned([str(ffmpeg), "-nostdin", "-hide_banner", "-v", "error", "-threads", "1",
|
||||||
|
"-f", "lavfi", "-i", "color=c=black:s=16x16:r=1:d=33",
|
||||||
|
"-itsoffset", "0.125", "-f", "lavfi", "-i", "anullsrc=r=48000:cl=mono:d=31", "-c:v", "ffv1",
|
||||||
|
"-c:a", "pcm_s16le", "-threads", "1", "-output_ts_offset", "5", "-y", str(source)],
|
||||||
|
scratch, "mux", [(source, 4 * 1024 * 1024)], 30)
|
||||||
|
run_owned([str(ffprobe), "-v", "error", "-show_entries", "stream=index,codec_type,duration,start_time:stream_tags=DURATION:format=duration", "-of", "json", str(source)], scratch, "probe", [], 30)
|
||||||
|
probe = json.loads((scratch / "probe.log").read_text())
|
||||||
|
audio = next(row for row in probe["streams"] if row["codec_type"] == "audio")
|
||||||
|
# Matroska stores duration in tags; test conversion is explicit, not a format fallback.
|
||||||
|
if "duration" not in audio:
|
||||||
|
h, m, sec = audio["tags"]["DURATION"].split(":")
|
||||||
|
audio["duration"] = int(h) * 3600 + int(m) * 60 + float(sec) - float(audio["start_time"])
|
||||||
|
start, end, _ = primary_audio_range(probe)
|
||||||
|
self.assertEqual((start, end), (5125, 36125))
|
||||||
|
self.assertGreater(float(probe["format"]["duration"]), end / 1000)
|
||||||
|
wav = scratch / "tail.wav"
|
||||||
|
run_owned(decode_command(ffmpeg, source, 35125, 1000, wav, audio["index"]), scratch, "tail", [(wav, 970000)], 30)
|
||||||
|
self.assertGreaterEqual(decoded_frames(wav, 1000), 15984)
|
||||||
|
|
||||||
@unittest.skipUnless(os.environ.get("SEARCHI_TEST_FFMPEG"), "Requires verified CPU FFmpeg")
|
@unittest.skipUnless(os.environ.get("SEARCHI_TEST_FFMPEG"), "Requires verified CPU FFmpeg")
|
||||||
def test_actual_ffmpeg_full_window_and_partial_tail_sample_coverage(self):
|
def test_actual_ffmpeg_full_window_and_partial_tail_sample_coverage(self):
|
||||||
ffmpeg = Path(os.environ["SEARCHI_TEST_FFMPEG"])
|
ffmpeg = Path(os.environ["SEARCHI_TEST_FFMPEG"])
|
||||||
|
|||||||
Generated
+486
@@ -0,0 +1,486 @@
|
|||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "adler2"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anyhow"
|
||||||
|
version = "1.0.104"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "autocfg"
|
||||||
|
version = "1.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "base64"
|
||||||
|
version = "0.22.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bitflags"
|
||||||
|
version = "2.13.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "block-buffer"
|
||||||
|
version = "0.10.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||||
|
dependencies = [
|
||||||
|
"generic-array",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bytemuck"
|
||||||
|
version = "1.25.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "byteorder-lite"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfg-if"
|
||||||
|
version = "1.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cpufeatures"
|
||||||
|
version = "0.2.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crc32fast"
|
||||||
|
version = "1.5.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crypto-common"
|
||||||
|
version = "0.1.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||||
|
dependencies = [
|
||||||
|
"generic-array",
|
||||||
|
"typenum",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "digest"
|
||||||
|
version = "0.10.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||||
|
dependencies = [
|
||||||
|
"block-buffer",
|
||||||
|
"crypto-common",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "errno"
|
||||||
|
version = "0.3.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fastrand"
|
||||||
|
version = "2.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fdeflate"
|
||||||
|
version = "0.3.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
|
||||||
|
dependencies = [
|
||||||
|
"simd-adler32",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "flate2"
|
||||||
|
version = "1.1.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||||
|
dependencies = [
|
||||||
|
"crc32fast",
|
||||||
|
"miniz_oxide 0.9.1",
|
||||||
|
"zlib-rs",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "generic-array"
|
||||||
|
version = "0.14.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||||
|
dependencies = [
|
||||||
|
"typenum",
|
||||||
|
"version_check",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "getrandom"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"libc",
|
||||||
|
"r-efi",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "image"
|
||||||
|
version = "0.25.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||||
|
dependencies = [
|
||||||
|
"bytemuck",
|
||||||
|
"byteorder-lite",
|
||||||
|
"image-webp",
|
||||||
|
"moxcms",
|
||||||
|
"num-traits",
|
||||||
|
"png",
|
||||||
|
"zune-core",
|
||||||
|
"zune-jpeg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "image-webp"
|
||||||
|
version = "0.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder-lite",
|
||||||
|
"quick-error",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "itoa"
|
||||||
|
version = "1.0.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "jpeg-decoder"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libc"
|
||||||
|
version = "0.2.190"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "linux-raw-sys"
|
||||||
|
version = "0.12.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memchr"
|
||||||
|
version = "2.8.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "miniz_oxide"
|
||||||
|
version = "0.8.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||||
|
dependencies = [
|
||||||
|
"adler2",
|
||||||
|
"simd-adler32",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "miniz_oxide"
|
||||||
|
version = "0.9.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||||
|
dependencies = [
|
||||||
|
"adler2",
|
||||||
|
"simd-adler32",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "moxcms"
|
||||||
|
version = "0.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
"pxfm",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-traits"
|
||||||
|
version = "0.2.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "once_cell"
|
||||||
|
version = "1.21.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "png"
|
||||||
|
version = "0.18.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"crc32fast",
|
||||||
|
"fdeflate",
|
||||||
|
"flate2",
|
||||||
|
"miniz_oxide 0.8.9",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.107"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pxfm"
|
||||||
|
version = "0.1.30"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quick-error"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.47"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "r-efi"
|
||||||
|
version = "6.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustix"
|
||||||
|
version = "1.1.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"errno",
|
||||||
|
"libc",
|
||||||
|
"linux-raw-sys",
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "searchi-semantic"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"base64",
|
||||||
|
"image",
|
||||||
|
"jpeg-decoder",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"sha2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "searchi-speech-rank"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"searchi-semantic",
|
||||||
|
"serde_json",
|
||||||
|
"sha2",
|
||||||
|
"tempfile",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||||
|
dependencies = [
|
||||||
|
"serde_core",
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_core"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||||
|
dependencies = [
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_derive"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_json"
|
||||||
|
version = "1.0.154"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
||||||
|
dependencies = [
|
||||||
|
"itoa",
|
||||||
|
"memchr",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
"zmij",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sha2"
|
||||||
|
version = "0.10.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"cpufeatures",
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "simd-adler32"
|
||||||
|
version = "0.3.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "3.0.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tempfile"
|
||||||
|
version = "3.27.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||||
|
dependencies = [
|
||||||
|
"fastrand",
|
||||||
|
"getrandom",
|
||||||
|
"once_cell",
|
||||||
|
"rustix",
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "typenum"
|
||||||
|
version = "1.20.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.27"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "version_check"
|
||||||
|
version = "0.9.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-link"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-sys"
|
||||||
|
version = "0.61.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zlib-rs"
|
||||||
|
version = "0.6.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zmij"
|
||||||
|
version = "1.0.23"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-core"
|
||||||
|
version = "0.5.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-jpeg"
|
||||||
|
version = "0.5.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
|
||||||
|
dependencies = [
|
||||||
|
"zune-core",
|
||||||
|
]
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
[package]
|
||||||
|
name = "searchi-speech-rank"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
publish = false
|
||||||
|
[workspace]
|
||||||
|
[dependencies]
|
||||||
|
anyhow = "1"
|
||||||
|
serde_json = "1"
|
||||||
|
sha2 = "0.10"
|
||||||
|
searchi-semantic = { path = "../../crates/searchi-semantic" }
|
||||||
|
tempfile = "3"
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# Genuine Google transcript ranking diagnostic
|
||||||
|
|
||||||
|
This internal experiment ranks **actual recognized ASR text** with the shared
|
||||||
|
Google EmbeddingGemma 2 adapter. It is separate from production ANN/hybrid search
|
||||||
|
and does not add speech recognition to the application.
|
||||||
|
|
||||||
|
All valid segments from the NASA video and four music/sound competitors enter
|
||||||
|
the same candidate pool. Original source filenames are the only document titles;
|
||||||
|
recognized text is unchanged. Frozen relevance intervals never select, rewrite,
|
||||||
|
filter or rank candidates. General Google search queries use the normal 512-token
|
||||||
|
query role; document encoding uses 2048 tokens. The document process is dropped
|
||||||
|
before opening the query process. Both use forced CPU and one thread.
|
||||||
|
|
||||||
|
Long segments use the actual Google tokenizer's source-preserving windows.
|
||||||
|
Every vector's original segment and UTF-8 byte range are recorded. Exact cosine
|
||||||
|
scores are computed from genuine Google vectors and aggregated by maximum over
|
||||||
|
token windows belonging to the same original ASR segment. All vector and segment
|
||||||
|
ranks are retained; at most the first 100 segment hits per query are exported for
|
||||||
|
the existing scorer. Ties use stable source order. No confidence threshold or
|
||||||
|
oracle matching is applied.
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cargo test --manifest-path tools/speech-rank/Cargo.toml --locked
|
||||||
|
cargo build --release --manifest-path tools/speech-rank/Cargo.toml --locked
|
||||||
|
$env:SEARCHI_FORCE_CPU = '1'
|
||||||
|
tools/speech-rank/target/release/searchi-speech-rank.exe .searchi/speech/base-results.json fixtures/media/fixture.json C:/path/to/verified-models C:/path/to/searchi-gemma-worker.exe .searchi/speech/base-google-ranks.json
|
||||||
|
python tools/speech-eval/score.py --rankings .searchi/speech/base-google-ranks.json --output .searchi/speech/base-google-score.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Run only after the shared inference CPU slot is released. The output directory
|
||||||
|
must exist, and the output filename must be new. `create_new` reserves evidence
|
||||||
|
before costly work; atomic JSON checkpoints retain incomplete phases. Inputs
|
||||||
|
are bounded, source/model hashes verified, and recognized segment timestamps
|
||||||
|
reconstructed from actual Whisper JSON to reject stale normalized text. Reports
|
||||||
|
from truncated decoding or an older policy are rejected. The artifact records
|
||||||
|
the actual ASR bytes, worker/runner/model hashes and process/timing information.
|
||||||
|
|
||||||
|
Invalid ASR windows are explicitly marked ineligible with reasons, never quietly
|
||||||
|
discarded. A report with any excluded window has `quality_score_eligible: false`:
|
||||||
|
do not publish a quality score from a reduced competitor pool. Valid hallucinated
|
||||||
|
music or sound text remains eligible and must compete. `complete` reports prove
|
||||||
|
the diagnostic finished, not speech accuracy.
|
||||||
|
|
||||||
|
Exported `start_ms`/`end_ms` refer to the full original media timeline. They are
|
||||||
|
ASR segment approximations, potentially affected by 30-second recognition
|
||||||
|
boundaries; they are **not word alignment**. A token window retains its containing
|
||||||
|
segment's same interval. The existing eight frozen NASA queries constitute a tiny
|
||||||
|
diagnostic, not held-out calibration, general media quality or production proof.
|
||||||
@@ -0,0 +1,440 @@
|
|||||||
|
//! Genuine Google retrieval over recognized ASR text; labels never select segments.
|
||||||
|
use anyhow::{ensure, Context, Result};
|
||||||
|
use searchi_semantic::{Device, GemmaEmbedder, GEMMA_DIMENSIONS};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::{
|
||||||
|
collections::{BTreeMap, BTreeSet},
|
||||||
|
fs,
|
||||||
|
io::{Read, Write},
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
time::Instant,
|
||||||
|
};
|
||||||
|
|
||||||
|
const FIXTURE_SHA: &str = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393";
|
||||||
|
const ASSETS: [&str; 5] = ["clip04", "music01", "music02", "sound01", "sound02"];
|
||||||
|
const POLICY: &str = "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times";
|
||||||
|
|
||||||
|
fn digest(path: &Path) -> Result<String> {
|
||||||
|
let mut file = fs::File::open(path)?;
|
||||||
|
let mut hash = Sha256::new();
|
||||||
|
let mut buffer = [0; 65536];
|
||||||
|
loop {
|
||||||
|
let n = file.read(&mut buffer)?;
|
||||||
|
if n == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
hash.update(&buffer[..n]);
|
||||||
|
}
|
||||||
|
Ok(format!("{:x}", hash.finalize()))
|
||||||
|
}
|
||||||
|
fn number(value: &Value) -> Result<u64> {
|
||||||
|
value
|
||||||
|
.as_u64()
|
||||||
|
.context("Expected unsigned integer milliseconds")
|
||||||
|
}
|
||||||
|
fn checkpoint(path: &Path, report: &Value) -> Result<()> {
|
||||||
|
let parent = path
|
||||||
|
.parent()
|
||||||
|
.filter(|p| !p.as_os_str().is_empty())
|
||||||
|
.unwrap_or(Path::new("."));
|
||||||
|
let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
|
||||||
|
temporary.write_all(&serde_json::to_vec_pretty(report)?)?;
|
||||||
|
temporary.as_file().sync_all()?;
|
||||||
|
temporary.persist(path)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reconstruct absolute segment coordinates from actual whisper output, never labels.
|
||||||
|
fn parsed_segments(raw: &Value, offset: u64, duration: u64) -> Result<Vec<Value>> {
|
||||||
|
let rows = raw["transcription"]
|
||||||
|
.as_array()
|
||||||
|
.context("Missing actual transcription")?;
|
||||||
|
ensure!(rows.len() <= 1000, "Oversized transcription");
|
||||||
|
let mut segments = Vec::new();
|
||||||
|
let mut previous = 0;
|
||||||
|
for row in rows {
|
||||||
|
let start = number(&row["offsets"]["from"])?;
|
||||||
|
let end = number(&row["offsets"]["to"])?;
|
||||||
|
let text = row["text"].as_str().context("Invalid recognized text")?;
|
||||||
|
ensure!(
|
||||||
|
start >= previous && start <= end && end <= duration && text.len() <= 4096,
|
||||||
|
"Invalid actual ASR segment"
|
||||||
|
);
|
||||||
|
previous = start;
|
||||||
|
if !text.trim().is_empty() {
|
||||||
|
segments.push(json!({"start_ms":offset.checked_add(start).context("Timestamp overflow")?,"end_ms":offset.checked_add(end).context("Timestamp overflow")?,"text":text}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(segments)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec<Value>, Vec<Value>)> {
|
||||||
|
ensure!(
|
||||||
|
asr["version"] == 1 && asr["upstream_commit"] == "d1be6fde11ac6e0407606b4e42fe72d34add8037",
|
||||||
|
"Unknown ASR implementation"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
matches!(
|
||||||
|
(asr["candidate"].as_str(), asr["model_sha256"].as_str()),
|
||||||
|
(
|
||||||
|
Some("base"),
|
||||||
|
Some("60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe")
|
||||||
|
) | (
|
||||||
|
Some("small"),
|
||||||
|
Some("1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b")
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"Unknown ASR model"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
asr["complete"] == true
|
||||||
|
&& asr["fixture_sha256"] == FIXTURE_SHA
|
||||||
|
&& asr["policy"] == POLICY
|
||||||
|
&& asr["backend"] == "forced CPU"
|
||||||
|
&& asr["threads"] == 1,
|
||||||
|
"Incomplete, stale, or unrecognized ASR report"
|
||||||
|
);
|
||||||
|
for field in [
|
||||||
|
"model_sha256",
|
||||||
|
"worker_sha256",
|
||||||
|
"harness_sha256",
|
||||||
|
"ffmpeg_sha256",
|
||||||
|
"ffprobe_sha256",
|
||||||
|
] {
|
||||||
|
ensure!(asr[field].as_str().is_some_and(|hash| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit())), "Missing ASR provenance");
|
||||||
|
}
|
||||||
|
let rows = asr["rows"].as_array().context("Missing ASR rows")?;
|
||||||
|
ensure!(rows.len() == 5, "All five competing assets required");
|
||||||
|
let mut seen = BTreeSet::new();
|
||||||
|
let mut segments = Vec::new();
|
||||||
|
let mut excluded = Vec::new();
|
||||||
|
for row in rows {
|
||||||
|
let asset_id = row["asset"].as_str().context("Missing asset ID")?;
|
||||||
|
ensure!(
|
||||||
|
ASSETS.contains(&asset_id) && seen.insert(asset_id),
|
||||||
|
"Unknown/duplicate asset"
|
||||||
|
);
|
||||||
|
let asset = fixture["assets"]
|
||||||
|
.as_array()
|
||||||
|
.context("Missing fixture assets")?
|
||||||
|
.iter()
|
||||||
|
.find(|asset| asset["id"] == asset_id)
|
||||||
|
.context("Unknown source")?;
|
||||||
|
ensure!(
|
||||||
|
row["source_sha256"] == asset["sha256"],
|
||||||
|
"Stale source provenance"
|
||||||
|
);
|
||||||
|
let start = number(&row["audio_start_ms"])?;
|
||||||
|
let end = number(&row["audio_end_ms"])?;
|
||||||
|
ensure!(
|
||||||
|
start < end && end - start <= 600000 && number(&row["duration_ms"])? == end - start,
|
||||||
|
"Unbounded audio range"
|
||||||
|
);
|
||||||
|
let windows = row["windows"].as_array().context("Missing windows")?;
|
||||||
|
ensure!(windows.len() <= 20, "Oversized window list");
|
||||||
|
let mut next = start;
|
||||||
|
for (window_index, window) in windows.iter().enumerate() {
|
||||||
|
let offset = number(&window["offset_ms"])?;
|
||||||
|
let duration = number(&window["duration_ms"])?;
|
||||||
|
ensure!(
|
||||||
|
offset == next && duration == (end - offset).min(30000) && duration > 0,
|
||||||
|
"Missing, overlapping or truncated source windows"
|
||||||
|
);
|
||||||
|
next = offset.checked_add(duration).context("Window overflow")?;
|
||||||
|
if let Some(error) = window.get("validation_error") {
|
||||||
|
ensure!(
|
||||||
|
error.as_str().is_some_and(|error| !error.is_empty()),
|
||||||
|
"Invalid exclusion reason"
|
||||||
|
);
|
||||||
|
if window["error_stage"] == "decode_coverage" {
|
||||||
|
let frames = number(&window["decoded_frames"])?;
|
||||||
|
ensure!(
|
||||||
|
window["asr_not_run"] == true
|
||||||
|
&& (frames > duration * 16
|
||||||
|
|| frames.saturating_add(16) < duration * 16),
|
||||||
|
"False decode exclusion"
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
ensure!(
|
||||||
|
parsed_segments(&window["actual_whisper_json"], offset, duration).is_err(),
|
||||||
|
"Valid recognized text must not be excluded"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
excluded.push(json!({"asset":asset_id,"window_index":window_index,"offset_ms":offset,"duration_ms":duration,"eligible":false,"reason":error}));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let frames = number(&window["decoded_frames"])?;
|
||||||
|
ensure!(
|
||||||
|
frames <= duration * 16 && frames + 16 >= duration * 16,
|
||||||
|
"Stale/truncated decoded audio coverage"
|
||||||
|
);
|
||||||
|
let actual = parsed_segments(&window["actual_whisper_json"], offset, duration)?;
|
||||||
|
ensure!(
|
||||||
|
window["segments"] == json!(actual),
|
||||||
|
"Normalized ASR segments differ from actual output"
|
||||||
|
);
|
||||||
|
let title = Path::new(asset["path"].as_str().context("Missing source filename")?)
|
||||||
|
.file_name()
|
||||||
|
.context("Missing filename")?
|
||||||
|
.to_string_lossy();
|
||||||
|
for (segment_index, segment) in actual.into_iter().enumerate() {
|
||||||
|
segments.push(json!({"asset":asset_id,"window_index":window_index,"segment_index":segment_index,"source_window_offset_ms":offset,"source_window_duration_ms":duration,"start_ms":segment["start_ms"],"end_ms":segment["end_ms"],"text":segment["text"],"title":title}));
|
||||||
|
ensure!(segments.len() <= 5000, "Too many recognized segments");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ensure!(next == end, "Incomplete source-window coverage");
|
||||||
|
}
|
||||||
|
ensure!(
|
||||||
|
asr["quality_failed"] == !excluded.is_empty(),
|
||||||
|
"ASR validation status inconsistent"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
asr["validation_failures"]
|
||||||
|
.as_array()
|
||||||
|
.context("Missing validation failures")?
|
||||||
|
.len()
|
||||||
|
== excluded.len(),
|
||||||
|
"ASR validation failure count inconsistent"
|
||||||
|
);
|
||||||
|
Ok((segments, excluded))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cosine(a: &[f32], b: &[f32]) -> Result<f64> {
|
||||||
|
ensure!(
|
||||||
|
a.len() == GEMMA_DIMENSIONS && b.len() == GEMMA_DIMENSIONS,
|
||||||
|
"Wrong vector dimensions"
|
||||||
|
);
|
||||||
|
let (mut dot, mut aa, mut bb) = (0.0f64, 0.0f64, 0.0f64);
|
||||||
|
for (&a, &b) in a.iter().zip(b) {
|
||||||
|
ensure!(a.is_finite() && b.is_finite(), "Nonfinite vector");
|
||||||
|
let (a, b) = (f64::from(a), f64::from(b));
|
||||||
|
dot += a * b;
|
||||||
|
aa += a * a;
|
||||||
|
bb += b * b;
|
||||||
|
}
|
||||||
|
ensure!(aa > 0.0 && bb > 0.0, "Zero vector");
|
||||||
|
Ok(dot / (aa * bb).sqrt())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() -> Result<()> {
|
||||||
|
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
||||||
|
ensure!(
|
||||||
|
args.len() == 5,
|
||||||
|
"Supply ASR_REPORT MEDIA_FIXTURE MODEL_DIR GOOGLE_WORKER NEW_OUTPUT_JSON"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"),
|
||||||
|
"CPU only"
|
||||||
|
);
|
||||||
|
let asr_path = PathBuf::from(&args[0]);
|
||||||
|
let fixture_path = PathBuf::from(&args[1]);
|
||||||
|
let models = PathBuf::from(&args[2]);
|
||||||
|
let worker = PathBuf::from(&args[3]);
|
||||||
|
let output = PathBuf::from(&args[4]);
|
||||||
|
fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.open(&output)
|
||||||
|
.context("Cannot reserve new output")?;
|
||||||
|
let mut report =
|
||||||
|
json!({"complete":false,"phase":"verification","rankings":{},"raw_queries":[]});
|
||||||
|
checkpoint(&output, &report)?;
|
||||||
|
ensure!(
|
||||||
|
fs::metadata(&asr_path)?.len() <= 16 * 1024 * 1024,
|
||||||
|
"Oversized ASR report"
|
||||||
|
);
|
||||||
|
let fixture_bytes = fs::read(&fixture_path)?;
|
||||||
|
ensure!(
|
||||||
|
format!("{:x}", Sha256::digest(&fixture_bytes)) == FIXTURE_SHA,
|
||||||
|
"Frozen media fixture changed"
|
||||||
|
);
|
||||||
|
let fixture: Value = serde_json::from_slice(&fixture_bytes)?;
|
||||||
|
let asr_bytes = fs::read(&asr_path)?;
|
||||||
|
let asr_hash = format!("{:x}", Sha256::digest(&asr_bytes));
|
||||||
|
let asr: Value = serde_json::from_slice(&asr_bytes)?;
|
||||||
|
let (segments, excluded) = eligible(&asr, &fixture)?;
|
||||||
|
ensure!(!segments.is_empty(), "No eligible recognized text");
|
||||||
|
for asset in fixture["assets"]
|
||||||
|
.as_array()
|
||||||
|
.context("Missing assets")?
|
||||||
|
.iter()
|
||||||
|
.filter(|asset| ASSETS.contains(&asset["id"].as_str().unwrap_or_default()))
|
||||||
|
{
|
||||||
|
let source = fixture_path
|
||||||
|
.parent()
|
||||||
|
.context("Missing fixture parent")?
|
||||||
|
.join(asset["path"].as_str().context("Missing asset path")?);
|
||||||
|
ensure!(
|
||||||
|
fs::metadata(&source)?.len() == number(&asset["bytes"])?
|
||||||
|
&& digest(&source)? == asset["sha256"],
|
||||||
|
"Frozen media source changed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let manifest_path =
|
||||||
|
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../embeddinggemma2/model-manifest.json");
|
||||||
|
let manifest: Value = serde_json::from_slice(&fs::read(&manifest_path)?)?;
|
||||||
|
for asset in manifest["files"]
|
||||||
|
.as_array()
|
||||||
|
.context("Missing model files")?
|
||||||
|
{
|
||||||
|
let path = models.join(asset["path"].as_str().context("Missing model path")?);
|
||||||
|
ensure!(
|
||||||
|
fs::metadata(&path)?.len() == number(&asset["bytes"])?
|
||||||
|
&& digest(&path)? == asset["sha256"],
|
||||||
|
"Google model bytes changed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
report["fixture_sha256"] = json!(FIXTURE_SHA);
|
||||||
|
report["asr_report_sha256"] = json!(asr_hash);
|
||||||
|
report["asr_provenance"] = asr.clone();
|
||||||
|
report["google_worker_sha256"] = json!(digest(&worker)?);
|
||||||
|
report["runner_sha256"] = json!(digest(&std::env::current_exe()?)?);
|
||||||
|
report["google_model_manifest_sha256"] = json!(digest(&manifest_path)?);
|
||||||
|
report["google_model"] = manifest;
|
||||||
|
report["segments"] = json!(segments);
|
||||||
|
report["excluded_windows"] = json!(excluded);
|
||||||
|
report["excluded_window_count"] = json!(excluded.len());
|
||||||
|
report["quality_score_eligible"] = json!(excluded.is_empty());
|
||||||
|
report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration.");
|
||||||
|
report["phase"] = json!("document_encoding");
|
||||||
|
checkpoint(&output, &report)?;
|
||||||
|
let start = Instant::now();
|
||||||
|
let mut model = GemmaEmbedder::load_with_device(&models, &worker, 1, Device::Cpu)?;
|
||||||
|
report["document_worker_info"] = json!(model.info);
|
||||||
|
let mut vectors = Vec::new();
|
||||||
|
let mut mappings = Vec::new();
|
||||||
|
for (segment_id, segment) in segments.iter().enumerate() {
|
||||||
|
let text = segment["text"].as_str().context("Missing text")?;
|
||||||
|
let title = segment["title"].as_str().context("Missing title")?;
|
||||||
|
for window in model.split_windows(text, title)? {
|
||||||
|
let mut encoded = model.embed_documents(&[&window.text], title)?;
|
||||||
|
ensure!(encoded.len() == 1, "Missing document embedding");
|
||||||
|
vectors.push(encoded.remove(0));
|
||||||
|
mappings.push(json!({"segment_id":segment_id,"text_start_byte":window.start_byte,"text_end_byte":window.end_byte}));
|
||||||
|
ensure!(vectors.len() <= 10000, "Too many token windows");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report["document_encoding_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0);
|
||||||
|
report["vector_source_mapping"] = json!(mappings);
|
||||||
|
drop(model);
|
||||||
|
let start = Instant::now();
|
||||||
|
let mut query_model = GemmaEmbedder::load_query_with_device(&models, &worker, 1, Device::Cpu)?;
|
||||||
|
report["query_worker_info"] = json!(query_model.info);
|
||||||
|
report["query_startup_ms"] = json!(start.elapsed().as_secs_f64() * 1000.0);
|
||||||
|
let queries: Vec<_> = fixture["queries"]
|
||||||
|
.as_array()
|
||||||
|
.context("Missing fixture queries")?
|
||||||
|
.iter()
|
||||||
|
.filter(|query| query["category"] == "video-speech")
|
||||||
|
.collect();
|
||||||
|
ensure!(
|
||||||
|
queries.len() == 8,
|
||||||
|
"Exactly eight frozen speech queries required"
|
||||||
|
);
|
||||||
|
report["phase"] = json!("query_ranking");
|
||||||
|
checkpoint(&output, &report)?;
|
||||||
|
for query in queries {
|
||||||
|
let start = Instant::now();
|
||||||
|
let vector =
|
||||||
|
query_model.embed_query(query["text"].as_str().context("Missing query text")?)?;
|
||||||
|
let mut raw = Vec::new();
|
||||||
|
let mut best: BTreeMap<usize, f64> = BTreeMap::new();
|
||||||
|
for (id, document) in vectors.iter().enumerate() {
|
||||||
|
let score = cosine(&vector, document)?;
|
||||||
|
let segment_id = number(&mappings[id]["segment_id"])? as usize;
|
||||||
|
best.entry(segment_id)
|
||||||
|
.and_modify(|old| *old = old.max(score))
|
||||||
|
.or_insert(score);
|
||||||
|
raw.push(json!({"vector_id":id,"segment_id":segment_id,"cosine":score}));
|
||||||
|
}
|
||||||
|
raw.sort_by(|a, b| {
|
||||||
|
b["cosine"]
|
||||||
|
.as_f64()
|
||||||
|
.unwrap()
|
||||||
|
.total_cmp(&a["cosine"].as_f64().unwrap())
|
||||||
|
.then_with(|| a["vector_id"].as_u64().cmp(&b["vector_id"].as_u64()))
|
||||||
|
});
|
||||||
|
let mut ranks: Vec<_> = best.into_iter().collect();
|
||||||
|
ranks.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
|
||||||
|
let all: Vec<_> = ranks.into_iter().map(|(id,score)| json!({"asset":segments[id]["asset"],"start_ms":segments[id]["start_ms"],"end_ms":segments[id]["end_ms"],"segment_id":id,"cosine":score})).collect();
|
||||||
|
report["rankings"][query["id"].as_str().context("Missing query ID")?] =
|
||||||
|
json!(all.iter().take(100).collect::<Vec<_>>());
|
||||||
|
report["raw_queries"].as_array_mut().context("Missing output queries")?.push(json!({"id":query["id"],"text":query["text"],"elapsed_ms":start.elapsed().as_secs_f64()*1000.0,"all_segment_ranks":all,"all_vector_ranks":raw}));
|
||||||
|
checkpoint(&output, &report)?;
|
||||||
|
}
|
||||||
|
ensure!(
|
||||||
|
report["asr_report_sha256"] == digest(&asr_path)?,
|
||||||
|
"ASR report changed during ranking"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
digest(&fixture_path)? == FIXTURE_SHA
|
||||||
|
&& report["google_worker_sha256"] == digest(&worker)?
|
||||||
|
&& report["google_model_manifest_sha256"] == digest(&manifest_path)?,
|
||||||
|
"Fixture or runtime provenance changed during ranking"
|
||||||
|
);
|
||||||
|
report["complete"] = json!(true);
|
||||||
|
report["phase"] = json!("complete");
|
||||||
|
checkpoint(&output, &report)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
#[test]
|
||||||
|
fn timestamps_are_absolute_milliseconds_and_invalid_tail_rejected() {
|
||||||
|
let raw =
|
||||||
|
json!({"transcription":[{"offsets":{"from":100,"to":2000},"text":" untouched text "}]});
|
||||||
|
assert_eq!(
|
||||||
|
parsed_segments(&raw, 30000, 3000).unwrap(),
|
||||||
|
vec![json!({"start_ms":30100,"end_ms":32000,"text":" untouched text "})]
|
||||||
|
);
|
||||||
|
assert!(parsed_segments(&raw, 30000, 1000).is_err());
|
||||||
|
assert!(parsed_segments(&raw, u64::MAX, 3000).is_err());
|
||||||
|
assert!(parsed_segments(
|
||||||
|
&json!({"transcription":[{"offsets":{"from":1.0,"to":2},"text":"x"}]}),
|
||||||
|
0,
|
||||||
|
30
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn incomplete_and_stale_asr_are_rejected_before_inference() {
|
||||||
|
assert!(eligible(&json!({"complete":false}), &json!({})).is_err());
|
||||||
|
assert!(eligible(
|
||||||
|
&json!({"complete":true,"fixture_sha256":"stale"}),
|
||||||
|
&json!({})
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn source_identity_coverage_and_recognized_text_cannot_be_forged() {
|
||||||
|
let hash = "a".repeat(64);
|
||||||
|
let assets: Vec<_> = ASSETS
|
||||||
|
.iter()
|
||||||
|
.map(|id| json!({"id":id,"sha256":hash,"path":format!("{id}.mp3")}))
|
||||||
|
.collect();
|
||||||
|
let raw = json!({"transcription":[{"offsets":{"from":10,"to":900},"text":"actual unit-test text"}]});
|
||||||
|
let rows:Vec<_>=ASSETS.iter().map(|id|json!({"asset":id,"source_sha256":hash,"audio_start_ms":1000,"audio_end_ms":2000,"duration_ms":1000,"windows":[{"offset_ms":1000,"duration_ms":1000,"decoded_frames":16000,"actual_whisper_json":raw,"segments":parsed_segments(&raw,1000,1000).unwrap()}]})).collect();
|
||||||
|
let report = json!({"version":1,"complete":true,"fixture_sha256":FIXTURE_SHA,"policy":POLICY,"backend":"forced CPU","threads":1,"upstream_commit":"d1be6fde11ac6e0407606b4e42fe72d34add8037","candidate":"base","model_sha256":"60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe","worker_sha256":hash,"harness_sha256":hash,"ffmpeg_sha256":hash,"ffprobe_sha256":hash,"quality_failed":false,"validation_failures":[],"rows":rows});
|
||||||
|
let fixture = json!({"assets":assets});
|
||||||
|
assert_eq!(eligible(&report, &fixture).unwrap().0.len(), 5);
|
||||||
|
let mut changed = report.clone();
|
||||||
|
changed["rows"][0]["asset"] = json!("unknown");
|
||||||
|
assert!(eligible(&changed, &fixture).is_err());
|
||||||
|
let mut changed = report.clone();
|
||||||
|
changed["rows"][0]["source_sha256"] = json!("stale");
|
||||||
|
assert!(eligible(&changed, &fixture).is_err());
|
||||||
|
let mut changed = report.clone();
|
||||||
|
changed["rows"][0]["windows"][0]["segments"][0]["start_ms"] = json!(10);
|
||||||
|
assert!(eligible(&changed, &fixture).is_err());
|
||||||
|
let mut changed = report.clone();
|
||||||
|
changed["rows"][0]["windows"][0]["decoded_frames"] = json!(8000);
|
||||||
|
assert!(eligible(&changed, &fixture).is_err());
|
||||||
|
let mut changed = report;
|
||||||
|
changed["rows"][0]["windows"][0]["validation_error"] = json!("hide a valid competitor");
|
||||||
|
assert!(eligible(&changed, &fixture).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user