From 6353d02c042f7255d3443f1970a511b302696ca0 Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 18:11:07 +0200 Subject: [PATCH] Freeze actual Google music metadata and precise speech timing diagnostics --- tools/media-eval/Cargo.lock | 5 + tools/media-eval/README.md | 27 +++ tools/media-eval/src/bin/music_metadata.rs | 224 +++++++++++++++++++++ tools/speech-eval/README.md | 27 +++ tools/speech-eval/run.py | 30 ++- tools/speech-eval/test_token_alignment.py | 60 ++++++ tools/speech-eval/token-policy.json | 14 ++ tools/speech-eval/token_alignment.py | 54 +++++ 8 files changed, 440 insertions(+), 1 deletion(-) create mode 100644 tools/media-eval/src/bin/music_metadata.rs create mode 100644 tools/speech-eval/test_token_alignment.py create mode 100644 tools/speech-eval/token-policy.json create mode 100644 tools/speech-eval/token_alignment.py diff --git a/tools/media-eval/Cargo.lock b/tools/media-eval/Cargo.lock index d6e6d2e..2370be9 100644 --- a/tools/media-eval/Cargo.lock +++ b/tools/media-eval/Cargo.lock @@ -869,11 +869,16 @@ name = "searchi-core" version = "0.1.0" dependencies = [ "anyhow", + "libc", "lopdf", "quick-xml", "rusqlite", + "searchi-media", "serde", + "serde_json", + "sha2 0.10.9", "walkdir", + "windows-sys", "zip", ] diff --git a/tools/media-eval/README.md b/tools/media-eval/README.md index 9b39b1a..fcd7a0b 100644 --- a/tools/media-eval/README.md +++ b/tools/media-eval/README.md @@ -33,3 +33,30 @@ The dense reference samples video every two seconds and PCM in thirty-second win - Model agreement with another implementation, synthetic document-seed scores, and this small-media benchmark each answer different questions. None is production quality certification. Detailed provenance is in [the fixture README](../../fixtures/media/README.md). The final assessment belongs in [MEDIA-QUALITY.md](../../docs/MEDIA-QUALITY.md). + +## Music metadata hypothesis diagnostic + +`music_metadata` compares raw audio with the per-file maximum over real audio +and one Google-embedded metadata passage. It has no tuned weight or rejection +threshold and does not change production. `prepare` freezes actual FFprobe tags, +source hashes, default production audio windows, and all 65 existing queries +before inference. Existing plans/reports cannot be overwritten. + +```powershell +cargo test --manifest-path tools/media-eval/Cargo.toml --bin music_metadata --locked +cargo run --manifest-path tools/media-eval/Cargo.toml --bin music_metadata --locked -- prepare fixtures/media .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-metadata-v1-plan.json +$env:SEARCHI_FORCE_CPU = '1' +cargo run --manifest-path tools/media-eval/Cargo.toml --bin music_metadata --locked -- run fixtures/media .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe .searchi/qa/music-metadata-v1-plan.json .searchi/models/embeddinggemma2 .searchi/runtime/auto/Release/searchi-gemma-worker.exe .searchi/qa/music-metadata-v1-results.json +``` + +The whitelist is artist, title, album, genre and composer in that fixed order, +bounded to 2048 UTF-8 bytes. Missing/blank tags produce no metadata vector; +encoder, comment, URLs and copyright boilerplate are excluded. Literal misleading +tags remain misleading evidence: this tool does not verify musical instruments +or generate a description. Metadata evidence has no invented timestamp. + +This is the known two-music-file pool, not mixed production retrieval. Other-file +positives are explicitly ineligible, filters remain explicitly unsupported, and +all unfiltered negatives remain visible without abstention. Existing queries are +diagnostics rather than new held-out evidence. Unseen music with missing and +misleading tags needs independently frozen validation before deployment. diff --git a/tools/media-eval/src/bin/music_metadata.rs b/tools/media-eval/src/bin/music_metadata.rs new file mode 100644 index 0000000..ae28fed --- /dev/null +++ b/tools/media-eval/src/bin/music_metadata.rs @@ -0,0 +1,224 @@ +//! Frozen two-file hypothesis diagnostic; not production or abstention evidence. +use anyhow::{ensure, Context, Result}; +use searchi_media::{audio_windows, decode_audio, probe, FfmpegTools, MediaLimits}; +use searchi_semantic::{Device, GemmaEmbedder}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, fs, io::Write, path::PathBuf, time::Instant}; + +const POLICY: &str = + "music-metadata-v1:artist-title-album-genre-composer:2048bytes:max-real-evidence:no-cutoff"; +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn metadata_text(tags: &BTreeMap) -> String { + let mut text = String::new(); + for key in ["artist", "title", "album", "genre", "composer"] { + if let Some(value) = tags.get(key).filter(|v| !v.trim().is_empty()) { + let prefix = format!("{key}: "); + let available = 2048usize.saturating_sub(text.len() + prefix.len() + 1); + let mut end = value.len().min(available); + while !value.is_char_boundary(end) { + end -= 1; + } + if end == 0 { + break; + } + text.push_str(&prefix); + text.push_str(&value[..end]); + text.push('\n'); + } + } + text +} +fn write_new(path: &PathBuf, value: &Value) -> Result<()> { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path)?; + file.write_all(&serde_json::to_vec_pretty(value)?)?; + file.sync_all()?; + Ok(()) +} +fn main() -> Result<()> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + ensure!( + args.len() == 5 || args.len() == 8, + "prepare CORPUS FFMPEG FFPROBE PLAN or run CORPUS FFMPEG FFPROBE PLAN MODEL WORKER OUTPUT" + ); + let mode = args[0].to_string_lossy(); + ensure!( + (mode == "prepare" && args.len() == 5) || (mode == "run" && args.len() == 8), + "Invalid mode/arguments" + ); + let root = PathBuf::from(&args[1]); + let fixture_bytes = fs::read(root.join("fixture.json"))?; + let fixture: Value = serde_json::from_slice(&fixture_bytes)?; + let lock: Value = serde_json::from_slice(&fs::read(root.join("labels.lock.json"))?)?; + ensure!( + lock["fixture_sha256"] == digest(&fixture_bytes), + "Frozen fixture changed" + ); + let tools = FfmpegTools::new(PathBuf::from(&args[2]), PathBuf::from(&args[3]))?; + let limits = MediaLimits::default(); + let mut assets = Vec::new(); + let mut probes = Vec::new(); + for asset in fixture["assets"] + .as_array() + .context("Assets missing")? + .iter() + .filter(|a| a["kind"] == "music") + { + let path = root.join(asset["path"].as_str().context("Path missing")?); + ensure!( + asset["sha256"] == digest(&fs::read(&path)?), + "Music bytes changed" + ); + let metadata = probe(&tools, &path, &limits)?; + assets.push(json!({"id":asset["id"],"path":asset["path"],"sha256":asset["sha256"],"metadata_text":metadata_text(&metadata.tags),"all_actual_tags":metadata.tags,"audio_windows":audio_windows(&metadata,&limits)?})); + probes.push((path, metadata)); + } + ensure!( + assets.len() == 2, + "This diagnostic requires the known two-file pool" + ); + let plan = json!({"policy":POLICY,"fixture_sha256":digest(&fixture_bytes),"assets":assets,"queries":fixture["queries"],"scope":"All frozen queries retained; other-file positives and unsupported filters are not quality eligible. Raw audio versus per-file maximum of raw audio and real metadata; no threshold, no weights. Misleading tags are not verified musical descriptions; empty tags add no evidence."}); + let plan_path = PathBuf::from(&args[4]); + if mode == "prepare" { + return write_new(&plan_path, &plan); + } + ensure!( + std::env::var("SEARCHI_FORCE_CPU").as_deref() == Ok("1"), + "CPU required" + ); + let frozen_bytes = fs::read(&plan_path)?; + ensure!( + serde_json::from_slice::(&frozen_bytes)? == plan, + "Prepared inputs or policy changed" + ); + let output = PathBuf::from(&args[7]); + ensure!(!output.exists(), "Existing output must be preserved"); + let mut model = GemmaEmbedder::load_with_device(&args[5], &args[6], 1, Device::Cpu)?; + ensure!( + model.info.device.to_lowercase().contains("cpu"), + "Worker is not CPU" + ); + let started = Instant::now(); + let mut evidence = Vec::new(); + for (asset, (path, metadata)) in assets.iter().zip(&probes) { + for window in audio_windows(metadata, &limits)? { + let vector = model.embed_audio_pcm(&decode_audio(&tools, path, window, &limits)?)?; + evidence.push((json!({"asset":asset["id"],"kind":"audio","start_ms":window.start_ms,"duration_ms":window.duration_ms}),vector)); + } + let text = asset["metadata_text"] + .as_str() + .context("Metadata missing")?; + if !text.is_empty() { + let vectors = model.embed_documents(&[text], "Music file metadata")?; + ensure!(vectors.len() == 1, "Expected one metadata vector"); + evidence.push(( + json!({"asset":asset["id"],"kind":"metadata","text":text}), + vectors[0].clone(), + )); + } + ensure!( + asset["sha256"] == digest(&fs::read(path)?), + "Source changed during inference" + ); + println!("Encoded {}", asset["id"]); + } + let encoding_ms = started.elapsed().as_secs_f64() * 1000.0; + let mut rows = Vec::new(); + for query in fixture["queries"].as_array().context("Queries missing")? { + if query.get("filters").is_some() { + rows.push(json!({"query":query,"unsupported":"Pool-only diagnostic does not enforce production filters"})); + continue; + } + let vector = model.embed_query(query["text"].as_str().context("Query missing")?)?; + let scores: Vec<_> = evidence + .iter() + .map(|(source, encoded)| { + ( + source, + vector + .iter() + .zip(encoded) + .map(|(a, b)| f64::from(*a) * f64::from(*b)) + .sum::(), + ) + }) + .collect(); + let mut lanes = Vec::new(); + for metadata in [false, true] { + let mut files = Vec::new(); + for asset in &assets { + let best = scores + .iter() + .filter(|(source, _)| { + source["asset"] == asset["id"] && (metadata || source["kind"] == "audio") + }) + .max_by(|a, b| a.1.total_cmp(&b.1)) + .context("Missing audio evidence")?; + files.push(json!({"source":best.0,"score":best.1})); + } + files.sort_by(|a, b| { + b["score"] + .as_f64() + .unwrap() + .total_cmp(&a["score"].as_f64().unwrap()) + }); + lanes.push(json!({"metadata_enabled":metadata,"file_rankings":files})); + } + let eligible = query["expected"].as_array().is_some_and(|a| { + a.iter() + .all(|id| assets.iter().any(|asset| asset["id"] == *id)) + }); + rows.push(json!({"query":query,"pool_quality_eligible":eligible,"lanes":lanes})); + } + write_new( + &output, + &json!({"plan":plan,"plan_sha256":digest(&frozen_bytes),"worker":model.info,"worker_sha256":digest(&fs::read(&args[6])?),"encoding_ms":encoding_ms,"evidence_count":evidence.len(),"rows":rows,"production_policy_changed":false}), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn empty_and_untrusted_boilerplate_add_no_evidence() { + assert!(metadata_text(&BTreeMap::from([ + ("encoder".into(), "Violin".into()), + ("artist".into(), " ".into()) + ])) + .is_empty()); + } + #[test] + fn unicode_is_bounded_without_inventing_descriptions() { + let text = metadata_text(&BTreeMap::from([ + ("title".into(), "ä".repeat(3000)), + ("genre".into(), "Jazz".into()), + ])); + assert!(text.len() <= 2048); + assert!(text.starts_with("title: ä")); + assert!(!text.contains("Jazz")); + } + #[test] + fn stable_whitelist_order_and_literal_misleading_values() { + assert_eq!( + metadata_text(&BTreeMap::from([ + ("artist".into(), "A".into()), + ("title".into(), "Not actually a violin".into()) + ])), + "artist: A\ntitle: Not actually a violin\n" + ); + } + #[test] + fn existing_report_is_preserved() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("report.json"); + fs::write(&path, b"original")?; + assert!(write_new(&path, &json!({})).is_err()); + assert_eq!(fs::read(path)?, b"original"); + Ok(()) + } +} diff --git a/tools/speech-eval/README.md b/tools/speech-eval/README.md index 1e5cc1e..69fb0b3 100644 --- a/tools/speech-eval/README.md +++ b/tools/speech-eval/README.md @@ -62,3 +62,30 @@ English recording do not validate German speech or general transcription quality Pinned CLI flag/output reference: [upstream cli.cpp](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/examples/cli/cli.cpp). + +## Frozen ordinary token-time comparison + +`token-policy.json` declares an internal base-only comparison before inference. +Add `--token-sentences` to the existing command. The worker uses full JSON, +ordinary token timestamps and unchanged recognition flags; DTW is disabled. +The harness retains original raw JSON and segments, then adds exact recognized +sentence substrings and their token intervals in `aligned_segments`. No label +selects a sentence, source, word or start. All five sources and complete original +window coverage remain mandatory; empty speech is valid. + +Structural tokens cannot become passage text. Ordinary text-token times must be +present, ordered and inside the source segment/window; reconstructed token text +must match original recognized bytes. Invalid alignment is retained as an error +and invalidates an accepted score. No clipping, invented time or omitted failing +competitor is permitted. Rankers must independently reconstruct these passages +from raw JSON and verify policy/contract provenance before publishing quality. + +```powershell +$env:SEARCHI_FORCE_CPU = '1' +python tools/speech-eval/run.py --whisper .searchi/speech/build-cpu/bin/Release/whisper-cli.exe --whisper-sha256 12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8 --ffmpeg .searchi/media-runtime/bin/ffmpeg.exe --ffprobe .searchi/media-runtime/bin/ffprobe.exe --model .searchi/speech/models/ggml-base.bin --candidate base --token-sentences --scratch-parent .searchi/speech/token-scratch --output .searchi/speech/base-token-sentences-v1.json +``` + +Run only with the coordinator's available CPU inference slot. The previous +base-only run took about 251 seconds including decode/process startup. Token +alignment has not yet been timed; each ASR process retains the 600-second limit. +No production behavior or quality claim follows from structural tests. diff --git a/tools/speech-eval/run.py b/tools/speech-eval/run.py index ebd7057..3f44335 100644 --- a/tools/speech-eval/run.py +++ b/tools/speech-eval/run.py @@ -148,10 +148,19 @@ def main(): parser.add_argument("--ffprobe", type=Path, required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--candidate", choices=MODELS, required=True) + parser.add_argument("--token-sentences", action="store_true", help="Frozen ordinary token-time diagnostic; retains original segments") parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json")) parser.add_argument("--scratch-parent", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() + if args.output.exists(): + raise ValueError("Never overwrite experiment evidence") + if args.token_sentences and args.candidate != "base": + raise ValueError("Frozen token experiment accepts multilingual base only") + if args.token_sentences: + policy = json.loads(Path(__file__).with_name("token-policy.json").read_text(encoding="utf-8")) + if policy["worker_sha256"] != args.whisper_sha256: + raise ValueError("Frozen token experiment worker mismatch") size, digest = MODELS[args.candidate] if args.model.stat().st_size != size or sha256(args.model) != digest: raise ValueError("Model size/SHA verification failed") @@ -200,20 +209,34 @@ def main(): else: asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1", "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav), - "--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600) + "--output-json-full" if args.token_sentences else "--output-json", + "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600) raw = json.loads(output.read_text(encoding="utf-8")) result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms)) result["windows"][-1]["decoded_frames"] = frames + if args.token_sentences: + from token_alignment import aligned_segments + try: + result["windows"][-1]["aligned_segments"] = aligned_segments(raw, offset, length) + except (ValueError, KeyError, TypeError) as error: + result["windows"][-1]["alignment_validation_error"] = str(error) args.output.parent.mkdir(parents=True, exist_ok=True) checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture), "model_sha256": digest, "worker_sha256": args.whisper_sha256, "rows": [*rows, result]} + if args.token_sentences: + checkpoint["alignment_policy"] = policy["policy"] + checkpoint["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json")) args.output.write_text(json.dumps(checkpoint, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") if source.stat().st_size != asset["bytes"] or sha256(source) != asset["sha256"]: raise ValueError("Source changed during recognition") rows.append(result) failures = [{"asset": row["asset"], "offset_ms": window["offset_ms"], "error": window["validation_error"]} for row in rows for window in row["windows"] if "validation_error" in window] + if args.token_sentences: + failures.extend({"asset": row["asset"], "offset_ms": window["offset_ms"], + "error": window["alignment_validation_error"]} + for row in rows for window in row["windows"] if "alignment_validation_error" in window) report = {"version": 1, "complete": True, "quality_failed": bool(failures), "validation_failures": failures, "scope": "Actual ASR diagnostic, not Google retrieval or production integration", "fixture_sha256": sha256(args.fixture), "upstream_commit": UPSTREAM, @@ -223,6 +246,11 @@ def main(): "harness_sha256": sha256(Path(__file__)), "policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times", "rows": rows} + if args.token_sentences: + from token_alignment import POLICY + report["alignment_policy"] = POLICY + report["alignment_contract_sha256"] = sha256(Path(__file__).with_name("token_alignment.py")) + report["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json")) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") diff --git a/tools/speech-eval/test_token_alignment.py b/tools/speech-eval/test_token_alignment.py new file mode 100644 index 0000000..11b321d --- /dev/null +++ b/tools/speech-eval/test_token_alignment.py @@ -0,0 +1,60 @@ +"""Structural tests only; actual word-time quality requires genuine inference.""" +import unittest +from token_alignment import aligned_segments + + +def fixture(): + tokens = [{"id": 100 + index, "text": text, + "offsets": {"from": start, "to": end}} + for index, (text, start, end) in enumerate([ + (" Hello", 100, 400), (" world.", 400, 900), + (" Guten", 1100, 1400), (" Tag!", 1400, 1900)])] + return {"transcription": [{"text": " Hello world. Guten Tag!", + "offsets": {"from": 0, "to": 2000}, "tokens": tokens}]} + + +class TokenAlignmentTests(unittest.TestCase): + def test_exact_sentences_absolute_nonzero_source_coordinates(self): + result = aligned_segments(fixture(), 30000, 2000) + self.assertEqual([row["text"] for row in result], [" Hello world.", " Guten Tag!"]) + self.assertEqual([(row["start_ms"], row["end_ms"]) for row in result], + [(30100, 30900), (31100, 31900)]) + self.assertEqual(result[0]["original_start_ms"], 30000) + + def test_structural_tokens_never_enter_candidates(self): + raw = fixture() + raw["transcription"][0]["tokens"].insert(0, {"id": 50364, "text": "[_BEG_]"}) + self.assertEqual(len(aligned_segments(raw, 0, 2000)), 2) + + def test_missing_nonmonotonic_outside_segment_and_invalid_special_rejected(self): + mutations = [lambda token: token.pop("offsets"), + lambda token: token.update(offsets={"from": -1, "to": 100}), + lambda token: token.update(offsets={"from": 300, "to": 100}), + lambda token: token.update(offsets={"from": 0, "to": 2500}), + lambda token: token.update(id=50364, text="ordinary words"), + lambda token: token.update(text=" changed")] + for mutate in mutations: + raw = fixture() + mutate(raw["transcription"][0]["tokens"][1]) + with self.assertRaises(ValueError): + aligned_segments(raw, 0, 2000) + + def test_no_punctuation_preserves_whole_text(self): + raw = fixture() + for token in raw["transcription"][0]["tokens"]: + token["text"] = token["text"].rstrip(".!") + raw["transcription"][0]["text"] = "".join(t["text"] for t in raw["transcription"][0]["tokens"]) + result = aligned_segments(raw, 0, 2000) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["text"], raw["transcription"][0]["text"]) + + def test_empty_speech_is_valid_and_missing_tokens_are_rejected(self): + self.assertEqual(aligned_segments({"transcription": []}, 0, 1000), []) + raw = fixture() + del raw["transcription"][0]["tokens"] + with self.assertRaises(ValueError): + aligned_segments(raw, 0, 2000) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/speech-eval/token-policy.json b/tools/speech-eval/token-policy.json new file mode 100644 index 0000000..82baf6a --- /dev/null +++ b/tools/speech-eval/token-policy.json @@ -0,0 +1,14 @@ +{ + "schema": 1, + "policy": "ordinary-token-sentences-v1;punctuation-boundaries;exact-text;no-dtw;no-labels", + "candidate": "base", + "worker_sha256": "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8", + "fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393", + "assets": ["clip04", "music01", "music02", "sound01", "sound02"], + "query_count": 8, + "asr_flags": ["--no-gpu", "--threads", "1", "--processors", "1", "--language", "auto", "--output-json-full"], + "boundaries": "Sentence-final punctuation in ordinary token text; remainder retained, exact recognized text unchanged", + "acceptance": "Every original source/window retained; invalid token time or text invalidates accepted score; no label-driven passage selection", + "comparison": "Frozen base segment baseline; file top1, strict timestamp-start hits, interval overlap, recognition omissions and processing cost reported separately", + "production_default": false +} diff --git a/tools/speech-eval/token_alignment.py b/tools/speech-eval/token_alignment.py new file mode 100644 index 0000000..24e0787 --- /dev/null +++ b/tools/speech-eval/token_alignment.py @@ -0,0 +1,54 @@ +"""Strict ordinary Whisper token-time sentence subdivision, never label driven.""" +import re + +POLICY = "ordinary-token-sentences-v1;punctuation-boundaries;exact-text;no-dtw;no-labels" +# Pinned multilingual Whisper vocabulary: ordinary text ids precede EOT. +TEXT_TOKEN_LIMIT = 50257 +BOUNDARY = re.compile(r'[.!?。！？]["\u201d\u2019\')\]]*$') + + +def aligned_segments(document, offset_ms, duration_ms): + from run import segments + originals = segments(document, offset_ms, duration_ms) + result = [] + raw_rows = [row for row in document["transcription"] if row["text"].strip()] + for index, (row, original) in enumerate(zip(raw_rows, originals)): + tokens = row.get("tokens") + if not isinstance(tokens, list) or not 0 < len(tokens) <= 4096: + raise ValueError("Missing or oversized token times") + accepted = [] + previous = 0 + for token in tokens: + token_id, text = token.get("id"), token.get("text") + if type(token_id) is not int or token_id < 0 or not isinstance(text, str): + raise ValueError("Invalid token identity") + # Structural tokens must never become recognized text candidates. + if token_id >= TEXT_TOKEN_LIMIT: + if not text.startswith("[_") or not text.endswith("]"): + raise ValueError("Unrecognized special token") + continue + times = token.get("offsets", {}) + start, end = times.get("from"), times.get("to") + if (type(start) is not int or type(end) is not int + or not 0 <= start <= end <= duration_ms or start < previous + or not row["offsets"]["from"] <= start <= end <= row["offsets"]["to"]): + raise ValueError("Invalid ordinary token timestamps") + previous = start + accepted.append((text, start, end)) + if "".join(token[0] for token in accepted) != row["text"]: + raise ValueError("Token text differs from original recognized segment") + phrase = [] + for token_index, token in enumerate(accepted): + phrase.append(token) + if BOUNDARY.search(token[0]) or token_index == len(accepted) - 1: + text = "".join(part[0] for part in phrase) + if text.strip(): + result.append({"start_ms": offset_ms + phrase[0][1], + "end_ms": offset_ms + phrase[-1][2], "text": text, + "original_segment_index": index, + "original_start_ms": original["start_ms"], + "original_end_ms": original["end_ms"]}) + phrase = [] + if len(result) > 10000: + raise ValueError("Oversized aligned passage pool") + return result