diff --git a/tools/speech-eval/README.md b/tools/speech-eval/README.md index 69fb0b3..886ed89 100644 --- a/tools/speech-eval/README.md +++ b/tools/speech-eval/README.md @@ -89,3 +89,31 @@ Run only with the coordinator's available CPU inference slot. The previous base-only run took about 251 seconds including decode/process startup. Token alignment has not yet been timed; each ASR process retains the 600-second limit. No production behavior or quality claim follows from structural tests. + +## Separate DTW point-span comparison + +`dtw-policy.json` freezes a separate base experiment. Replace `--token-sentences` +with `--dtw-sentences` and use a new output such as +`.searchi/speech/base-dtw-sentences-v1.json`. This adds `--dtw base +--no-flash-attn`: the pinned runtime disables DTW when flash attention is on. +Native `t_dtw` is a centisecond point, approximately the moment a token is output, +not a start/end interval. Multiply by ten; first and last ordinary-token points +define an approximate sentence point span. One-token sentences can have zero span. +Only source-window-bounded globally monotonic integer points are accepted. +Original ordinary interval errors are retained separately and never repaired. +Missing/invalid DTW points invalidate accepted scoring for the new comparison. +Frozen ordinary-v1 output remains failed and unchanged. + +Pinned native default DTW graph workspace is 128 MiB; this is additional workspace, +not a total memory ceiling. Attention storage and disabled flash attention can +increase memory/time. Input remains 30 seconds, one CPU thread, 4 MiB output, +4096 tokens/segment and 600 seconds/process; observed peak memory/runtime must +be measured before production use. No inference has established this policy. +Future harness reports capture the executing-source SHA at startup and preload +alignment helpers before processing. The first ordinary-v1 run read its harness +SHA at completion while the on-disk harness had changed; that field cannot prove +the exact executing harness bytes. Its original raw token failures remain valid +diagnostic observations, and its excluded window already prohibits an accepted +quality score. Preserve that historical limitation instead of rewriting output. +References: pinned [token data/API](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/include/whisper.h) +and [DTW implementation](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/src/whisper.cpp). diff --git a/tools/speech-eval/dtw-policy.json b/tools/speech-eval/dtw-policy.json new file mode 100644 index 0000000..fb961c5 --- /dev/null +++ b/tools/speech-eval/dtw-policy.json @@ -0,0 +1,13 @@ +{ + "schema": 1, + "policy": "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels", + "candidate": "base", + "worker_sha256": "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8", + "fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393", + "assets": ["clip04", "music01", "music02", "sound01", "sound02"], + "query_count": 8, + "asr_flags": ["--no-gpu", "--threads", "1", "--processors", "1", "--language", "auto", "--output-json-full", "--dtw", "base", "--no-flash-attn"], + "point_units": "Native t_dtw centiseconds multiplied by ten; first/last ordinary-token points define approximate sentence point span, never native token duration", + "acceptance": "All sources/windows retained; ordinary interval failures retained separately; only valid source-window-bounded globally monotonic DTW points can produce accepted aligned candidates", + "production_default": false +} diff --git a/tools/speech-eval/dtw_alignment.py b/tools/speech-eval/dtw_alignment.py new file mode 100644 index 0000000..eade9d6 --- /dev/null +++ b/tools/speech-eval/dtw_alignment.py @@ -0,0 +1,47 @@ +"""Frozen DTW sentence point spans, independent of ordinary token intervals.""" +from token_alignment import BOUNDARY, TEXT_TOKEN_LIMIT + +POLICY = "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels" + + +def aligned_segments(document, offset_ms, duration_ms): + from run import segments + originals = segments(document, offset_ms, duration_ms) + raw_rows = [row for row in document["transcription"] if row["text"].strip()] + result = [] + previous = 0 + for index, (row, original) in enumerate(zip(raw_rows, originals)): + tokens = row.get("tokens") + if not isinstance(tokens, list) or not 0 < len(tokens) <= 4096: + raise ValueError("Missing or oversized DTW tokens") + accepted = [] + for token in tokens: + token_id, text = token.get("id"), token.get("text") + if type(token_id) is not int or token_id < 0 or not isinstance(text, str): + raise ValueError("Invalid DTW token identity") + if token_id >= TEXT_TOKEN_LIMIT: + if not text.startswith("[_") or not text.endswith("]"): + raise ValueError("Unrecognized special token") + continue + point = token.get("t_dtw") + if type(point) is not int or not 0 <= point * 10 <= duration_ms or point * 10 < previous: + raise ValueError("Missing, invalid or nonmonotonic DTW point") + previous = point * 10 + accepted.append((text, previous)) + if "".join(token[0] for token in accepted) != row["text"]: + raise ValueError("DTW text differs from original recognized segment") + phrase = [] + for token_index, token in enumerate(accepted): + phrase.append(token) + if BOUNDARY.search(token[0]) or token_index == len(accepted) - 1: + text = "".join(part[0] for part in phrase) + if text.strip(): + result.append({"start_ms": offset_ms + phrase[0][1], + "end_ms": offset_ms + phrase[-1][1], "text": text, + "original_segment_index": index, + "original_start_ms": original["start_ms"], + "original_end_ms": original["end_ms"]}) + phrase = [] + if len(result) > 10000: + raise ValueError("Oversized DTW passages") + return result diff --git a/tools/speech-eval/run.py b/tools/speech-eval/run.py index 3f44335..7447ab6 100644 --- a/tools/speech-eval/run.py +++ b/tools/speech-eval/run.py @@ -19,6 +19,8 @@ MODELS = { ASSETS = ("clip04", "music01", "music02", "sound01", "sound02") FIXTURE_SHA = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393" MAX_OUTPUT = 4 * 1024 * 1024 +# Capture before any expensive work; end-of-run reads can describe another file. +HARNESS_SHA256_AT_START = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() def sha256(path): @@ -149,18 +151,32 @@ def main(): parser.add_argument("--model", type=Path, required=True) parser.add_argument("--candidate", choices=MODELS, required=True) parser.add_argument("--token-sentences", action="store_true", help="Frozen ordinary token-time diagnostic; retains original segments") + parser.add_argument("--dtw-sentences", action="store_true", help="Separate frozen DTW point-span diagnostic") parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json")) parser.add_argument("--scratch-parent", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() if args.output.exists(): raise ValueError("Never overwrite experiment evidence") - if args.token_sentences and args.candidate != "base": + if args.token_sentences and args.dtw_sentences: + raise ValueError("Select one separate alignment experiment") + alignment_mode = args.token_sentences or args.dtw_sentences + alignment_file = "dtw_alignment.py" if args.dtw_sentences else "token_alignment.py" + policy_file = "dtw-policy.json" if args.dtw_sentences else "token-policy.json" + if alignment_mode and args.candidate != "base": raise ValueError("Frozen token experiment accepts multilingual base only") - if args.token_sentences: - policy = json.loads(Path(__file__).with_name("token-policy.json").read_text(encoding="utf-8")) + if alignment_mode: + policy = json.loads(Path(__file__).with_name(policy_file).read_text(encoding="utf-8")) if policy["worker_sha256"] != args.whisper_sha256: raise ValueError("Frozen token experiment worker mismatch") + alignment_contract_sha = sha256(Path(__file__).with_name(alignment_file)) + alignment_policy_sha = sha256(Path(__file__).with_name(policy_file)) + # Load helpers now; future file edits cannot change a later lazy import. + if args.dtw_sentences: + from dtw_alignment import aligned_segments + from token_alignment import aligned_segments as ordinary_segments + else: + from token_alignment import aligned_segments size, digest = MODELS[args.candidate] if args.model.stat().st_size != size or sha256(args.model) != digest: raise ValueError("Model size/SHA verification failed") @@ -209,31 +225,38 @@ def main(): else: asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1", "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav), - "--output-json-full" if args.token_sentences else "--output-json", + "--output-json-full" if alignment_mode else "--output-json", + *(["--dtw", "base", "--no-flash-attn"] if args.dtw_sentences else []), "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600) raw = json.loads(output.read_text(encoding="utf-8")) result["windows"].append(recognized_window(raw, offset, length, decode_ms, asr_ms)) result["windows"][-1]["decoded_frames"] = frames - if args.token_sentences: - from token_alignment import aligned_segments + if alignment_mode: + if args.dtw_sentences: + try: + ordinary_segments(raw, offset, length) + except (ValueError, KeyError, TypeError) as error: + result["windows"][-1]["ordinary_alignment_validation_error"] = str(error) try: result["windows"][-1]["aligned_segments"] = aligned_segments(raw, offset, length) except (ValueError, KeyError, TypeError) as error: result["windows"][-1]["alignment_validation_error"] = str(error) args.output.parent.mkdir(parents=True, exist_ok=True) checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture), + "harness_sha256": HARNESS_SHA256_AT_START, "model_sha256": digest, "worker_sha256": args.whisper_sha256, "rows": [*rows, result]} - if args.token_sentences: + if alignment_mode: checkpoint["alignment_policy"] = policy["policy"] - checkpoint["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json")) + checkpoint["alignment_policy_sha256"] = alignment_policy_sha + checkpoint["alignment_contract_sha256"] = alignment_contract_sha args.output.write_text(json.dumps(checkpoint, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") if source.stat().st_size != asset["bytes"] or sha256(source) != asset["sha256"]: raise ValueError("Source changed during recognition") rows.append(result) failures = [{"asset": row["asset"], "offset_ms": window["offset_ms"], "error": window["validation_error"]} for row in rows for window in row["windows"] if "validation_error" in window] - if args.token_sentences: + if alignment_mode: failures.extend({"asset": row["asset"], "offset_ms": window["offset_ms"], "error": window["alignment_validation_error"]} for row in rows for window in row["windows"] if "alignment_validation_error" in window) @@ -243,14 +266,14 @@ def main(): "worker_sha256": args.whisper_sha256, "model_sha256": digest, "candidate": args.candidate, "ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe), "backend": "forced CPU", "threads": 1, - "harness_sha256": sha256(Path(__file__)), + "harness_sha256": HARNESS_SHA256_AT_START, + "harness_file_changed_during_run": sha256(Path(__file__)) != HARNESS_SHA256_AT_START, "policy": "primary-audio-range-v3;aresample16000-before-atrim;samplecoverage-1ms;nonoverlapping30s;independent-process-per-window;auto-language;segment-times", "rows": rows} - if args.token_sentences: - from token_alignment import POLICY - report["alignment_policy"] = POLICY - report["alignment_contract_sha256"] = sha256(Path(__file__).with_name("token_alignment.py")) - report["alignment_policy_sha256"] = sha256(Path(__file__).with_name("token-policy.json")) + if alignment_mode: + report["alignment_policy"] = policy["policy"] + report["alignment_contract_sha256"] = alignment_contract_sha + report["alignment_policy_sha256"] = alignment_policy_sha args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") diff --git a/tools/speech-eval/test_contract.py b/tools/speech-eval/test_contract.py index 5954172..e5ab1b3 100644 --- a/tools/speech-eval/test_contract.py +++ b/tools/speech-eval/test_contract.py @@ -6,11 +6,24 @@ import tempfile import sys import os import wave +import importlib.util +import hashlib from run import segments, run_owned, recognized_window, decode_command, decoded_frames, primary_audio_range from score import score, require_eligible class ContractTests(unittest.TestCase): + def test_harness_source_hash_is_captured_before_file_changes(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "run-copy.py" + original = Path(__file__).with_name("run.py").read_bytes() + path.write_bytes(original) + spec = importlib.util.spec_from_file_location("speech_start_hash_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + path.write_bytes(original + b"\n# changed after startup\n") + self.assertEqual(module.HARNESS_SHA256_AT_START, hashlib.sha256(original).hexdigest()) + self.assertNotEqual(module.HARNESS_SHA256_AT_START, module.sha256(path)) def test_scoring_rejects_incomplete_or_reduced_pool(self): for report in [{}, {"complete": False, "quality_score_eligible": True}, {"complete": True, "quality_score_eligible": False}, diff --git a/tools/speech-eval/test_dtw_alignment.py b/tools/speech-eval/test_dtw_alignment.py new file mode 100644 index 0000000..a53dbdb --- /dev/null +++ b/tools/speech-eval/test_dtw_alignment.py @@ -0,0 +1,21 @@ +import unittest +from dtw_alignment import aligned_segments + + +class DtwTests(unittest.TestCase): + def raw(self): + return {"transcription": [{"text": " Hello. Again!", "offsets": {"from": 0, "to": 1000}, + "tokens": [{"id": 100, "text": " Hello.", "t_dtw": 20, "offsets": {"from": 900, "to": 100}}, + {"id": 101, "text": " Again!", "t_dtw": 70}]}]} + + def test_points_are_centiseconds_and_ordinary_reversal_not_repaired(self): + result = aligned_segments(self.raw(), 30000, 1000) + self.assertEqual([(r["start_ms"], r["end_ms"]) for r in result], [(30200, 30200), (30700, 30700)]) + self.assertEqual("".join(r["text"] for r in result), " Hello. Again!") + + def test_invalid_missing_fractional_and_nonmonotonic_points_rejected(self): + for point in [-1, 101, 1.5, None, 10]: + raw = self.raw() + raw["transcription"][0]["tokens"][1]["t_dtw"] = point + with self.assertRaises(ValueError): + aligned_segments(raw, 0, 1000) diff --git a/tools/speech-rank/README.md b/tools/speech-rank/README.md index 4aefa0e..ae5311e 100644 --- a/tools/speech-rank/README.md +++ b/tools/speech-rank/README.md @@ -75,3 +75,11 @@ use the existing executable command with input `.searchi/speech/base-token-sentences-v1.json` and a new output filename, then the same frozen eight-query scorer. Do not overwrite old results or select passages with relevance labels. + +The separate `dtw-token-sentences-v1` experiment independently validates native +DTW centisecond points and uses the first/last token points as approximate +sentence spans. These are not token durations. Ordinary interval failures remain +in the embedded ASR evidence and are independently checked; they do not authorize +fabricating or clamping DTW points. Missing/invalid DTW points exclude a window +and prevent accepted scoring. Use a separate DTW report/output and retain the +failed ordinary-v1 run. The frozen five sources/eight queries remain unchanged. diff --git a/tools/speech-rank/src/alignment.rs b/tools/speech-rank/src/alignment.rs index 29405a7..83fceeb 100644 --- a/tools/speech-rank/src/alignment.rs +++ b/tools/speech-rank/src/alignment.rs @@ -7,6 +7,11 @@ pub const POLICY: &str = "ordinary-token-sentences-v1;punctuation-boundaries;exact-text;no-dtw;no-labels"; pub const POLICY_SHA: &str = "5bc603e792a3abcc2370d4d1f7a13f538f808fb3b959b506a16d5823d3d7a126"; pub const CONTRACT_SHA: &str = "aac049c5bf3e17b07f4f2f7d92bce18c27aef5a67364181e47815301e75ca520"; +pub const DTW_POLICY: &str = + "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels"; +pub const DTW_POLICY_SHA: &str = "0ea19b1a2ee9da75f0284de33aeca0fcdff0a77d88d232b0b25821a7115b83b5"; +pub const DTW_CONTRACT_SHA: &str = + "61594d0e77c561d5d3b7c9e63178a8123841516b8f8fa781458b21ab717ad47f"; pub fn enabled(asr: &Value) -> Result { if asr.get("alignment_policy").is_none() { @@ -18,9 +23,12 @@ pub fn enabled(asr: &Value) -> Result { return Ok(false); } ensure!( - asr["alignment_policy"] == POLICY + ((asr["alignment_policy"] == POLICY && asr["alignment_policy_sha256"] == POLICY_SHA - && asr["alignment_contract_sha256"] == CONTRACT_SHA + && asr["alignment_contract_sha256"] == CONTRACT_SHA) + || (asr["alignment_policy"] == DTW_POLICY + && asr["alignment_policy_sha256"] == DTW_POLICY_SHA + && asr["alignment_contract_sha256"] == DTW_CONTRACT_SHA)) && asr["candidate"] == "base" && asr["worker_sha256"] == "12c681b4bcd9c7679b647b9a041e121a2f9e72cb2a9c6f189f529ec8b79c36f8" @@ -38,12 +46,17 @@ fn boundary(text: &str) -> bool { } pub fn reconstruct(raw: &Value, offset: u64, duration: u64) -> Result> { + reconstruct_mode(raw, offset, duration, false) +} + +pub fn reconstruct_mode(raw: &Value, offset: u64, duration: u64, dtw: bool) -> Result> { let originals = parsed_segments(raw, offset, duration)?; let rows = raw["transcription"] .as_array() .context("Missing transcription")?; let mut result = Vec::new(); let mut original_index = 0; + let mut dtw_previous = 0; for row in rows { let text = row["text"].as_str().context("Missing original text")?; if text.trim().is_empty() { @@ -70,14 +83,29 @@ pub fn reconstruct(raw: &Value, offset: u64, duration: u64) -> Result ); continue; } - let start = number(&token["offsets"]["from"])?; - let end = number(&token["offsets"]["to"])?; + let (start, end) = if dtw { + let point = number(&token["t_dtw"])? + .checked_mul(10) + .context("DTW point overflow")?; + ensure!( + point >= dtw_previous && point <= duration, + "Invalid DTW source point" + ); + dtw_previous = point; + (point, point) + } else { + ( + number(&token["offsets"]["from"])?, + number(&token["offsets"]["to"])?, + ) + }; ensure!( start >= previous && start <= end && end <= duration - && start >= number(&row["offsets"]["from"])? - && end <= number(&row["offsets"]["to"])?, + && (dtw + || (start >= number(&row["offsets"]["from"])? + && end <= number(&row["offsets"]["to"])?)), "Invalid ordinary token time" ); previous = start; @@ -187,4 +215,23 @@ mod tests { changed["candidate"] = json!("small"); assert!(enabled(&changed).is_err()); } + #[test] + fn dtw_points_are_independent_bounded_monotonic_centiseconds() { + let mut raw = raw(); + for (index, point) in [(1, 20), (2, 70), (3, 90)] { + raw["transcription"][0]["tokens"][index]["t_dtw"] = json!(point); + } + raw["transcription"][0]["tokens"][1]["offsets"] = json!({"from":900,"to":100}); + assert!(reconstruct(&raw, 0, 2000).is_err()); + let result = reconstruct_mode(&raw, 30000, 2000, true).unwrap(); + assert_eq!(result[0]["start_ms"], 30200); + assert_eq!(result[0]["end_ms"], 30200); + assert_eq!(result[1]["start_ms"], 30700); + assert_eq!(result[1]["end_ms"], 30900); + for point in [json!(-1), json!(201), json!(1.5), json!(10), Value::Null] { + let mut changed = raw.clone(); + changed["transcription"][0]["tokens"][2]["t_dtw"] = point; + assert!(reconstruct_mode(&changed, 0, 2000, true).is_err()); + } + } } diff --git a/tools/speech-rank/src/main.rs b/tools/speech-rank/src/main.rs index 1f3fc12..1f4097f 100644 --- a/tools/speech-rank/src/main.rs +++ b/tools/speech-rank/src/main.rs @@ -190,8 +190,31 @@ fn eligible(asr: &Value, fixture: &Value) -> Result<(Vec, Vec)> { "Normalized ASR segments differ from actual output" ); let actual = if token_alignment { - let rebuilt = - alignment::reconstruct(&window["actual_whisper_json"], offset, duration); + let dtw = asr["alignment_policy"] == alignment::DTW_POLICY; + if dtw { + let ordinary_failed = + alignment::reconstruct(&window["actual_whisper_json"], offset, duration) + .is_err(); + ensure!( + window.get("ordinary_alignment_validation_error").is_some() + == ordinary_failed, + "Ordinary token failure evidence inconsistent" + ); + if ordinary_failed { + ensure!( + window["ordinary_alignment_validation_error"] + .as_str() + .is_some_and(|error| !error.is_empty()), + "Invalid ordinary failure reason" + ); + } + } + let rebuilt = alignment::reconstruct_mode( + &window["actual_whisper_json"], + offset, + duration, + dtw, + ); if let Some(error) = window.get("alignment_validation_error") { ensure!( error.as_str().is_some_and(|e| !e.is_empty()) && rebuilt.is_err(), @@ -347,10 +370,10 @@ fn main() -> Result<()> { report["quality_score_eligible"] = json!(excluded.is_empty()); report["scope"] = json!("Diagnostic ASR segment text retrieval; all five sources compete, original filename titles, exact cosine, max across token windows, no labels in ranking. Source millisecond segment approximation, not word alignment or production integration."); if alignment::enabled(&asr)? { - report["scope"] = json!("Diagnostic ordinary token-time sentence retrieval; all five sources compete, original exact recognized text, independently reconstructed approximate token intervals, no labels in ranking; not production integration."); - report["alignment_policy"] = json!(alignment::POLICY); - report["alignment_policy_sha256"] = json!(alignment::POLICY_SHA); - report["alignment_contract_sha256"] = json!(alignment::CONTRACT_SHA); + report["scope"] = json!("Diagnostic token-time sentence retrieval; all five sources compete, original exact recognized text, independently reconstructed approximate ordinary intervals or DTW point spans as declared in alignment_policy, no labels in ranking; not production integration."); + report["alignment_policy"] = asr["alignment_policy"].clone(); + report["alignment_policy_sha256"] = asr["alignment_policy_sha256"].clone(); + report["alignment_contract_sha256"] = asr["alignment_contract_sha256"].clone(); } report["phase"] = json!("document_encoding"); checkpoint(&output, &report)?;