"""Frozen DTW sentence point spans, independent of ordinary token intervals.""" from token_alignment import BOUNDARY, TEXT_TOKEN_LIMIT POLICY = "dtw-token-sentences-v1;centisecond-points;first-last-point-span;exact-text;no-labels" def aligned_segments(document, offset_ms, duration_ms): from run import segments originals = segments(document, offset_ms, duration_ms) raw_rows = [row for row in document["transcription"] if row["text"].strip()] result = [] previous = 0 for index, (row, original) in enumerate(zip(raw_rows, originals)): tokens = row.get("tokens") if not isinstance(tokens, list) or not 0 < len(tokens) <= 4096: raise ValueError("Missing or oversized DTW tokens") accepted = [] for token in tokens: token_id, text = token.get("id"), token.get("text") if type(token_id) is not int or token_id < 0 or not isinstance(text, str): raise ValueError("Invalid DTW token identity") if token_id >= TEXT_TOKEN_LIMIT: if not text.startswith("[_") or not text.endswith("]"): raise ValueError("Unrecognized special token") continue point = token.get("t_dtw") if type(point) is not int or not 0 <= point * 10 <= duration_ms or point * 10 < previous: raise ValueError("Missing, invalid or nonmonotonic DTW point") previous = point * 10 accepted.append((text, previous)) if "".join(token[0] for token in accepted) != row["text"]: raise ValueError("DTW text differs from original recognized segment") phrase = [] for token_index, token in enumerate(accepted): phrase.append(token) if BOUNDARY.search(token[0]) or token_index == len(accepted) - 1: text = "".join(part[0] for part in phrase) if text.strip(): result.append({"start_ms": offset_ms + phrase[0][1], "end_ms": offset_ms + phrase[-1][1], "text": text, "original_segment_index": index, "original_start_ms": original["start_ms"], "original_end_ms": original["end_ms"]}) phrase = [] if len(result) > 10000: raise ValueError("Oversized DTW passages") return result