Prepare bounded speech diagnostics and document mixed routing risks

tools/speech-eval/README.md

@@ -0,0 +1,51 @@
+# Actual local speech diagnostic
+
+Research harness only; transcription is not implemented in searchi yet.
+See [the integration proposal](../../docs/SPEECH-ALIGNMENT.md). No actual results
+are checked in until genuine runs complete. Google remains the semantic model.
+
+`run.py` requires separately provisioned, pinned whisper.cpp v1.9.5 CLI, multilingual
+base/small model and existing verified FFmpeg/FFprobe. It never builds/downloads.
+Record the CLI build SHA beforehand and pass that hash; a matching hash alone does
+not establish upstream provenance or dependency closure. Verify the build recipe
+and dynamic libraries separately. CPU slot coordination is required before runs.
+
+```powershell
+$env:SEARCHI_FORCE_CPU = '1'
+python tools/speech-eval/run.py --whisper PATH_TO_PINNED_CLI --whisper-sha256 RECORDED_BUILD_SHA --ffmpeg PATH_TO_VERIFIED_FFMPEG --ffprobe PATH_TO_VERIFIED_FFPROBE --model PATH_TO_VERIFIED_BASE --candidate base --scratch-parent .searchi/speech-scratch --output .searchi/qa/speech-base.json
+python -m unittest discover -s tools/speech-eval -p 'test_*.py'
+```
+
+The CLI receives pinned-source-confirmed `--no-gpu`, one thread and one processor.
+Source and model SHA/size are checked; NASA, both real instrumental music assets
+and both non-speech effects come from unchanged media-v1. Each source is limited
+to ten minutes for this proof, decoded into 30-second 16kHz mono signed-16 WAV
+windows with bounded owned scratch. CLI processes restart per window: cost includes
+model load, not retained-session throughput. This deliberately simple experiment
+does not implement overlap/dedup or production budgets. Segment JSON offsets are
+already milliseconds and get the window offset added exactly once.
+
+Each completed window writes an incomplete checkpoint; the final report marks
+complete. Errors retain bounded owned logs outside temporary scratch, which cleans
+decoded WAV/JSON on exit. Cancellation terminates only owned subprocess trees.
+Limits are supervisory checks, not OS resource sandbox guarantees. Recognized text
+is private local diagnostic data; never commit raw logs/transcripts by default.
+Music/non-speech output requires actual audio review: nonempty recognized text is
+not automatically proof of hallucination and empty output is not universal safety.
+
+`score.py` consumes **externally ranked genuine Google transcript results**, using
+the existing eight frozen NASA intervals unchanged. Input has `fixture_sha256`,
+`asr_report_sha256`, `google_worker_sha256`, and `rankings` mapping query ids to
+ordered `{asset,start_ms,end_ms}` records. It reports top1 overlap and strict start
+point hits; it never chooses a segment by looking at the answer intervals.
+Hash fields must be checked against retained artifacts by the evaluator; they are
+provenance declarations, not a replacement for inspecting those artifacts.
+
+Unit tests use explicitly synthetic structural data to verify bounds/units and
+scoring behavior. They provide no speech-recognition or search quality evidence.
+The first proof must manually inspect NASA recognition against independently
+published captions, then run real Google ranking. Eight paired queries from one
+English recording do not validate German speech or general transcription quality.
+
+Pinned CLI flag/output reference:
+[upstream cli.cpp](https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/examples/cli/cli.cpp).

tools/speech-eval/run.py

@@ -0,0 +1,182 @@
+"""Isolated actual whisper.cpp CPU diagnostic; never downloads or builds tools."""
+import argparse
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import signal
+import subprocess
+import tempfile
+import time
+import wave
+
+UPSTREAM = "d1be6fde11ac6e0407606b4e42fe72d34add8037"
+MODELS = {
+    "base": (147951465, "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe"),
+    "small": (487601967, "1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b"),
+}
+ASSETS = ("clip04", "music01", "music02", "sound01", "sound02")
+FIXTURE_SHA = "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393"
+MAX_OUTPUT = 4 * 1024 * 1024
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for block in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def stop(child):
+    if os.name == "nt":
+        subprocess.run(["taskkill", "/PID", str(child.pid), "/T", "/F"],
+                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                       creationflags=subprocess.CREATE_NO_WINDOW, check=False)
+    else:
+        os.killpg(child.pid, signal.SIGKILL)
+    child.wait()
+
+
+def run_owned(command, scratch, name, outputs, timeout):
+    """Bound time/output; terminate only our process tree on failure/interruption."""
+    log = scratch / f"{name}.log"
+    kwargs = {"creationflags": subprocess.CREATE_NO_WINDOW} if os.name == "nt" else {"start_new_session": True}
+    started = time.perf_counter()
+    with log.open("wb") as stream:
+        child = subprocess.Popen(command, stdin=subprocess.DEVNULL,
+                                 stdout=stream, stderr=subprocess.STDOUT, **kwargs)
+        try:
+            while child.poll() is None:
+                if time.perf_counter() - started > timeout:
+                    raise TimeoutError(f"{name} exceeded {timeout}s")
+                if any(path.exists() and path.stat().st_size > limit for path, limit in [(log, MAX_OUTPUT), *outputs]):
+                    raise ValueError(f"{name} exceeded output budget")
+                time.sleep(0.05)
+            if child.returncode:
+                raise RuntimeError(f"{name} failed with exit {child.returncode}; owned diagnostic log: {log}")
+            if any(path.exists() and path.stat().st_size > limit for path, limit in [(log, MAX_OUTPUT), *outputs]):
+                raise ValueError(f"{name} exceeded output budget")
+        except BaseException as error:
+            if child.poll() is None:
+                stop(child)
+            stream.flush()
+            # Preserve bounded actual diagnostic bytes outside ephemeral scratch.
+            with tempfile.NamedTemporaryFile(prefix=f"speech-{name}-failure-", suffix=".log", dir=scratch.parent, delete=False) as retained:
+                with log.open("rb") as source:
+                    retained.write(source.read(MAX_OUTPUT))
+                print(f"Owned failure diagnostic retained: {retained.name}")
+                diagnostic = {"stage": name, "elapsed_ms": (time.perf_counter() - started) * 1000,
+                              "exit_code": child.returncode, "owned_pid": child.pid, "error_type": type(error).__name__,
+                              "bounded_log": retained.name, "complete": False}
+            Path(retained.name).with_suffix(".json").write_text(json.dumps(diagnostic, indent=2) + "\n", encoding="utf-8")
+            raise
+        finally:
+            if child.poll() is None:
+                stop(child)
+    return (time.perf_counter() - started) * 1000
+
+
+def segments(document, offset_ms, duration_ms):
+    raw = document.get("transcription")
+    if not isinstance(raw, list) or len(raw) > 1000:
+        raise ValueError("Missing or oversized transcription array")
+    parsed = []
+    previous = 0
+    for row in raw:
+        start, end = row["offsets"]["from"], row["offsets"]["to"]
+        text = row["text"]
+        if type(start) is not int or type(end) is not int or not 0 <= start <= end <= duration_ms or start < previous:
+            raise ValueError("Invalid local ASR timestamps")
+        if not isinstance(text, str) or len(text.encode("utf-8")) > 4096:
+            raise ValueError("Invalid recognized text")
+        previous = start
+        if text.strip():
+            parsed.append({"start_ms": offset_ms + start, "end_ms": offset_ms + end, "text": text})
+    return parsed
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--whisper", type=Path, required=True)
+    parser.add_argument("--whisper-sha256", required=True, help="Previously recorded pinned build SHA, not computed for acceptance here")
+    parser.add_argument("--ffmpeg", type=Path, required=True)
+    parser.add_argument("--ffprobe", type=Path, required=True)
+    parser.add_argument("--model", type=Path, required=True)
+    parser.add_argument("--candidate", choices=MODELS, required=True)
+    parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json"))
+    parser.add_argument("--scratch-parent", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    size, digest = MODELS[args.candidate]
+    if args.model.stat().st_size != size or sha256(args.model) != digest:
+        raise ValueError("Model size/SHA verification failed")
+    if sha256(args.whisper) != args.whisper_sha256:
+        raise ValueError("Pinned worker build SHA verification failed")
+    if sha256(args.fixture) != FIXTURE_SHA:
+        raise ValueError("Frozen media-v1 fixture SHA mismatch")
+    fixture = json.loads(args.fixture.read_text(encoding="utf-8"))
+    if not set(ASSETS).issubset({asset["id"] for asset in fixture["assets"]}):
+        raise ValueError("Missing proof assets")
+    rows = []
+    args.scratch_parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="speech-proof-", dir=args.scratch_parent) as directory:
+        scratch = Path(directory)
+        for asset in fixture["assets"]:
+            if asset["id"] not in ASSETS:
+                continue
+            source = args.fixture.parent / asset["path"]
+            if source.stat().st_size != asset["bytes"] or sha256(source) != asset["sha256"]:
+                raise ValueError(f"Frozen asset mismatch: {asset['id']}")
+            # Probe stdout is bounded by the same owned log, then parsed directly.
+            run_owned([str(args.ffprobe.resolve()), "-v", "error", "-protocol_whitelist", "file,pipe",
+                       "-show_entries", "format=duration", "-of", "json", str(source.resolve())],
+                      scratch, "probe", [], 30)
+            probe = json.loads((scratch / "probe.log").read_text(encoding="utf-8"))
+            duration_s = float(probe["format"]["duration"])
+            if not math.isfinite(duration_s) or not 0 < duration_s <= 600:
+                raise ValueError("Proof duration exceeds ten-minute bound")
+            duration_ms = math.ceil(duration_s * 1000)
+            result = {"asset": asset["id"], "source_sha256": asset["sha256"], "duration_ms": duration_ms, "windows": []}
+            for offset in range(0, duration_ms, 30000):
+                length = min(30000, duration_ms - offset)
+                wav = scratch / "window.wav"
+                prefix = scratch / "recognized"
+                output = prefix.with_suffix(".json")
+                output.unlink(missing_ok=True)
+                decode_ms = run_owned([str(args.ffmpeg.resolve()), "-nostdin", "-hide_banner", "-v", "error",
+                    "-protocol_whitelist", "file,pipe", "-threads", "1", "-ss", str(offset / 1000), "-t", str(length / 1000),
+                    "-i", str(source.resolve()), "-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
+                    "-af", f"atrim=end_sample={length * 16}", "-c:a", "pcm_s16le", "-threads", "1", "-y", str(wav)],
+                    scratch, "decode", [(wav, 970000)], 60)
+                with wave.open(str(wav), "rb") as audio:
+                    if audio.getframerate() != 16000 or audio.getnchannels() != 1 or audio.getsampwidth() != 2:
+                        raise ValueError("Invalid decoded PCM")
+                asr_ms = run_owned([str(args.whisper.resolve()), "--no-gpu", "--threads", "1", "--processors", "1",
+                    "--language", "auto", "--model", str(args.model.resolve()), "--file", str(wav),
+                    "--output-json", "--output-file", str(prefix)], scratch, "asr", [(output, MAX_OUTPUT)], 600)
+                raw = json.loads(output.read_text(encoding="utf-8"))
+                result["windows"].append({"offset_ms": offset, "duration_ms": length, "decode_ms": decode_ms,
+                    "asr_process_ms": asr_ms, "segments": segments(raw, offset, length), "actual_whisper_json": raw})
+                args.output.parent.mkdir(parents=True, exist_ok=True)
+                checkpoint = {"complete": False, "fixture_sha256": sha256(args.fixture),
+                              "model_sha256": digest, "worker_sha256": args.whisper_sha256,
+                              "rows": [*rows, result]}
+                args.output.write_text(json.dumps(checkpoint, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+            if source.stat().st_size != asset["bytes"] or sha256(source) != asset["sha256"]:
+                raise ValueError("Source changed during recognition")
+            rows.append(result)
+    report = {"version": 1, "complete": True, "scope": "Actual ASR diagnostic, not Google retrieval or production integration",
+              "fixture_sha256": sha256(args.fixture), "upstream_commit": UPSTREAM,
+              "worker_sha256": args.whisper_sha256, "model_sha256": digest, "candidate": args.candidate,
+              "ffmpeg_sha256": sha256(args.ffmpeg), "ffprobe_sha256": sha256(args.ffprobe),
+              "backend": "forced CPU", "threads": 1,
+              "policy": "nonoverlapping30s;independent-process-per-window;auto-language;segment-times",
+              "rows": rows}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()

tools/speech-eval/score.py

@@ -0,0 +1,53 @@
+"""Score externally ranked genuine transcript hits against unchanged media v1."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+from run import FIXTURE_SHA
+
+
+def score(fixture, rankings):
+    queries = [query for query in fixture["queries"] if query["category"] == "video-speech"]
+    if len(queries) != 8:
+        raise ValueError("Expected the eight frozen speech queries")
+    ids = {query["id"] for query in queries}
+    if set(rankings) != ids:
+        raise ValueError("Rankings must cover exactly eight frozen speech queries")
+    rows = []
+    for query in queries:
+        hits = rankings[query["id"]]
+        if not isinstance(hits, list) or len(hits) > 100:
+            raise ValueError("Invalid ranking length")
+        hit = hits[0] if hits else None
+        point = overlap = False
+        if hit:
+            start, end = hit["start_ms"], hit["end_ms"]
+            if type(start) is not int or type(end) is not int or not 0 <= start <= end:
+                raise ValueError("Invalid ranked timestamp")
+            intervals = query["intervals"].get(hit["asset"], [])
+            point = any(left <= start / 1000 <= right for left, right in intervals)
+            overlap = any(min(end / 1000, right) > max(start / 1000, left) for left, right in intervals)
+        rows.append({"query": query["id"], "point_hit": point, "overlap_hit": overlap})
+    return {"scope": "Externally ranked transcript top1, no oracle segment selection",
+            "queries": len(rows), "point_hits": sum(row["point_hit"] for row in rows),
+            "overlap_hits": sum(row["overlap_hit"] for row in rows), "rows": rows}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--fixture", type=Path, default=Path("fixtures/media/fixture.json"))
+    parser.add_argument("--rankings", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    data = args.fixture.read_bytes()
+    rankings = json.loads(args.rankings.read_text(encoding="utf-8"))
+    digest = hashlib.sha256(data).hexdigest()
+    if digest != FIXTURE_SHA or rankings["fixture_sha256"] != digest or not rankings.get("asr_report_sha256") or not rankings.get("google_worker_sha256"):
+        raise ValueError("Missing/mismatched actual ASR/Google provenance")
+    report = score(json.loads(data), rankings["rankings"])
+    report.update({key: rankings[key] for key in ("fixture_sha256", "asr_report_sha256", "google_worker_sha256")})
+    args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()

tools/speech-eval/test_contract.py

@@ -0,0 +1,72 @@
+"""Structural synthetic protocol tests; these are not ASR/retrieval proof."""
+import json
+from pathlib import Path
+import unittest
+import tempfile
+import sys
+from run import segments, run_owned
+from score import score
+
+
+class ContractTests(unittest.TestCase):
+    def test_timeout_and_output_overflow_terminate_owned_child(self):
+        for name, code, error, timeout in [
+            ("timeout", "import time; time.sleep(10)", TimeoutError, 0.1),
+            ("overflow", "import sys,time; sys.stdout.write('x'*5000000); sys.stdout.flush(); time.sleep(10)", ValueError, 5),
+        ]:
+            with self.subTest(name=name), tempfile.TemporaryDirectory() as parent:
+                checkpoint = Path(parent) / "checkpoint.json"
+                checkpoint.write_text('{"complete":false,"rows":[1]}')
+                with tempfile.TemporaryDirectory(dir=parent) as scratch:
+                    with self.assertRaises(error):
+                        run_owned([sys.executable, "-c", code], Path(scratch), name, [], timeout)
+                diagnostics = list(Path(parent).glob(f"speech-{name}-failure-*.json"))
+                self.assertEqual(len(diagnostics), 1)
+                report = json.loads(diagnostics[0].read_text())
+                self.assertIsNotNone(report["exit_code"])
+                self.assertEqual(report["error_type"], error.__name__)
+                self.assertLessEqual(Path(report["bounded_log"]).stat().st_size, 4 * 1024 * 1024)
+                self.assertEqual(json.loads(checkpoint.read_text())["rows"], [1])
+
+    def test_failure_diagnostic_survives_scratch_cleanup(self):
+        with tempfile.TemporaryDirectory() as parent:
+            with tempfile.TemporaryDirectory(dir=parent) as scratch:
+                with self.assertRaises(RuntimeError):
+                    run_owned([sys.executable, "-c", "print('structural failure'); raise SystemExit(3)"], Path(scratch), "contract", [], 5)
+            diagnostics = list(Path(parent).glob("speech-contract-failure-*.json"))
+            self.assertEqual(len(diagnostics), 1)
+            report = json.loads(diagnostics[0].read_text())
+            self.assertEqual(report["exit_code"], 3)
+            self.assertGreater(report["elapsed_ms"], 0)
+            self.assertIn("structural failure", Path(report["bounded_log"]).read_text())
+
+    def test_offsets_applied_once(self):
+        actual = segments({"transcription": [{"offsets": {"from": 1500, "to": 2400}, "text": "test"}]}, 30000, 30000)
+        self.assertEqual(actual[0]["start_ms"], 31500)
+        self.assertEqual(actual[0]["end_ms"], 32400)
+
+    def test_reject_invalid_times_and_text(self):
+        for start, end, text in [(False, 2, "x"), (-1, 2, "x"), (2, 1, "x"), (0, 30001, "x"), (0, 1, "x" * 4097)]:
+            with self.assertRaises(ValueError):
+                segments({"transcription": [{"offsets": {"from": start, "to": end}, "text": text}]}, 0, 30000)
+
+    def test_reject_nonmonotonic_times(self):
+        with self.assertRaises(ValueError):
+            segments({"transcription": [{"offsets": {"from": start, "to": start + 1}, "text": "x"} for start in [10, 2]]}, 0, 30000)
+
+    def test_missing_rankings_fail(self):
+        fixture = json.loads((Path(__file__).resolve().parents[2] / "fixtures/media/fixture.json").read_text())
+        with self.assertRaises(ValueError):
+            score(fixture, {})
+
+    def test_point_is_distinct_from_overlap(self):
+        fixture = json.loads((Path(__file__).resolve().parents[2] / "fixtures/media/fixture.json").read_text())
+        queries = [query for query in fixture["queries"] if query["category"] == "video-speech"]
+        rankings = {query["id"]: [{"asset": "clip04", "start_ms": 0, "end_ms": 100000}] for query in queries}
+        report = score(fixture, rankings)
+        self.assertEqual(report["point_hits"], 0)
+        self.assertEqual(report["overlap_hits"], 8)
+
+
+if __name__ == "__main__":
+    unittest.main()