From 36bd51000b0acd6433b5586b2d1f89efd994c41f Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 21:38:31 +0200 Subject: [PATCH] Package verified speech runtime in Linux headless archives --- docs/LINUX-SPEECH-0.1.6.md | 21 +++++++++++ docs/ROADMAP.md | 3 ++ tools/package_linux.py | 29 ++++++++++++--- tools/test_package_linux.py | 74 +++++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 6 deletions(-) diff --git a/docs/LINUX-SPEECH-0.1.6.md b/docs/LINUX-SPEECH-0.1.6.md index 3c43512..42369f2 100644 --- a/docs/LINUX-SPEECH-0.1.6.md +++ b/docs/LINUX-SPEECH-0.1.6.md @@ -70,3 +70,24 @@ Exact mono 16-kHz signed PCM16 samples were converted to float32 by division by 32768, preserving all 360192 samples. PCM SHA-256: `56948226ac5a0ad4b70523a856dad0851929c7067957f339e9a5b30fc9b369ce`. Fixture preparation succeeded; native recognition has not yet been executed. + +## Headless release preparation + +The locked offline Linux release build of `searchi-cli` and +`searchi-extract-worker` from source `61afa08` passed in 58.38 seconds. +The extraction helper was staged with its current source fingerprint and +verified. CLI executable: 21921224 bytes, SHA-256 +`f7602d41eeab80c74e00014ab51297223eae651755707b341951c2e624807d94`. +This is compilation and provenance evidence, not a new published archive. + +`tools/package_linux.py --speech-runtime PATH` now includes a verified normal +speech runtime in its isolated `speech-runtime/` directory. The worker is +executable; inventory, libraries and approved notices are included. Archive +instructions explain speech-model setup and indexing, and metadata identifies +speech as experimental. Omitting this parameter preserves reproduction of older +speech-free packages. New speech-enabled releases must provide it. + +Twenty-one focused package tests passed, including archive hashes/permissions, +instructions, and rejection of internal CPU artifacts, tampering, absent notices +and invalid ELF closure while preserving an existing output. These use fixture +artifacts and do not replace the pending genuine native-runtime verification. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 641b7db..c5531f5 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -49,6 +49,9 @@ Google EmbeddingGemma 2 remains the main text/image/audio/video search model. - [x] Current Rust speech client unit and actual subprocess lifecycle checks passed on Ubuntu 24.04 WSL; headless client dependency compilation passed. [Evidence](LINUX-SPEECH-0.1.6.md); native Linux ASR and updated packages remain open. + - [x] Current Linux headless release compilation and extraction provenance; + add verified speech-runtime archive support with setup instructions and + 21 packaging tests. Native runtime and public Linux speech delivery remain open. - [x] Genuine shared-service ASR/Google lifecycle passed: verified speech excerpts, zero raw reencodes through policy failure/retry, and retained query reader; [proof](SPEECH-INTEGRATION.md). diff --git a/tools/package_linux.py b/tools/package_linux.py index c1cbf60..bcff054 100644 --- a/tools/package_linux.py +++ b/tools/package_linux.py @@ -18,6 +18,7 @@ import tarfile import tempfile from embeddinggemma2.runtime_provenance import verify as verify_provenance from stage_extraction_worker import verify as verify_extraction +from stage_speech_worker import verify_directory as verify_speech ROOT = Path(__file__).resolve().parents[1] MAX_FILE = 256 * 1024 * 1024 @@ -173,7 +174,7 @@ def media_files(root, media): return selected -def package(root, executable, runtime, media, output, llama_license=None, compression="gzip"): +def package(root, executable, runtime, media, output, llama_license=None, compression="gzip", speech_runtime=None): if compression not in {"gzip", "xz"}: raise ValueError("Unsupported package compression") root, executable, runtime, media, output = map(Path, (root, executable, runtime, media, output)) @@ -195,6 +196,11 @@ def package(root, executable, runtime, media, output, llama_license=None, compre or set(cli_metadata.get("NEEDED", [])) - SYSTEM): raise ValueError("CLI has unexpected native dependency or search path") selected = {name: (runtime / name, record) for name, record in expected.items()} + if speech_runtime is not None: + speech_runtime = Path(speech_runtime) + speech_records = verify_speech(speech_runtime, root, 'linux', 'x86_64') + selected.update({name: (speech_runtime / PurePosixPath(name).name, record) + for name, record in speech_records.items()}) selected.update(media_files(root, media)) selected["searchi"] = (executable, {"sha256": digest(executable), "bytes": executable.stat().st_size}) license_path = Path(llama_license) if llama_license else root / ".searchi/references/llama.cpp/LICENSE" @@ -225,9 +231,15 @@ def package(root, executable, runtime, media, output, llama_license=None, compre checksum = hashlib.sha256(data).hexdigest() if len(data) != expected_record["bytes"] or checksum != expected_record["sha256"]: raise ValueError(f"Source changed during packaging: {name}") - mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "searchi-extract-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644 + mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "searchi-extract-worker", "speech-runtime/searchi-speech-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644 add(archive, name, data, mode) records.append({"path": name, "bytes": len(data), "sha256": checksum, "mode": oct(mode)}) + speech_instructions = ("\nExperimental local speech search is included. Download verified recognition assets (about 149 MB):\n" + "./searchi setup --speech --model-dir ./speech-models\n" + "Then index recognized passages alongside Google media embeddings:\n" + "./searchi embed --db ./searchi.sqlite --model-dir ./models --embedding-worker ./searchi-gemma-worker --speech-worker ./speech-runtime/searchi-speech-worker --speech-model-dir ./speech-models\n" + "Search uses the same command above; spoken passages carry estimated source timestamps.\n" + "Recognition and search remain local. Music/no-answer relevance and precise timestamps remain experimental.\n") if speech_runtime is not None else "" instructions = ("searchi — Find what matters. Locally.\n\n" "Extract the entire archive into its own folder. This is a headless development candidate, not a production installer.\n" "Use ./searchi --help. Setup downloads about 865 MB of verified Google model assets:\n" @@ -237,14 +249,18 @@ def package(root, executable, runtime, media, output, llama_license=None, compre "./searchi search --db ./searchi.sqlite 'what you remember' --model-dir ./models --embedding-worker ./searchi-gemma-worker\n" "Media helpers are discovered automatically beside this CLI. Ordinary search stays local.\n" "The normal runtime includes Vulkan and CPU fallback. Graphics drivers and a compatible system Vulkan loader are optional.\n" - "Requires a compatible Linux libc/C++ ABI; clean-machine and distribution coverage remain open.\n").encode() + "Requires a compatible Linux libc/C++ ABI; clean-machine and distribution coverage remain open.\n" + speech_instructions).encode() add(archive, "START-HERE.txt", instructions, 0o644) records.append({"path": "START-HERE.txt", "bytes": len(instructions), "sha256": hashlib.sha256(instructions).hexdigest(), "mode": oct(0o644)}) - add(archive, "PACKAGE.json", json.dumps({"platform": "linux", "architecture": "x86_64", + metadata = {"platform": "linux", "architecture": "x86_64", "status": "unsigned headless development candidate; distribution ABI and GPU hardware validation pending", "system_libraries": sorted(SYSTEM), "optional_gpu_loader": "libvulkan.so.1", - "files": records}, indent=2).encode(), 0o644) + "files": records} + if speech_runtime is not None: + metadata['features'] = {'local_speech': True, 'speech_status': 'experimental', + 'speech_runtime': 'speech-runtime/searchi-speech-worker'} + add(archive, "PACKAGE.json", json.dumps(metadata, indent=2).encode(), 0o644) os.replace(temporary_path, output) finally: temporary_path.unlink(missing_ok=True) @@ -264,13 +280,14 @@ def main(): parser.add_argument("--executable", type=Path, default=ROOT / "target/release/searchi") parser.add_argument("--runtime", type=Path, default=ROOT / ".searchi/runtime/auto") parser.add_argument("--media", type=Path, default=ROOT / ".searchi/media-runtime") + parser.add_argument("--speech-runtime", type=Path, help="Verified normal auto speech runtime directory") parser.add_argument("--llama-license", type=Path) parser.add_argument("--compression", choices=["gzip", "xz"], default="gzip") parser.add_argument("--output", type=Path) args = parser.parse_args() if args.output is None: args.output = ROOT / (".searchi/packages/searchi-linux-x86_64.tar.xz" if args.compression == "xz" else ".searchi/packages/searchi-linux-x86_64.tar.gz") - package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license, args.compression) + package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license, args.compression, args.speech_runtime) print(f"Packaged Linux x86_64 development candidate: {args.output}") diff --git a/tools/test_package_linux.py b/tools/test_package_linux.py index 7d06152..789a1dd 100644 --- a/tools/test_package_linux.py +++ b/tools/test_package_linux.py @@ -11,6 +11,7 @@ from unittest import mock import package_linux as pack from embeddinggemma2 import runtime_provenance as provenance import stage_extraction_worker as extraction +import stage_speech_worker as speech def elf(path, machine=62): @@ -22,6 +23,79 @@ def elf(path, machine=62): class LinuxPackageTests(unittest.TestCase): + def speech_fixture(self, fixture): + root = fixture[0] + runtime = root / 'speech-runtime' + runtime.mkdir() + for name in speech.INPUTS: + path = speech.input_path(root, name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(speech.input_path(speech.ROOT, name).read_bytes()) + files = [] + for name in ('searchi-speech-worker', 'libwhisper.so.1', 'libggml-cpu-x64.so', 'libggml-vulkan.so'): + path = runtime / name + elf(path) + files.append({'path': name, 'bytes': path.stat().st_size, 'sha256': pack.digest(path)}) + (runtime / 'runtime-manifest.json').write_text(json.dumps({'schema': 1, + 'upstream_commit': speech.UPSTREAM, 'device': 'auto', 'distribution': True, + 'source_hashes': {name: pack.digest(speech.input_path(root, name)) for name in speech.INPUTS}, + 'files': files})) + for name in speech.NOTICES: + notice = root / 'assets/licenses' / name + notice.parent.mkdir(parents=True, exist_ok=True) + notice.write_text('Approved test notice ' + name) + (runtime / name).write_bytes(notice.read_bytes()) + return runtime + + def test_speech_archive_has_isolated_verified_files_modes_and_instructions(self): + with tempfile.TemporaryDirectory() as directory: + fixture = self.fixture(directory) + runtime = self.speech_fixture(fixture) + with mock.patch.object(pack, 'dynamic', side_effect=self.metadata), \ + mock.patch.object(speech, 'verify_linux_closure') as closure: + pack.package(*fixture, speech_runtime=runtime) + closure.assert_called_once_with(runtime) + first = fixture[-1].read_bytes() + pack.package(*fixture, speech_runtime=runtime) + self.assertEqual(first, fixture[-1].read_bytes()) + with tarfile.open(fixture[-1]) as archive: + manifest = json.load(archive.extractfile('PACKAGE.json')) + self.assertTrue(manifest['features']['local_speech']) + self.assertEqual(manifest['features']['speech_status'], 'experimental') + self.assertEqual(archive.getmember('speech-runtime/searchi-speech-worker').mode, 0o755) + for name in (*speech.NOTICES, 'runtime-manifest.json'): + self.assertEqual(archive.getmember('speech-runtime/' + name).mode, 0o644) + for record in manifest['files']: + data = archive.extractfile(record['path']).read() + self.assertEqual(record['sha256'], hashlib.sha256(data).hexdigest()) + instructions = archive.extractfile('START-HERE.txt').read().decode() + self.assertIn('setup --speech --model-dir ./speech-models', instructions) + self.assertIn('--speech-worker ./speech-runtime/searchi-speech-worker', instructions) + self.assertIn('--speech-model-dir ./speech-models', instructions) + + def test_invalid_speech_runtime_preserves_previous_archive(self): + for mutation in ('cpu', 'tampered', 'notice', 'closure'): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + fixture = self.fixture(directory) + runtime = self.speech_fixture(fixture) + fixture[-1].write_bytes(b'previous archive') + if mutation == 'cpu': + path = runtime / 'runtime-manifest.json' + manifest = json.loads(path.read_text()) + manifest.update(device='cpu', distribution=False) + path.write_text(json.dumps(manifest)) + elif mutation == 'tampered': + (runtime / 'libwhisper.so.1').write_bytes(b'tampered') + elif mutation == 'notice': + (runtime / speech.NOTICES[0]).unlink() + with mock.patch.object(pack, 'dynamic', side_effect=self.metadata), \ + mock.patch.object(speech, 'verify_linux_closure', + side_effect=ValueError('Invalid ELF closure') if mutation == 'closure' else None): + with self.assertRaises(ValueError): + pack.package(*fixture, speech_runtime=runtime) + self.assertEqual(fixture[-1].read_bytes(), b'previous archive') + self.assertEqual(list(fixture[-1].parent.glob('*.partial')), []) + def test_extraction_missing_stale_tampered_or_wrong_architecture_preserves_output(self): for mutation in ("missing", "inventory_missing", "stale", "tampered", "architecture"): with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: