Package verified speech runtime in Linux headless archives

docs/LINUX-SPEECH-0.1.6.md

@@ -70,3 +70,24 @@ Exact mono 16-kHz signed PCM16 samples were converted to float32 by division by
 32768, preserving all 360192 samples. PCM SHA-256:
 `56948226ac5a0ad4b70523a856dad0851929c7067957f339e9a5b30fc9b369ce`.
 Fixture preparation succeeded; native recognition has not yet been executed.
+
+## Headless release preparation
+
+The locked offline Linux release build of `searchi-cli` and
+`searchi-extract-worker` from source `61afa08` passed in 58.38 seconds.
+The extraction helper was staged with its current source fingerprint and
+verified. CLI executable: 21921224 bytes, SHA-256
+`f7602d41eeab80c74e00014ab51297223eae651755707b341951c2e624807d94`.
+This is compilation and provenance evidence, not a new published archive.
+
+`tools/package_linux.py --speech-runtime PATH` now includes a verified normal
+speech runtime in its isolated `speech-runtime/` directory. The worker is
+executable; inventory, libraries and approved notices are included. Archive
+instructions explain speech-model setup and indexing, and metadata identifies
+speech as experimental. Omitting this parameter preserves reproduction of older
+speech-free packages. New speech-enabled releases must provide it.
+
+Twenty-one focused package tests passed, including archive hashes/permissions,
+instructions, and rejection of internal CPU artifacts, tampering, absent notices
+and invalid ELF closure while preserving an existing output. These use fixture
+artifacts and do not replace the pending genuine native-runtime verification.

docs/ROADMAP.md

@@ -49,6 +49,9 @@ Google EmbeddingGemma 2 remains the main text/image/audio/video search model.
   - [x] Current Rust speech client unit and actual subprocess lifecycle checks
     passed on Ubuntu 24.04 WSL; headless client dependency compilation passed.
     [Evidence](LINUX-SPEECH-0.1.6.md); native Linux ASR and updated packages remain open.
+  - [x] Current Linux headless release compilation and extraction provenance;
+    add verified speech-runtime archive support with setup instructions and
+    21 packaging tests. Native runtime and public Linux speech delivery remain open.
   - [x] Genuine shared-service ASR/Google lifecycle passed: verified speech
     excerpts, zero raw reencodes through policy failure/retry, and retained
     query reader; [proof](SPEECH-INTEGRATION.md).

tools/package_linux.py

@@ -18,6 +18,7 @@ import tarfile
 import tempfile
 from embeddinggemma2.runtime_provenance import verify as verify_provenance
 from stage_extraction_worker import verify as verify_extraction
+from stage_speech_worker import verify_directory as verify_speech
 
 ROOT = Path(__file__).resolve().parents[1]
 MAX_FILE = 256 * 1024 * 1024
@@ -173,7 +174,7 @@ def media_files(root, media):
     return selected
 
 
-def package(root, executable, runtime, media, output, llama_license=None, compression="gzip"):
+def package(root, executable, runtime, media, output, llama_license=None, compression="gzip", speech_runtime=None):
     if compression not in {"gzip", "xz"}:
         raise ValueError("Unsupported package compression")
     root, executable, runtime, media, output = map(Path, (root, executable, runtime, media, output))
@@ -195,6 +196,11 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
             or set(cli_metadata.get("NEEDED", [])) - SYSTEM):
         raise ValueError("CLI has unexpected native dependency or search path")
     selected = {name: (runtime / name, record) for name, record in expected.items()}
+    if speech_runtime is not None:
+        speech_runtime = Path(speech_runtime)
+        speech_records = verify_speech(speech_runtime, root, 'linux', 'x86_64')
+        selected.update({name: (speech_runtime / PurePosixPath(name).name, record)
+                         for name, record in speech_records.items()})
     selected.update(media_files(root, media))
     selected["searchi"] = (executable, {"sha256": digest(executable), "bytes": executable.stat().st_size})
     license_path = Path(llama_license) if llama_license else root / ".searchi/references/llama.cpp/LICENSE"
@@ -225,9 +231,15 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
                 checksum = hashlib.sha256(data).hexdigest()
                 if len(data) != expected_record["bytes"] or checksum != expected_record["sha256"]:
                     raise ValueError(f"Source changed during packaging: {name}")
-                mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "searchi-extract-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644
+                mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "searchi-extract-worker", "speech-runtime/searchi-speech-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644
                 add(archive, name, data, mode)
                 records.append({"path": name, "bytes": len(data), "sha256": checksum, "mode": oct(mode)})
+            speech_instructions = ("\nExperimental local speech search is included. Download verified recognition assets (about 149 MB):\n"
+                "./searchi setup --speech --model-dir ./speech-models\n"
+                "Then index recognized passages alongside Google media embeddings:\n"
+                "./searchi embed --db ./searchi.sqlite --model-dir ./models --embedding-worker ./searchi-gemma-worker --speech-worker ./speech-runtime/searchi-speech-worker --speech-model-dir ./speech-models\n"
+                "Search uses the same command above; spoken passages carry estimated source timestamps.\n"
+                "Recognition and search remain local. Music/no-answer relevance and precise timestamps remain experimental.\n") if speech_runtime is not None else ""
             instructions = ("searchi — Find what matters. Locally.\n\n"
                 "Extract the entire archive into its own folder. This is a headless development candidate, not a production installer.\n"
                 "Use ./searchi --help. Setup downloads about 865 MB of verified Google model assets:\n"
@@ -237,14 +249,18 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
                 "./searchi search --db ./searchi.sqlite 'what you remember' --model-dir ./models --embedding-worker ./searchi-gemma-worker\n"
                 "Media helpers are discovered automatically beside this CLI. Ordinary search stays local.\n"
                 "The normal runtime includes Vulkan and CPU fallback. Graphics drivers and a compatible system Vulkan loader are optional.\n"
-                "Requires a compatible Linux libc/C++ ABI; clean-machine and distribution coverage remain open.\n").encode()
+                "Requires a compatible Linux libc/C++ ABI; clean-machine and distribution coverage remain open.\n" + speech_instructions).encode()
             add(archive, "START-HERE.txt", instructions, 0o644)
             records.append({"path": "START-HERE.txt", "bytes": len(instructions),
                 "sha256": hashlib.sha256(instructions).hexdigest(), "mode": oct(0o644)})
-            add(archive, "PACKAGE.json", json.dumps({"platform": "linux", "architecture": "x86_64",
+            metadata = {"platform": "linux", "architecture": "x86_64",
                 "status": "unsigned headless development candidate; distribution ABI and GPU hardware validation pending",
                 "system_libraries": sorted(SYSTEM), "optional_gpu_loader": "libvulkan.so.1",
-                "files": records}, indent=2).encode(), 0o644)
+                "files": records}
+            if speech_runtime is not None:
+                metadata['features'] = {'local_speech': True, 'speech_status': 'experimental',
+                                        'speech_runtime': 'speech-runtime/searchi-speech-worker'}
+            add(archive, "PACKAGE.json", json.dumps(metadata, indent=2).encode(), 0o644)
         os.replace(temporary_path, output)
     finally:
         temporary_path.unlink(missing_ok=True)
@@ -264,13 +280,14 @@ def main():
     parser.add_argument("--executable", type=Path, default=ROOT / "target/release/searchi")
     parser.add_argument("--runtime", type=Path, default=ROOT / ".searchi/runtime/auto")
     parser.add_argument("--media", type=Path, default=ROOT / ".searchi/media-runtime")
+    parser.add_argument("--speech-runtime", type=Path, help="Verified normal auto speech runtime directory")
     parser.add_argument("--llama-license", type=Path)
     parser.add_argument("--compression", choices=["gzip", "xz"], default="gzip")
     parser.add_argument("--output", type=Path)
     args = parser.parse_args()
     if args.output is None:
         args.output = ROOT / (".searchi/packages/searchi-linux-x86_64.tar.xz" if args.compression == "xz" else ".searchi/packages/searchi-linux-x86_64.tar.gz")
-    package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license, args.compression)
+    package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license, args.compression, args.speech_runtime)
     print(f"Packaged Linux x86_64 development candidate: {args.output}")
 
 

tools/test_package_linux.py

@@ -11,6 +11,7 @@ from unittest import mock
 import package_linux as pack
 from embeddinggemma2 import runtime_provenance as provenance
 import stage_extraction_worker as extraction
+import stage_speech_worker as speech
 
 
 def elf(path, machine=62):
@@ -22,6 +23,79 @@ def elf(path, machine=62):
 
 
 class LinuxPackageTests(unittest.TestCase):
+    def speech_fixture(self, fixture):
+        root = fixture[0]
+        runtime = root / 'speech-runtime'
+        runtime.mkdir()
+        for name in speech.INPUTS:
+            path = speech.input_path(root, name)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(speech.input_path(speech.ROOT, name).read_bytes())
+        files = []
+        for name in ('searchi-speech-worker', 'libwhisper.so.1', 'libggml-cpu-x64.so', 'libggml-vulkan.so'):
+            path = runtime / name
+            elf(path)
+            files.append({'path': name, 'bytes': path.stat().st_size, 'sha256': pack.digest(path)})
+        (runtime / 'runtime-manifest.json').write_text(json.dumps({'schema': 1,
+            'upstream_commit': speech.UPSTREAM, 'device': 'auto', 'distribution': True,
+            'source_hashes': {name: pack.digest(speech.input_path(root, name)) for name in speech.INPUTS},
+            'files': files}))
+        for name in speech.NOTICES:
+            notice = root / 'assets/licenses' / name
+            notice.parent.mkdir(parents=True, exist_ok=True)
+            notice.write_text('Approved test notice ' + name)
+            (runtime / name).write_bytes(notice.read_bytes())
+        return runtime
+
+    def test_speech_archive_has_isolated_verified_files_modes_and_instructions(self):
+        with tempfile.TemporaryDirectory() as directory:
+            fixture = self.fixture(directory)
+            runtime = self.speech_fixture(fixture)
+            with mock.patch.object(pack, 'dynamic', side_effect=self.metadata), \
+                    mock.patch.object(speech, 'verify_linux_closure') as closure:
+                pack.package(*fixture, speech_runtime=runtime)
+                closure.assert_called_once_with(runtime)
+                first = fixture[-1].read_bytes()
+                pack.package(*fixture, speech_runtime=runtime)
+                self.assertEqual(first, fixture[-1].read_bytes())
+            with tarfile.open(fixture[-1]) as archive:
+                manifest = json.load(archive.extractfile('PACKAGE.json'))
+                self.assertTrue(manifest['features']['local_speech'])
+                self.assertEqual(manifest['features']['speech_status'], 'experimental')
+                self.assertEqual(archive.getmember('speech-runtime/searchi-speech-worker').mode, 0o755)
+                for name in (*speech.NOTICES, 'runtime-manifest.json'):
+                    self.assertEqual(archive.getmember('speech-runtime/' + name).mode, 0o644)
+                for record in manifest['files']:
+                    data = archive.extractfile(record['path']).read()
+                    self.assertEqual(record['sha256'], hashlib.sha256(data).hexdigest())
+                instructions = archive.extractfile('START-HERE.txt').read().decode()
+                self.assertIn('setup --speech --model-dir ./speech-models', instructions)
+                self.assertIn('--speech-worker ./speech-runtime/searchi-speech-worker', instructions)
+                self.assertIn('--speech-model-dir ./speech-models', instructions)
+
+    def test_invalid_speech_runtime_preserves_previous_archive(self):
+        for mutation in ('cpu', 'tampered', 'notice', 'closure'):
+            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
+                fixture = self.fixture(directory)
+                runtime = self.speech_fixture(fixture)
+                fixture[-1].write_bytes(b'previous archive')
+                if mutation == 'cpu':
+                    path = runtime / 'runtime-manifest.json'
+                    manifest = json.loads(path.read_text())
+                    manifest.update(device='cpu', distribution=False)
+                    path.write_text(json.dumps(manifest))
+                elif mutation == 'tampered':
+                    (runtime / 'libwhisper.so.1').write_bytes(b'tampered')
+                elif mutation == 'notice':
+                    (runtime / speech.NOTICES[0]).unlink()
+                with mock.patch.object(pack, 'dynamic', side_effect=self.metadata), \
+                        mock.patch.object(speech, 'verify_linux_closure',
+                            side_effect=ValueError('Invalid ELF closure') if mutation == 'closure' else None):
+                    with self.assertRaises(ValueError):
+                        pack.package(*fixture, speech_runtime=runtime)
+                self.assertEqual(fixture[-1].read_bytes(), b'previous archive')
+                self.assertEqual(list(fixture[-1].parent.glob('*.partial')), [])
+
     def test_extraction_missing_stale_tampered_or_wrong_architecture_preserves_output(self):
         for mutation in ("missing", "inventory_missing", "stale", "tampered", "architecture"):
             with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: