karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

Package verified speech runtime in Linux headless archives

Browse Source
This commit is contained in:
karl committed 2026-10-13 21:38:31 +02:00
1 parent fb36c53e2e
commit 36bd51000b
4 files changed
+121 -6
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/package_linux.py
+23 -6
View File
@@ -18,6 +18,7 @@ import tarfile
import tempfile
from embeddinggemma2.runtime_provenance import verify as verify_provenance
from stage_extraction_worker import verify as verify_extraction
from stage_speech_worker import verify_directory as verify_speech
ROOT = Path(__file__).resolve().parents[1]
MAX_FILE = 256 * 1024 * 1024
@@ -173,7 +174,7 @@ def media_files(root, media):
return selected
def package(root, executable, runtime, media, output, llama_license=None, compression="gzip"):
def package(root, executable, runtime, media, output, llama_license=None, compression="gzip", speech_runtime=None):
if compression not in {"gzip", "xz"}:
raise ValueError("Unsupported package compression")
root, executable, runtime, media, output = map(Path, (root, executable, runtime, media, output))
@@ -195,6 +196,11 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
or set(cli_metadata.get("NEEDED", [])) - SYSTEM):
raise ValueError("CLI has unexpected native dependency or search path")
selected = {name: (runtime / name, record) for name, record in expected.items()}
if speech_runtime is not None:
speech_runtime = Path(speech_runtime)
speech_records = verify_speech(speech_runtime, root, 'linux', 'x86_64')
selected.update({name: (speech_runtime / PurePosixPath(name).name, record)
for name, record in speech_records.items()})
selected.update(media_files(root, media))
selected["searchi"] = (executable, {"sha256": digest(executable), "bytes": executable.stat().st_size})
license_path = Path(llama_license) if llama_license else root / ".searchi/references/llama.cpp/LICENSE"
@@ -225,9 +231,15 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
checksum = hashlib.sha256(data).hexdigest()
if len(data) != expected_record["bytes"] or checksum != expected_record["sha256"]:
raise ValueError(f"Source changed during packaging: {name}")
mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "searchi-extract-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644
mode = 0o755 if name in {"searchi", "searchi-gemma-worker", "searchi-extract-worker", "speech-runtime/searchi-speech-worker", "media-runtime/bin/ffmpeg", "media-runtime/bin/ffprobe"} or ".so" in name else 0o644
add(archive, name, data, mode)
records.append({"path": name, "bytes": len(data), "sha256": checksum, "mode": oct(mode)})
speech_instructions = ("\nExperimental local speech search is included. Download verified recognition assets (about 149 MB):\n"
"./searchi setup --speech --model-dir ./speech-models\n"
"Then index recognized passages alongside Google media embeddings:\n"
"./searchi embed --db ./searchi.sqlite --model-dir ./models --embedding-worker ./searchi-gemma-worker --speech-worker ./speech-runtime/searchi-speech-worker --speech-model-dir ./speech-models\n"
"Search uses the same command above; spoken passages carry estimated source timestamps.\n"
"Recognition and search remain local. Music/no-answer relevance and precise timestamps remain experimental.\n") if speech_runtime is not None else ""
instructions = ("searchi — Find what matters. Locally.\n\n"
"Extract the entire archive into its own folder. This is a headless development candidate, not a production installer.\n"
"Use ./searchi --help. Setup downloads about 865 MB of verified Google model assets:\n"
@@ -237,14 +249,18 @@ def package(root, executable, runtime, media, output, llama_license=None, compre
"./searchi search --db ./searchi.sqlite 'what you remember' --model-dir ./models --embedding-worker ./searchi-gemma-worker\n"
"Media helpers are discovered automatically beside this CLI. Ordinary search stays local.\n"
"The normal runtime includes Vulkan and CPU fallback. Graphics drivers and a compatible system Vulkan loader are optional.\n"
"Requires a compatible Linux libc/C++ ABI; clean-machine and distribution coverage remain open.\n").encode()
"Requires a compatible Linux libc/C++ ABI; clean-machine and distribution coverage remain open.\n" + speech_instructions).encode()
add(archive, "START-HERE.txt", instructions, 0o644)
records.append({"path": "START-HERE.txt", "bytes": len(instructions),
"sha256": hashlib.sha256(instructions).hexdigest(), "mode": oct(0o644)})
add(archive, "PACKAGE.json", json.dumps({"platform": "linux", "architecture": "x86_64",
metadata = {"platform": "linux", "architecture": "x86_64",
"status": "unsigned headless development candidate; distribution ABI and GPU hardware validation pending",
"system_libraries": sorted(SYSTEM), "optional_gpu_loader": "libvulkan.so.1",
"files": records}, indent=2).encode(), 0o644)
"files": records}
if speech_runtime is not None:
metadata['features'] = {'local_speech': True, 'speech_status': 'experimental',
'speech_runtime': 'speech-runtime/searchi-speech-worker'}
add(archive, "PACKAGE.json", json.dumps(metadata, indent=2).encode(), 0o644)
os.replace(temporary_path, output)
finally:
temporary_path.unlink(missing_ok=True)
@@ -264,13 +280,14 @@ def main():
parser.add_argument("--executable", type=Path, default=ROOT / "target/release/searchi")
parser.add_argument("--runtime", type=Path, default=ROOT / ".searchi/runtime/auto")
parser.add_argument("--media", type=Path, default=ROOT / ".searchi/media-runtime")
parser.add_argument("--speech-runtime", type=Path, help="Verified normal auto speech runtime directory")
parser.add_argument("--llama-license", type=Path)
parser.add_argument("--compression", choices=["gzip", "xz"], default="gzip")
parser.add_argument("--output", type=Path)
args = parser.parse_args()
if args.output is None:
args.output = ROOT / (".searchi/packages/searchi-linux-x86_64.tar.xz" if args.compression == "xz" else ".searchi/packages/searchi-linux-x86_64.tar.gz")
package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license, args.compression)
package(ROOT, args.executable, args.runtime, args.media, args.output, args.llama_license, args.compression, args.speech_runtime)
print(f"Packaged Linux x86_64 development candidate: {args.output}")