Add local spoken passages with Google retrieval and isolated runtime setup

This commit is contained in:
karl committed 2026-10-13 20:18:44 +02:00
1 parent bc9459785d
commit dab703413e
55 files changed
+3633 -70

No files matched your search

+2 -2
View File
@@ -258,7 +258,7 @@ def _preserved_snapshot(destination, owned):
return snapshot
def stage_resources(destination, sources, validate=None):
def stage_resources(destination, sources, validate=None, owned_names=None):
"""Replace a verified complete resource directory and preserve other assets."""
destination = Path(destination).absolute()
if destination.parent != destination.parent.resolve():
@@ -272,7 +272,7 @@ def stage_resources(destination, sources, validate=None):
_check_tree(destination)
if (destination / ".searchi-stage-complete").exists():
raise RuntimeError("Reserved staging completion marker exists without its journal")
owned = _previous_owned(destination)
owned = _previous_owned(destination) if owned_names is None else set(owned_names)
owned_keys = {_resource_key(path) for path in owned}
preserved = _preserved_snapshot(destination, owned_keys)
records = []
+3
View File
@@ -15,6 +15,7 @@ import tempfile
import zipfile
from embeddinggemma2.runtime_provenance import verify as verify_provenance
from stage_extraction_worker import verify as verify_extraction
from stage_speech_worker import verify as verify_speech
ROOT = Path(__file__).resolve().parents[1]
CRT_LIBRARIES = {"concrt140.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll", "msvcp140_atomic_wait.dll", "msvcp140_codecvt_ids.dll", "vccorlib140.dll", "vcruntime140.dll", "vcruntime140_1.dll", "vcruntime140_threads.dll"}
@@ -148,6 +149,7 @@ def package(root, executable, output, cli=None):
media = media_provenance(root, resources, architecture)
expected.update(media)
expected.update(verify_extraction(resources, root, "windows", architecture))
expected.update(verify_speech(resources, root, "windows", architecture))
selected = [(executable, "searchi.exe"), (root / "LICENSE", "LICENSE.txt")]
if cli is not None:
if cli.is_symlink() or not cli.is_file() or not 0 < cli.stat().st_size <= MAX_MEMBER:
@@ -200,6 +202,7 @@ def package(root, executable, output, cli=None):
"Extract the entire folder, then open searchi.exe. Keep its helpers alongside it.\n"
"Windows needs WebView2. The Microsoft C++ runtime ships with these helpers; Visual Studio is not required.\n"
"First launch downloads approximately 865 MB of verified Google model files.\n"
"Local spoken-passage recognition adds approximately 149 MB of verified speech models.\n"
"Choose Add folder to index your own files; ordinary search stays local.\n"
"This is an unsigned development build, not a certified production release.\n")
if cli is not None:
+49
View File
@@ -0,0 +1,49 @@
cmake_minimum_required(VERSION 3.21)
project(searchi_speech_runtime LANGUAGES C CXX)
find_package(Git REQUIRED)
execute_process(COMMAND "${GIT_EXECUTABLE}" -C "${WHISPER_SOURCE}" rev-parse HEAD OUTPUT_VARIABLE revision OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE status)
if(NOT status EQUAL 0 OR NOT revision STREQUAL "d1be6fde11ac6e0407606b4e42fe72d34add8037")
message(FATAL_ERROR "Provide the pinned whisper.cpp source checkout")
endif()
option(SEARCHI_INTERNAL_CPU "Internal Windows VM CPU tool, not distribution" OFF)
set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE)
foreach(provider CUDA SYCL HIP BLAS)
set(GGML_${provider} OFF CACHE BOOL "" FORCE)
endforeach()
if(SEARCHI_INTERNAL_CPU)
set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)
set(GGML_VULKAN OFF CACHE BOOL "" FORCE)
set(GGML_METAL OFF CACHE BOOL "" FORCE)
else()
set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
set(GGML_BACKEND_DL ON CACHE BOOL "" FORCE)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64)$")
set(GGML_CPU_ALL_VARIANTS ON CACHE BOOL "" FORCE)
endif()
if(APPLE)
set(GGML_METAL ON CACHE BOOL "" FORCE)
set(GGML_VULKAN OFF CACHE BOOL "" FORCE)
else()
find_package(Vulkan REQUIRED COMPONENTS glslc)
set(GGML_VULKAN ON CACHE BOOL "" FORCE)
set(GGML_METAL OFF CACHE BOOL "" FORCE)
endif()
endif()
add_subdirectory("${WHISPER_SOURCE}" whisper)
target_compile_features(whisper PRIVATE cxx_std_17)
add_executable(searchi-speech-worker worker.cpp)
target_compile_features(searchi-speech-worker PRIVATE cxx_std_17)
target_link_libraries(searchi-speech-worker PRIVATE whisper)
if(NOT SEARCHI_INTERNAL_CPU)
target_compile_definitions(searchi-speech-worker PRIVATE SEARCHI_DISTRIBUTION_AUTO=1)
endif()
if(MSVC)
target_compile_options(searchi-speech-worker PRIVATE /utf-8)
endif()
if(UNIX)
set_target_properties(searchi-speech-worker PROPERTIES BUILD_RPATH "$ORIGIN;@loader_path" INSTALL_RPATH "$ORIGIN;@loader_path")
endif()
+79
View File
@@ -0,0 +1,79 @@
# Local speech runtime
This worker uses pinned whisper.cpp `d1be6fde11ac6e0407606b4e42fe72d34add8037`,
multilingual base and Silero v6.2.0. Google EmbeddingGemma 2 continues to embed
recognized passages; speech recognition never replaces Google semantic search.
Model download/verification belongs to searchi's shared asset setup.
Normal builds use one auto runtime with dynamic CPU and Vulkan (Windows/Linux)
or Metal (macOS) backends. Missing acceleration SDKs fail configuration. Native
speech GGML libraries belong to this isolated runtime directory, never Google's
directory. The explicit `--device cpu` build is an internal VM testing tool.
```sh
python tools/speech-runtime/build.py --source PATH_TO_CLEAN_PINNED_WHISPER_CPP --output NEW_OWNED_BUILD_DIRECTORY --device auto
```
Output `runtime/runtime-manifest.json` records every runtime file's exact size/SHA,
the pinned upstream commit and build-input hashes. The Rust client checks all
inventory files before launch and incorporates the inventory hash in its source
completion fingerprint. A changed backend DLL therefore invalidates reuse.
The builder compiles only; native Linux/macOS loading, full system-dependency
closure, actual GPU speed/fallback and release packaging still require evidence.
Windows auto builds reuse the verified Vulkan SDK/loader provisioning and selected
Visual Studio CRT bundling helpers. The isolated runtime inventories its own
`vulkan-1.dll`, full release CRT set and their redistribution notices. Shared helper
inputs (including imported provenance/macOS inspection modules) are hashed along
with the five speech build inputs; app-root DLLs are not assumed to satisfy them.
Actual internal v3 CPU evidence: English and German fixture tests passed with
`SEARCHI_FORCE_CPU=1`, the Auto client API, supplementary Unicode model/runtime
paths, repeated stable bounded passages and persistent model reuse. English ASR
was 26.31/27.80 seconds, German 28.13/22.08 seconds (first/warm) on this shared VM;
warm model loading was zero and fallback cost zero. Worker SHA-256 was
`299c1b784eac46b31d228453a399e1378f327fc9ed5e34ab950c804da33e433a`;
inventory SHA-256 `4904995685a5e55e0a790d9cc409dc8458e39c30cda4097c5d70bf75cb5c976c`.
This static CPU artifact does not prove normal dynamic-loader isolation, GPU
execution, speech retrieval quality or precise independently annotated timing.
The persistent worker receives binary requests: operation byte (1 live VAD,
2 contiguous recognition), little-endian u32 sample count and mono little-endian
f32 PCM, at most 480000 finite samples. Responses have a four-byte little-endian
JSON byte length, maximum 4 MiB, strict UTF-8. No arbitrary source paths or stored
VAD reports enter requests. Model paths are startup arguments; Windows uses
`wmain` and explicit UTF-16 → UTF-8 conversion. Native logs are consumed without
forwarding recognized bytes. Error text exposed by the parent is a fixed code.
VAD resets state for every ≤30-second source window; this is a **new live-window
policy**, not identical to the historical whole-source frozen diagnostic. It
retains original source intersections and sends actual contiguous speech PCM to
base ASR. Base loads lazily and remains warm. DTW uses the base preset, disabled
flash attention and one CPU thread. Adjacent native token byte pieces may be
joined to recover an actual UTF-8 codepoint; incomplete recognized bytes fail
closed. Sentence spans use first/last genuine DTW points and can have zero span.
No text replacement, clamped DTW time or label-specific passage exists.
The service admits a complete ≤30-second transcribe call to its shared background
budget. Metrics report native load, VAD, ASR and parent transport separately.
This does not preempt an executing native call or bound decoder/ANN CPU. A worker
operation has a 600-second IO/inference deadline; malformed framing, oversized
output, process exit and timeout stop/reap the owned worker. Native worker creates
no subprocesses. Cancellation between source windows belongs to the service.
Structural checks:
```sh
cargo test -p searchi-speech --locked
cargo clippy -p searchi-speech --all-targets --locked -- -D warnings
python -m unittest discover -s tools/speech-runtime
```
Owned subprocess transport fixtures require Python and can be run explicitly
with `cargo test -p searchi-speech actual_subprocess -- --ignored`; no model loads.
The native `searchi-speech-worker --self-test` checks strict UTF-8/JSON encoding
without loading models. Actual speech proof is a separate ignored CPU test using
`SEARCHI_TEST_SPEECH_WORKER`, `SEARCHI_TEST_SPEECH_BASE`,
`SEARCHI_TEST_SPEECH_VAD`, and verified `SEARCHI_TEST_SPEECH_PCM` (mono16k f32LE).
It relocates the runtime and exact models to owned Unicode paths, checks genuine
passages and persistent-context reuse, and cleans only that fixture.
Run actual inference only after the coordinator grants its CPU slot.
+124
View File
@@ -0,0 +1,124 @@
"""Build pinned isolated speech runtime; auto distributions require GPU SDKs."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import importlib.util
UPSTREAM = "d1be6fde11ac6e0407606b4e42fe72d34add8037"
def shared_helper(name):
path = Path(__file__).resolve().parent.parent / "embeddinggemma2" / name
spec = importlib.util.spec_from_file_location("speech_shared_" + path.stem, path)
module = importlib.util.module_from_spec(spec)
sys.path.insert(0, str(path.parent))
try:
spec.loader.exec_module(module)
finally:
sys.path.pop(0)
return module
def build_inputs():
own = Path(__file__).resolve().parent
paths = {name: own / name for name in ["CMakeLists.txt", "worker.cpp", "build.py", "cpu-only-loader.patch", "windows-model-path.patch"]}
paths.update({"embeddinggemma2/" + name: own.parent / "embeddinggemma2" / name
for name in ["build_runtime.py", "provision_vulkan.py", "runtime_provenance.py", "macos_runtime.py"]})
return {name: sha(path) for name, path in paths.items()}
def sha(path):
result = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(65536), b""):
result.update(chunk)
return result.hexdigest()
def validate_source(source):
revision = subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip()
dirty = subprocess.check_output(["git", "-C", str(source), "status", "--porcelain"], text=True).strip()
if revision != UPSTREAM or dirty:
raise ValueError("Speech build requires clean pinned whisper.cpp source")
def configuration(source, output, mode, cmake):
return [cmake, "-S", str(Path(__file__).resolve().parent), "-B", str(output / "build"),
"-DWHISPER_SOURCE=" + str(source), "-DCMAKE_BUILD_TYPE=Release",
"-DSEARCHI_INTERNAL_CPU=" + ("ON" if mode == "cpu" else "OFF"),
"-DWHISPER_BUILD_IS_DEV=OFF"]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--device", choices=["auto", "cpu"], default="auto")
parser.add_argument("--cmake", default="cmake")
args = parser.parse_args()
source, output = args.source.resolve(), args.output.resolve()
validate_source(source)
own = Path(__file__).resolve().parent
source_hashes = build_inputs()
# A fresh owned destination prevents stale CPU/DLL artifacts masquerading as auto.
output.mkdir(parents=True, exist_ok=False)
loader = None
if args.device == "auto" and sys.platform == "win32":
sdk, loader = shared_helper("provision_vulkan.py").provision()
os.environ["VULKAN_SDK"] = str(sdk)
patched = output / "whisper-source"
subprocess.run(["git", "clone", "--quiet", "--no-hardlinks", str(source), str(patched)], check=True)
subprocess.run(["git", "-C", str(patched), "checkout", "--quiet", "--detach", UPSTREAM], check=True)
for patch in ["cpu-only-loader.patch", "windows-model-path.patch"]:
subprocess.run(["git", "-C", str(patched), "apply", "--check", str(own / patch)], check=True)
subprocess.run(["git", "-C", str(patched), "apply", str(own / patch)], check=True)
for name, command in [("configure", configuration(patched, output, args.device, args.cmake)),
("build", [args.cmake, "--build", str(output / "build"), "--config", "Release", "--target", "searchi-speech-worker", "--parallel", "1"])]:
with (output / (name + ".log")).open("wb") as log:
result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=False)
if result.returncode:
raise RuntimeError(f"Speech {name} failed; inspect {output / (name + '.log')}")
validate_source(source)
if source_hashes != build_inputs():
raise ValueError("Speech build sources changed during compilation")
build = output / "build"
if args.device == "auto" and sys.platform == "win32":
shared_helper("build_runtime.py").bundle_windows_crt(build)
suffix = ".exe" if sys.platform == "win32" else ""
candidates = list(build.rglob("searchi-speech-worker" + suffix))
if len(candidates) != 1:
raise ValueError("Speech build executable is missing or ambiguous")
stage = output / "runtime"
stage.mkdir()
files = [candidates[0]]
if args.device == "auto":
files.extend(path for path in build.rglob("*") if path.is_file()
and (path.suffix.lower() in [".dll", ".dylib"] or ".so" in path.name))
if loader is not None:
files.extend([loader / "vulkan-1.dll", loader / "VULKAN-LICENSE.txt"])
files.extend(build.rglob("MSVC-REDISTRIBUTION.txt"))
names = [path.name.lower() for path in files]
required = "metal" if sys.platform == "darwin" else "vulkan"
if not any(required in name for name in names) or not any("ggml-cpu" in name for name in names):
raise ValueError("Auto speech runtime requires both acceleration and CPU backends")
records = {}
for path in files:
digest = sha(path)
if path.name in records and records[path.name]["sha256"] != digest:
raise ValueError("Ambiguous speech dependency basename")
shutil.copy2(path, stage / path.name)
records[path.name] = {"path": path.name, "bytes": path.stat().st_size, "sha256": digest}
manifest = {"schema": 1, "upstream_commit": UPSTREAM, "device": args.device,
"distribution": args.device == "auto", "source_hashes": source_hashes,
"files": list(records.values()), "native_os_gpu_checks": "unexecuted"}
(stage / "runtime-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"runtime": str(stage), "mode": args.device, "files": len(records)}))
if __name__ == "__main__":
main()
@@ -0,0 +1,31 @@
--- a/ggml/include/ggml-backend.h
+++ b/ggml/include/ggml-backend.h
@@ -260,6 +260,8 @@
// Load all known backends from dynamic libraries
GGML_API void ggml_backend_load_all(void);
GGML_API void ggml_backend_load_all_from_path(const char * dir_path);
+ // Searchi: explicit CPU-only startup must never load or probe GPU modules.
+ GGML_API void ggml_backend_load_cpu_only_from_path(const char * dir_path);
//
// Backend scheduler
--- a/ggml/src/ggml-backend-reg.cpp
+++ b/ggml/src/ggml-backend-reg.cpp
@@ -575,6 +575,17 @@
ggml_backend_load_all_from_path(nullptr);
}
+void ggml_backend_load_cpu_only_from_path(const char * dir_path) {
+#ifdef NDEBUG
+ const bool silent = true;
+#else
+ const bool silent = false;
+#endif
+ // Only CPU variants in the owned executable directory. No GPU registry
+ // probes, working-directory discovery or GGML_BACKEND_PATH injection.
+ ggml_backend_load_best("cpu", silent, dir_path);
+}
+
void ggml_backend_load_all_from_path(const char * dir_path) {
#ifdef NDEBUG
bool silent = true;
+25
View File
@@ -0,0 +1,25 @@
{
"schema": 1,
"pipeline": "whisper-base-silero-v6.2.0-dtw-base-v1",
"license": "MIT",
"files": [
{
"path": "ggml-base.bin",
"bytes": 147951465,
"sha256": "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe",
"repository": "ggerganov/whisper.cpp",
"revision": "5359861c739e955e79d9a303bcbc70fb988958b1"
},
{
"path": "ggml-silero-v6.2.0.bin",
"bytes": 885098,
"sha256": "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987",
"repository": "ggml-org/whisper-vad",
"revision": "9ffd54a1e1ee413ddf265af9913beaf518d1639b"
}
],
"license_sources": [
"https://github.com/ggml-org/whisper.cpp/blob/d1be6fde11ac6e0407606b4e42fe72d34add8037/LICENSE",
"https://github.com/snakers4/silero-vad/blob/1e261b036686cd0017d500ee96acd1c4ba572a9d/LICENSE"
]
}
+33
View File
@@ -0,0 +1,33 @@
import unittest
from pathlib import Path
import build
import tempfile
import subprocess
from unittest.mock import patch
class BuildContract(unittest.TestCase):
def test_real_git_source_rejects_wrong_revision_and_dirty_checkout(self):
with tempfile.TemporaryDirectory() as directory:
source = Path(directory)
def git(*args):
return subprocess.check_output(["git", "-C", str(source), *args], stderr=subprocess.STDOUT).decode().strip()
git("init", "--quiet")
(source / "source.txt").write_text("fixture")
git("add", "source.txt")
git("-c", "user.name=karl", "-c", "user.email=fixture@invalid", "commit", "--quiet", "-m", "fixture")
with self.assertRaises(ValueError):
build.validate_source(source)
with patch.object(build, "UPSTREAM", git("rev-parse", "HEAD")):
build.validate_source(source)
(source / "source.txt").write_text("changed")
with self.assertRaises(ValueError):
build.validate_source(source)
def test_shared_helpers_load_without_running_build_or_downloading(self):
self.assertTrue(callable(build.shared_helper("build_runtime.py").bundle_windows_crt))
self.assertTrue(callable(build.shared_helper("provision_vulkan.py").provision))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,42 @@
--- a/src/whisper.cpp
+++ b/src/whisper.cpp
@@ -30,6 +30,7 @@
#include <cstdio>
#include <cstring>
#include <fstream>
+#include <filesystem>
#include <functional>
#include <map>
#include <random>
@@ -3730,13 +3731,7 @@
struct whisper_context * whisper_init_from_file_with_params_no_state(const char * path_model, struct whisper_context_params params) {
WHISPER_LOG_INFO("%s: loading model from '%s'\n", __func__, path_model);
-#ifdef _MSC_VER
- // Convert UTF-8 path to wide string (UTF-16) for Windows, resolving character encoding issues.
- std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
- std::wstring path_model_wide = converter.from_bytes(path_model);
- auto fin = std::ifstream(path_model_wide, std::ios::binary);
-#else
- auto fin = std::ifstream(path_model, std::ios::binary);
-#endif
+ // C++17 u8path preserves supplementary Unicode on Windows and UTF-8 elsewhere.
+ auto fin = std::ifstream(std::filesystem::u8path(path_model), std::ios::binary);
if (!fin) {
WHISPER_LOG_ERROR("%s: failed to open '%s'\n", __func__, path_model);
return nullptr;
@@ -4877,13 +4871,8 @@
const char * path_model,
struct whisper_vad_context_params params) {
WHISPER_LOG_INFO("%s: loading VAD model from '%s'\n", __func__, path_model);
-#ifdef _MSC_VER
- std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
- std::wstring path_model_wide = converter.from_bytes(path_model);
- auto fin = std::ifstream(path_model_wide, std::ios::binary);
-#else
- auto fin = std::ifstream(path_model, std::ios::binary);
-#endif
+ // Same strict UTF-8 path contract as the base recognizer.
+ auto fin = std::ifstream(std::filesystem::u8path(path_model), std::ios::binary);
if (!fin) {
WHISPER_LOG_ERROR("%s: failed to open VAD model '%s'\n", __func__, path_model);
return nullptr;
+183
View File
@@ -0,0 +1,183 @@
// Isolated pinned whisper.cpp C API. Binary mono PCM input; strict UTF-8 JSON.
#include "whisper.h"
#include "ggml-backend.h"
#include <algorithm>
#include <chrono>
#include <cmath>
#include <cfloat>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#include <io.h>
#include <fcntl.h>
#endif
using Clock=std::chrono::steady_clock;
constexpr size_t MAX_SAMPLES=480000, MAX_OUTPUT=4*1024*1024;
constexpr const char* POLICY="live-window-vad-dtw-v1;silero-v6.2.0;base;30s;vad-reset;threshold0.5;minspeech250ms;minsilence100ms;maxspeechFLT_MAX;pad30ms;vad-overlap0.1s;source-sample-intersections;beam5;bestof5;temp0;tempinc0.2;entropy2.4;logprob-1;nospeech0.6;suppressblank;no-suppress-nst;auto-language;no-context;dtw-base;no-flash;contiguous-intervals;first-last-token-points;exact-utf8";
static double elapsed(Clock::time_point start){return std::chrono::duration<double,std::milli>(Clock::now()-start).count();}
static std::string backend="unknown";
static void logger(ggml_log_level level,const char * text,void*) {
// Native logs can contain recognized bytes. Never forward them in production.
const std::string value(text);
const auto using_at=value.find(": using "), end=value.find(" backend");
if(using_at!=std::string::npos && end!=std::string::npos && end>using_at+8) backend=value.substr(using_at+8,end-using_at-8);
if(value.find("no GPU found")!=std::string::npos)backend="cpu";
(void)level;
}
// Reject malformed sequences, overlong encodings, surrogates and > U+10FFFF.
static bool utf8(const std::string & text) {
size_t i=0;
while(i<text.size()) {
const uint8_t c=static_cast<uint8_t>(text[i++]);
if(c<128)continue;
unsigned n;uint32_t point,minimum;
if(c>=0xc2 && c<=0xdf){n=1;point=c&31;minimum=128;}
else if(c>=0xe0 && c<=0xef){n=2;point=c&15;minimum=2048;}
else if(c>=0xf0 && c<=0xf4){n=3;point=c&7;minimum=65536;}
else return false;
if(i+n>text.size())return false;
for(unsigned k=0;k<n;++k){const uint8_t next=static_cast<uint8_t>(text[i++]);if((next&0xc0)!=0x80)return false;point=(point<<6)|(next&63);}
if(point<minimum || point>0x10ffff || (point>=0xd800 && point<=0xdfff))return false;
}
return true;
}
static std::string quoted(const std::string & text) {
if(!utf8(text))throw std::runtime_error("malformed_recognition_utf8");
std::string result="\"";const char * hex="0123456789abcdef";
for(unsigned char c:text){if(c=='"'||c=='\\'){result+='\\';result+=static_cast<char>(c);}else if(c<32){result+="\\u00";result+=hex[c>>4];result+=hex[c&15];}else result+=static_cast<char>(c);}
return result+'"';
}
static void send(const std::string & json) {
if(json.empty() || json.size()>MAX_OUTPUT)throw std::runtime_error("output_limit");
uint32_t n=static_cast<uint32_t>(json.size());char prefix[4];for(int i=0;i<4;i++)prefix[i]=static_cast<char>((n>>(i*8))&255);
std::cout.write(prefix,4);std::cout.write(json.data(),json.size());std::cout.flush();
if(!std::cout)throw std::runtime_error("output_closed");
}
static bool boundary(std::string text) {
while(!text.empty() && std::string("\"')]").find(text.back())!=std::string::npos)text.pop_back();
for(const std::string suffix:{".","!","?","。","!","?"})if(text.size()>=suffix.size() && text.compare(text.size()-suffix.size(),suffix.size(),suffix)==0)return true;
return false;
}
struct Runtime {
std::string base_path,vad_path;bool auto_device;
whisper_context * base=nullptr;whisper_vad_context * vad=nullptr;
~Runtime(){if(base)whisper_free(base);if(vad)whisper_vad_free(vad);}
void load_base(bool gpu) {
if(!gpu)backend="cpu";
auto params=whisper_context_default_params();params.use_gpu=gpu;params.flash_attn=false;
params.dtw_token_timestamps=true;params.dtw_aheads_preset=WHISPER_AHEADS_BASE;
base=whisper_init_from_file_with_params(base_path.c_str(),params);
if(!base)throw std::runtime_error("base_load_failed");
}
std::string request(unsigned op,const std::vector<float>&pcm) {
double load=0,vad_time=0,asr_time=0;std::ostringstream body;
if(op==1) {
if(!vad){auto start=Clock::now();auto params=whisper_vad_default_context_params();params.n_threads=1;params.use_gpu=auto_device;
if(!auto_device)backend="cpu";
vad=whisper_vad_init_from_file_with_params(vad_path.c_str(),params);
if(!vad)throw std::runtime_error("vad_load_failed");load=elapsed(start);}
auto start=Clock::now();
if(!whisper_vad_detect_speech(vad,pcm.data(),static_cast<int>(pcm.size())))throw std::runtime_error("vad_failed");
const whisper_vad_params policy{0.5f,250,100,FLT_MAX,30,0.1f};
auto segments=whisper_vad_segments_from_probs(vad,policy);if(!segments)throw std::runtime_error("vad_intervals_failed");
body<<",\"intervals_samples\":[";size_t previous=0;bool comma=false;
const int n=whisper_vad_segments_n_segments(segments);
if(n<0||n>1000){whisper_vad_free_segments(segments);throw std::runtime_error("vad_interval_limit");}
for(int i=0;i<n;i++){
const double a=whisper_vad_segments_get_segment_t0(segments,i),b=whisper_vad_segments_get_segment_t1(segments,i);
if(!std::isfinite(a)||!std::isfinite(b)||a<0||a>=b||std::floor(a)!=a||std::floor(b)!=b){whisper_vad_free_segments(segments);throw std::runtime_error("vad_invalid_interval");}
// Native VAD tail padding intersects source samples; never invent PCM.
const size_t first=static_cast<size_t>(std::min(a*160.0,static_cast<double>(pcm.size())));
const size_t last=static_cast<size_t>(std::min(b*160.0,static_cast<double>(pcm.size())));
if(first<previous){whisper_vad_free_segments(segments);throw std::runtime_error("vad_overlapping_interval");}
previous=last;if(first==last)continue;if(comma)body<<',';comma=true;body<<'['<<first<<','<<last<<']';
}
whisper_vad_free_segments(segments);body<<']';vad_time=elapsed(start);
}else if(op==2) {
if(!base){auto start=Clock::now();load_base(auto_device);load=elapsed(start);}
auto params=whisper_full_default_params(WHISPER_SAMPLING_BEAM_SEARCH);
params.n_threads=1;params.language="auto";params.detect_language=false;
params.greedy.best_of=5;params.beam_search.beam_size=5;
params.temperature=0.0f;params.temperature_inc=0.2f;
params.entropy_thold=2.4f;params.logprob_thold=-1.0f;params.no_speech_thold=0.6f;
params.suppress_blank=true;params.suppress_nst=false;params.translate=false;
params.print_progress=false;params.print_realtime=false;params.print_timestamps=false;params.print_special=false;
params.token_timestamps=true;params.no_context=true;
auto start=Clock::now();int status=whisper_full(base,params,pcm.data(),static_cast<int>(pcm.size()));
if(status!=0)throw std::runtime_error("recognition_failed");
body<<",\"sentences\":[";bool comma=false;int64_t previous=0;
const int count=whisper_full_n_segments(base);if(count<0||count>1000)throw std::runtime_error("segment_limit");
for(int i=0;i<count;i++){
const std::string original=whisper_full_get_segment_text(base,i);
if(original.size()>4096||!utf8(original))throw std::runtime_error("malformed_recognition_utf8");
std::string exact,phrase,merged;int64_t first=-1,last=-1,merge_first=-1;
const int tokens=whisper_full_n_tokens(base,i);if(tokens<0||tokens>4096)throw std::runtime_error("token_limit");
auto append=[&](){if(phrase.find_first_not_of(" \r\n\t")!=std::string::npos){if(comma)body<<',';comma=true;body<<"{\"start_cs\":"<<first<<",\"end_cs\":"<<last<<",\"text\":"<<quoted(phrase)<<'}';}phrase.clear();first=-1;};
for(int j=0;j<tokens;j++){
auto token=whisper_full_get_token_data(base,i,j);if(token.id>=whisper_token_eot(base))continue;
if(token.t_dtw<previous || token.t_dtw<0 || static_cast<uint64_t>(token.t_dtw)*160>pcm.size())throw std::runtime_error("invalid_dtw_point");
previous=token.t_dtw;if(merge_first<0)merge_first=token.t_dtw;
merged+=whisper_token_to_str(base,token.id);
if(merged.size()>4096)throw std::runtime_error("token_text_limit");
if(!utf8(merged))continue;
exact+=merged;if(first<0)first=merge_first;last=token.t_dtw;phrase+=merged;
if(boundary(merged))append();merged.clear();merge_first=-1;
}
if(!merged.empty() || exact!=original)throw std::runtime_error("malformed_recognition_utf8");append();
}
body<<']';asr_time=elapsed(start);
}else throw std::runtime_error("unknown_operation");
std::ostringstream output;output<<"{\"ok\":true,\"operation\":"<<op<<",\"samples\":"<<pcm.size()<<",\"vad_ms\":"<<vad_time<<",\"asr_ms\":"<<asr_time<<",\"load_ms\":"<<load<<",\"backend_observed\":"<<quoted(backend)<<body.str()<<'}';return output.str();
}
};
static int run(const std::vector<std::string>&args){
if(args.size()==2 && args[1]=="--self-test"){
if(!utf8("\xe2\x80\x8d") || utf8("\xe2\x80") || utf8("\xed\xa0\x80") || utf8("\xc0\x80") || utf8("\xf4\x90\x80\x80"))return 5;
if(quoted(std::string("a\n\"\\"))!="\"a\\u000a\\\"\\\\\"")return 5;
return 0;
}
if(args.size()!=4)return 2;
const uint16_t endian=1;if(*reinterpret_cast<const uint8_t*>(&endian)!=1||sizeof(float)!=4)return 2;
#ifdef _WIN32
_setmode(_fileno(stdin),_O_BINARY);_setmode(_fileno(stdout),_O_BINARY);
SetDefaultDllDirectories(LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
#endif
if(args[3]!="auto" && args[3]!="cpu")return 2;
const char* force_cpu=std::getenv("SEARCHI_FORCE_CPU");
const bool auto_device=args[3]=="auto" && !(force_cpu && std::string(force_cpu)=="1");
whisper_log_set(logger,nullptr);
const auto directory=std::filesystem::u8path(args[0]).parent_path().u8string();
if(auto_device)ggml_backend_load_all_from_path(directory.c_str());
else ggml_backend_load_cpu_only_from_path(directory.c_str());
Runtime runtime{args[1],args[2],auto_device};
#ifdef SEARCHI_DISTRIBUTION_AUTO
const bool distribution_auto=true;
#else
const bool distribution_auto=false;
#endif
send(std::string("{\"ready\":true,\"protocol\":1,\"runtime\":\"d1be6fde11ac6e0407606b4e42fe72d34add8037\",\"distribution_auto\":")+(distribution_auto?"true":"false")+",\"policy\":"+quoted(POLICY)+'}');
while(true){char header[5];std::cin.read(header,5);if(std::cin.gcount()==0&&std::cin.eof())return 0;if(std::cin.gcount()!=5)return 3;
uint32_t count=0;for(int i=0;i<4;i++)count|=static_cast<uint32_t>(static_cast<uint8_t>(header[i+1]))<<(i*8);
if(count==0||count>MAX_SAMPLES)return 3;std::vector<float> pcm(count);
std::cin.read(reinterpret_cast<char*>(pcm.data()),count*4);if(!std::cin)return 3;
if(!std::all_of(pcm.begin(),pcm.end(),[](float v){return std::isfinite(v);} ))return 3;
try{send(runtime.request(static_cast<uint8_t>(header[0]),pcm));}
catch(const std::exception&error){send(std::string("{\"ok\":false,\"error\":")+quoted(error.what())+'}');}
}
}
#ifdef _WIN32
int wmain(int argc,wchar_t**argv){std::vector<std::string> args;for(int i=0;i<argc;i++){int n=WideCharToMultiByte(CP_UTF8,WC_ERR_INVALID_CHARS,argv[i],-1,nullptr,0,nullptr,nullptr);if(!n)return 2;std::string value(n,'\0');WideCharToMultiByte(CP_UTF8,WC_ERR_INVALID_CHARS,argv[i],-1,value.data(),n,nullptr,nullptr);value.pop_back();args.push_back(value);}return run(args);}
#else
int main(int argc,char**argv){return run(std::vector<std::string>(argv,argv+argc));}
#endif
+135
View File
@@ -0,0 +1,135 @@
"""Verify and atomically stage the isolated normal speech runtime; no inference."""
import argparse
import hashlib
import json
from pathlib import Path
import re
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tools/embeddinggemma2'))
from stage_runtime import stage_resources, CRT_LIBRARIES
from stage_extraction_worker import identity
UPSTREAM = 'd1be6fde11ac6e0407606b4e42fe72d34add8037'
INPUTS = ('CMakeLists.txt', 'worker.cpp', 'build.py',
'cpu-only-loader.patch', 'windows-model-path.patch',
'embeddinggemma2/build_runtime.py', 'embeddinggemma2/provision_vulkan.py',
'embeddinggemma2/runtime_provenance.py', 'embeddinggemma2/macos_runtime.py')
NOTICES = ('Whisper-cpp-MIT.txt', 'Whisper-model-MIT.txt', 'Silero-MIT.txt')
def digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def input_path(root, name):
return Path(root) / 'tools' / (name if '/' in name else 'speech-runtime/' + name)
def inventory(directory):
path = Path(directory) / 'runtime-manifest.json'
if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= 512 * 1024:
raise ValueError('Speech runtime inventory missing or oversized')
manifest = json.loads(path.read_bytes())
if (not isinstance(manifest, dict) or manifest.get('schema') != 1
or manifest.get('upstream_commit') != UPSTREAM
or manifest.get('device') != 'auto' or manifest.get('distribution') is not True):
raise ValueError('Normal releases require the pinned hardware-detecting speech runtime')
files = manifest.get('files')
if not isinstance(files, list) or not 1 <= len(files) <= 128:
raise ValueError('Speech runtime inventory is unbounded')
names, total = set(), 0
for record in files:
if not isinstance(record, dict):
raise ValueError('Invalid speech artifact')
name, size, sha = record.get('path'), record.get('bytes'), record.get('sha256')
if (not isinstance(name, str) or not re.fullmatch(r'[A-Za-z0-9_.-]+', name)
or name in {'.', '..'} or name.lower() in names
or type(size) is not int or not 0 < size <= 256 * 1024 * 1024
or not isinstance(sha, str) or not re.fullmatch(r'[0-9a-f]{64}', sha)):
raise ValueError('Invalid or duplicate speech artifact')
names.add(name.lower())
total += size
if total > 1024 * 1024 * 1024:
raise ValueError('Speech runtime exceeds size budget')
artifact = Path(directory) / name
if (artifact.is_symlink() or not artifact.is_file() or artifact.absolute() != artifact.resolve()
or artifact.stat().st_size != size or digest(artifact) != sha):
raise ValueError('Speech artifact size/SHA mismatch')
permitted = names | {'runtime-manifest.json'} | {name.lower() for name in NOTICES}
for path in Path(directory).iterdir():
if not path.is_file() or path.is_symlink() or path.name.lower() not in permitted:
raise ValueError('Unlisted speech runtime artifact')
return manifest
def verify_directory(directory, root=ROOT, platform='windows', architecture='x86_64'):
directory = Path(directory)
manifest = inventory(directory)
expected = {name: digest(input_path(root, name)) for name in INPUTS}
if manifest.get('source_hashes') != expected:
raise ValueError('Speech runtime build inputs are stale')
worker = 'searchi-speech-worker.exe' if platform == 'windows' else 'searchi-speech-worker'
detected_platform, detected_arch, _ = identity(directory / worker)
if (detected_platform, detected_arch) != (platform, architecture):
raise ValueError('Speech runtime architecture mismatch')
names = {record['path'] for record in manifest['files']}
accelerator = 'metal' if platform == 'macos' else 'vulkan'
if (worker not in names or not any('ggml-cpu' in name for name in names)
or not any(accelerator in name for name in names)):
raise ValueError('Speech runtime lacks CPU/acceleration backends')
windows_libraries = {'whisper.dll', 'ggml.dll', 'ggml-base.dll', 'vulkan-1.dll',
'VULKAN-LICENSE.txt', 'MSVC-REDISTRIBUTION.txt'} | CRT_LIBRARIES
if platform == 'windows' and not windows_libraries.issubset(names):
raise ValueError('Speech runtime lacks required shared libraries')
records = {'speech-runtime/' + record['path']: record for record in manifest['files']}
manifest_path = directory / 'runtime-manifest.json'
records['speech-runtime/runtime-manifest.json'] = {
'bytes': manifest_path.stat().st_size, 'sha256': digest(manifest_path)}
for name in NOTICES:
notice = directory / name
original = Path(root) / 'assets/licenses' / name
if (notice.is_symlink() or not notice.is_file() or notice.stat().st_size > 65536
or notice.stat().st_size != original.stat().st_size
or digest(notice) != digest(original)):
raise ValueError('Speech license notice missing or changed')
records['speech-runtime/' + name] = {'bytes': notice.stat().st_size, 'sha256': digest(notice)}
return records
def verify(resources, root=ROOT, platform='windows', architecture='x86_64'):
return verify_directory(Path(resources) / 'speech-runtime', root, platform, architecture)
def stage(runtime, destination, root=ROOT, platform='windows', architecture='x86_64'):
runtime, destination, root = map(Path, (runtime, destination, root))
manifest = inventory(runtime)
current = {record['path'] for record in manifest['files']} | {'runtime-manifest.json'}
owned = current | set(NOTICES)
if destination.exists():
previous = inventory(destination)
owned.update(record['path'] for record in previous['files'])
sources = [(runtime / name, name) for name in current]
sources = [(source, name) for source, name in sources if name not in NOTICES]
sources.extend((root / 'assets/licenses' / name, name) for name in NOTICES)
def validate(candidate):
verify_directory(candidate, root, platform, architecture)
stage_resources(destination, sources, validate, owned_names=owned)
verify_directory(destination, root, platform, architecture)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--runtime', type=Path, required=True)
parser.add_argument('--platform', choices=['windows', 'linux', 'macos'], required=True)
parser.add_argument('--architecture', choices=['x86_64', 'aarch64'], required=True)
args = parser.parse_args()
stage(args.runtime, ROOT / 'apps/desktop/src-tauri/resources/speech-runtime',
ROOT, args.platform, args.architecture)
print('Staged verified speech runtime')
if __name__ == '__main__':
main()
+46
View File
@@ -10,6 +10,7 @@ import zipfile
from package_windows import package, CRT_LIBRARIES
from embeddinggemma2 import runtime_provenance as provenance
import stage_extraction_worker as extraction
import stage_speech_worker as speech
def executable(path, machine=0x8664):
@@ -32,6 +33,29 @@ def inventory(resources):
class PackageTests(unittest.TestCase):
def test_speech_cpu_only_missing_and_tampered_runtime_preserve_previous_package(self):
for mutation in ('cpu', 'missing', 'tampered', 'stale', 'unlisted'):
with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
root, resources, app, output = self.fixture(directory)
output.write_bytes(b'previous package')
speech_directory = resources / 'speech-runtime'
if mutation == 'cpu':
path = speech_directory / 'runtime-manifest.json'
data = json.loads(path.read_text())
data.update(device='cpu', distribution=False)
path.write_text(json.dumps(data))
elif mutation == 'missing':
(speech_directory / 'searchi-speech-worker.exe').unlink()
elif mutation == 'tampered':
(speech_directory / 'ggml-vulkan.dll').write_bytes(b'changed')
elif mutation == 'stale':
(root / 'tools/speech-runtime/worker.cpp').write_bytes(b'changed source')
else:
executable(speech_directory / 'obsolete-ggml-cpu.dll')
with self.assertRaises(ValueError):
package(root, app, output)
self.assertEqual(output.read_bytes(), b'previous package')
def test_extraction_missing_tampered_stale_preserves_existing_package(self):
for mutation in ("missing", "tampered", "stale"):
with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
@@ -119,6 +143,28 @@ class PackageTests(unittest.TestCase):
helper = root / "build/searchi-extract-worker.exe"
executable(helper)
extraction.stage(helper, extraction.fingerprint(root), resources, root)
speech_directory = resources / 'speech-runtime'
speech_directory.mkdir()
for name in speech.INPUTS:
path = speech.input_path(root, name)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(speech.input_path(speech.ROOT, name).read_bytes())
for name in speech.NOTICES:
original = root / 'assets/licenses' / name
original.parent.mkdir(parents=True, exist_ok=True)
original.write_bytes((speech.ROOT / 'assets/licenses' / name).read_bytes())
(speech_directory / name).write_bytes(original.read_bytes())
speech_files = []
for name in ('searchi-speech-worker.exe', 'ggml-cpu-x64.dll', 'ggml-vulkan.dll',
'whisper.dll', 'ggml.dll', 'ggml-base.dll', 'vulkan-1.dll',
'VULKAN-LICENSE.txt', 'MSVC-REDISTRIBUTION.txt', *sorted(CRT_LIBRARIES)):
path = speech_directory / name
executable(path)
speech_files.append({'path': name, 'bytes': path.stat().st_size, 'sha256': speech.digest(path)})
(speech_directory / 'runtime-manifest.json').write_text(json.dumps({
'schema': 1, 'upstream_commit': speech.UPSTREAM, 'device': 'auto', 'distribution': True,
'source_hashes': {name: speech.digest(speech.input_path(root, name)) for name in speech.INPUTS},
'files': speech_files}))
return root, resources, app, root / "archive.zip"
def test_valid_pe_media_mutation_and_notice_corruption_preserve_previous_archive(self):
+44
View File
@@ -0,0 +1,44 @@
import json
from pathlib import Path
import shutil
import tempfile
import unittest
import stage_speech_worker as speech
import test_package_windows as windows
class SpeechStagingTests(unittest.TestCase):
def test_replacement_prunes_obsolete_declared_code_and_keeps_closed_inventory(self):
with tempfile.TemporaryDirectory() as directory:
root, resources, _, _ = windows.PackageTests().fixture(directory)
source = resources / 'speech-runtime'
destination = root / 'candidate-resources/speech-runtime'
shutil.copytree(source, destination)
obsolete = destination / 'obsolete-ggml-cpu.dll'
windows.executable(obsolete)
path = destination / 'runtime-manifest.json'
manifest = json.loads(path.read_text())
manifest['files'].append({'path': obsolete.name, 'bytes': obsolete.stat().st_size,
'sha256': speech.digest(obsolete)})
path.write_text(json.dumps(manifest))
speech.stage(source, destination, root)
self.assertFalse(obsolete.exists())
speech.verify_directory(destination, root)
def test_invalid_source_retains_original_runtime_and_leaves_no_partial_directory(self):
with tempfile.TemporaryDirectory() as directory:
root, resources, _, _ = windows.PackageTests().fixture(directory)
source = resources / 'speech-runtime'
destination = root / 'candidate-resources/speech-runtime'
shutil.copytree(source, destination)
previous = (destination / 'runtime-manifest.json').read_bytes()
windows.executable(source / 'unlisted.dll')
with self.assertRaisesRegex(ValueError, 'Unlisted'):
speech.stage(source, destination, root)
self.assertEqual((destination / 'runtime-manifest.json').read_bytes(), previous)
self.assertEqual(list(destination.parent.glob('.resources.stage-*')), [])
if __name__ == '__main__':
unittest.main()