karl/searchi
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases 10 Wiki Activity

feat: add multimodal search and granular performance diagnostics
Core CI / core (ubuntu-latest) (push) Failing after 3m31s
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Successful in 5m39s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s

Browse Source
This commit is contained in:
karl committed 2026-10-12 19:16:23 +02:00
1 parent d3bb8c7618
commit b8400c4fbf
110 files changed
+79395 -250
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+1
View File
@@ -9,5 +9,6 @@ __pycache__/
/apps/desktop/src-tauri/gen/
/apps/desktop/src-tauri/resources/searchi-gemma-worker*
/apps/desktop/src-tauri/resources/embeddinggemma2/
/apps/desktop/src-tauri/resources/media-runtime/bin/
/apps/desktop/src-tauri/icons/android/
/apps/desktop/src-tauri/icons/ios/
AGENTS.md
+3
View File
@@ -10,6 +10,9 @@ readiness without completed release gates and recorded evidence.
- Production uses available GPU/hardware acceleration with reliable CPU fallback;
the CPU-only rule is a VM testing constraint, not a product restriction.
- Keep the Rust engine independent of the desktop shell; CLI and UI share its API.
- Prefer established Rust crates and proven retrieval algorithms over custom
implementations. Keep platform adapters small, bound allocations and queues,
and measure performance before adding caches or changing ranking policies.
- Use bounded work, explicit errors, versioned storage, and offline defaults.
- Never substitute simulated embeddings for working semantic search.
- Production semantic search must use Google EmbeddingGemma 2 as in DigUp.
Cargo.lock
Generated
+130 -1
View File
@@ -207,6 +207,18 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
version = "1.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
[[package]]
name = "byteorder-lite"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
[[package]]
name = "bytes"
version = "1.12.1"
@@ -705,6 +717,15 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
[[package]]
name = "fdeflate"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
dependencies = [
"simd-adler32",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.14"
@@ -718,7 +739,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
dependencies = [
"crc32fast",
"miniz_oxide",
"miniz_oxide 0.9.1",
"zlib-rs",
]
@@ -1098,6 +1119,32 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "image"
version = "0.25.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
dependencies = [
"bytemuck",
"byteorder-lite",
"image-webp",
"moxcms",
"num-traits",
"png",
"zune-core",
"zune-jpeg",
]
[[package]]
name = "image-webp"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
dependencies = [
"byteorder-lite",
"quick-error",
]
[[package]]
name = "indexmap"
version = "2.14.2"
@@ -1145,6 +1192,12 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jpeg-decoder"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
[[package]]
name = "js-sys"
version = "0.3.106"
@@ -1300,6 +1353,16 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
"simd-adler32",
]
[[package]]
name = "miniz_oxide"
version = "0.9.1"
@@ -1343,6 +1406,16 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "moxcms"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
dependencies = [
"num-traits",
"pxfm",
]
[[package]]
name = "multiversion_no_op"
version = "1.0.0"
@@ -1486,6 +1559,19 @@ version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
[[package]]
name = "png"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
dependencies = [
"bitflags",
"crc32fast",
"fdeflate",
"flate2",
"miniz_oxide 0.8.9",
]
[[package]]
name = "portable-atomic"
version = "1.15.0"
@@ -1528,6 +1614,18 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "pxfm"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
[[package]]
name = "quick-error"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
[[package]]
name = "quick-xml"
version = "0.42.0"
@@ -1946,11 +2044,26 @@ dependencies = [
"zip",
]
[[package]]
name = "searchi-media"
version = "0.1.0"
dependencies = [
"anyhow",
"image",
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
]
[[package]]
name = "searchi-semantic"
version = "0.1.0"
dependencies = [
"anyhow",
"base64 0.22.1",
"image",
"jpeg-decoder",
"ort",
"serde",
"serde_json",
@@ -1966,6 +2079,7 @@ dependencies = [
"fs2",
"rusqlite",
"searchi-core",
"searchi-media",
"searchi-semantic",
"serde",
"serde_json",
@@ -2880,3 +2994,18 @@ dependencies = [
"log",
"simd-adler32",
]
[[package]]
name = "zune-core"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b"
[[package]]
name = "zune-jpeg"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
dependencies = [
"zune-core",
]
Cargo.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[workspace]
members = ["crates/searchi-core", "crates/searchi-cli", "crates/searchi-semantic", "crates/searchi-service", "crates/searchi-assets"]
members = ["crates/searchi-core", "crates/searchi-cli", "crates/searchi-semantic", "crates/searchi-service", "crates/searchi-assets", "crates/searchi-media"]
resolver = "2"
[workspace.package]
README.md
+118 -63
View File
@@ -2,89 +2,144 @@
**Find what matters. Locally.**
Local desktop search for Windows, macOS and Linux. Under active development;
production readiness is tracked in [the roadmap](docs/ROADMAP.md).
Local desktop search for Windows, macOS and Linux, under active development.
Production readiness is tracked in [the roadmap](docs/ROADMAP.md).
## Architecture
## What works
`crates/searchi-core` owns persistent indexing and retrieval. `crates/searchi-cli`
provides a headless interface. `apps/desktop` hosts the Tauri desktop app.
Search combines indexed filenames, SQLite FTS5 keyword retrieval, and real
Google EmbeddingGemma 2 semantic search with a persistent HNSW index. PDF pages
and DOCX paragraphs retain source locations. Image, audio, and video search remain
planned; the provisioned media projector is not yet used. Granola is not required.
The desktop UI is primary; the CLI builds separately without GUI dependencies.
Search combines instant indexed filenames, SQLite FTS5 content retrieval, and
**real Google EmbeddingGemma 2** vectors in a persisted HNSW index. PDF pages,
DOCX paragraphs and text passages retain their source locations. The same
shared Rust service now indexes **images, video frames, movie soundtracks,
audio and music**: actual pixels and sound enter Google's media projector in
the same 768-dimensional space as text queries. No generated captions,
simulated embeddings, replacement model, or Granola service is required.
Existing artist/title tags contribute keyword matches.
Production uses supported hardware acceleration with CPU fallback. All testing
in the Windows development VM uses CPU only. Indexed content stays
local. Treat the index as sensitive: it contains extracted document text.
The desktop is the primary interface. It automatically prepares verified local
models, indexes selected folders, and prewarms its query worker. Image previews
and audio/video controls use narrowly scoped local assets; timestamp results
seek inside the preview without autoplay. Open file launches the default
application, whose exact opening position is not controlled by Searchi.
The CLI provides the same retrieval engine without a GUI.
The desktop automatically downloads and verifies missing Google artifacts
(approximately 865 MB), reuses its local cache, and warms the query session in
the background. Development builds must build and stage the native worker first;
production installers and signing remain release gates. First-query performance
is measured separately from repeated-query performance.
**Media search is experimental.** File retrieval works on the independent seed
corpus, but precise spoken-topic timestamps, music hard negatives, and no-answer
handling still need improvement. Nearest-neighbor results are not calibrated
confidence. Video frames are sampled across the movie; every fleeting scene is
not guaranteed. Audio windows cover the primary stream within explicit resource
limits. See [actual media quality and failures](docs/MEDIA-QUALITY.md).
Production supports compiled hardware acceleration with CPU fallback. All
Windows VM tests use CPU only. Files, extracted text, metadata and embeddings
stay local; treat the index as sensitive. Initial verified model provisioning
requires approximately 865 MB of downloads. Search performs no remote inference.
## Copy-and-paste setup prompt
Paste this into your coding agent:
```text
Set up Searchi from https://tea.karlcom.de/karl/searchi.git in a new searchi
directory, or reuse the existing checkout without overwriting local changes.
Read AGENTS.md, SKILLS.md, docs/ROADMAP.md, and apps/desktop/README.md first.
Inspect the OS and installed tools. Install missing supported prerequisites:
Git, stable Rust, Node.js 24+, Python 3, CMake, native C/C++ tools and Tauri's
platform dependencies. Preserve existing credentials and never print secrets.
From the repository root, build and stage the pinned Google runtime:
folder, or reuse the checkout without overwriting local changes. Read AGENTS.md,
SKILLS.md, docs/ROADMAP.md, docs/BUILDING.md and apps/desktop/README.md first.
Inspect the OS/tools. Install supported missing prerequisites: Git, stable Rust,
Node.js 24 LTS, Python 3.11+, CMake, native C/C++ tools and Tauri dependencies.
Preserve credentials and never print secrets. Prefer established Rust crates
and portable native tools; keep engine APIs shared by CLI and desktop.
For Karl's Windows VM set SEARCHI_FORCE_CPU=1 and
WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS='--disable-gpu --disable-accelerated-video-decode'
before tests or launch. Never enable a GPU provider in that VM.
From the repository root:
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
python tools/embeddinggemma2/stage_runtime.py --backend cpu
Run cargo test --workspace --locked, cargo fmt --all -- --check, and
python tools/media-runtime/provision.py --stage
The decoder manifest supports Windows/Linux x86-64 and ARM64; macOS decoder
packaging is still a gate. On unsupported hosts report the real blocker and
use explicitly verified native tools only when their provenance is known.
Run cargo test --workspace --locked, cargo fmt --all --check and
cargo clippy --workspace --all-targets --locked -- -D warnings.
Run cargo test --manifest-path apps/desktop/src-tauri/Cargo.toml --locked.
In apps/desktop run npm ci, npm test, npm run format:check, and npm run build.
For Karl's Windows VM set SEARCHI_FORCE_CPU=1 and
WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS=--disable-gpu before tests or launch.
Launch the real app using npm run tauri dev from apps/desktop. Allow automatic
verified Google model setup; never substitute another model or mock results.
Ask which folders I want indexed; do not index my whole home automatically.
Verify filename and remembered-content searches on my selected files, including
persistence after restart. Report actual checks and blockers. Do not publish a
release or claim production readiness, GPU validation, or untested OS coverage.
In apps/desktop run npm ci, npm test, npm run format:check and npm run build.
Launch the real app with npm run tauri -- dev. Allow automatic verified Google
model setup and retained-worker warmup. Never substitute another model or mocks.
Ask which folders I want indexed; don't index my whole home automatically.
Verify filename/content/image/music/video searches and persistence on selected
files. Inspect media failure reports and actual preview seeking. Read
MEDIA-QUALITY.md before claiming precise speech timestamps or reliable absence.
Report actual checks and blockers. Do not publish a release or claim production
readiness, GPU validation, native visual QA, or untested OS coverage.
```
This is developer setup, not an installer. See the [desktop guide](apps/desktop/README.md)
and [official Tauri prerequisites](https://v2.tauri.app/start/prerequisites/).
The worker builder also supports `auto`, `metal`, `vulkan`, and `cuda`; those
require platform SDKs and separate hardware validation.
This is developer setup, not an installer. The worker builder also accepts
`auto`, `metal`, `vulkan` and `cuda`; each requires its platform SDK and separate
hardware validation. Standalone FFmpeg/FFprobe and notices are staged for the
desktop; end users should not install tools or configure decoder paths.
## Development
## Headless setup
Install stable Rust and the platform's C/C++ build prerequisites. Run:
Build with `cargo build --release --locked -p searchi-cli`. No Node, Tauri,
WebView or display server is needed. Build the native Google worker separately,
and provision verified assets and FFmpeg tools:
```sh
cargo test --workspace
cargo fmt --all -- --check
cargo clippy --workspace --all-targets -- -D warnings
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
python tools/media-runtime/provision.py
cargo run -p searchi-cli -- setup --model-dir .searchi/models/embeddinggemma2
cargo run -p searchi-cli -- index --db searchi.sqlite --root /path/to/files
```
On native Linux, unified document/media indexing and hybrid search use:
```sh
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --ffmpeg .searchi/media-runtime/bin/ffmpeg --ffprobe .searchi/media-runtime/bin/ffprobe --cpu-threads 1
./target/release/searchi search --db searchi.sqlite 'the recording about orange soil' --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --cpu-threads 1
```
These are current interfaces, not executed Linux release evidence. Windows uses
`Release/searchi-gemma-worker.exe` and `ffmpeg.exe`/`ffprobe.exe` instead.
`embed` includes media by default and reports failures. Use `--text-only`
explicitly for document-only vectors. `setup` provisions Google assets, not
worker/decoder executables. Installed release CLIs discover decoders under
`media-runtime/bin` beside the CLI; explicit paired paths or
`SEARCHI_FFMPEG_PATH`/`SEARCHI_FFPROBE_PATH` override discovery. Search never
silently downloads missing helpers. CLI invocations load their own worker;
the desktop retains a prewarmed worker for interactive queries.
## Search performance
Normal searches show result counts, measured search time and time until results
are displayed. Enable **Performance details** in desktop settings to
inspect UI scheduling, native queue/lock time, keyword retrieval, actual Google
query inference, ANN/cache/filter work and ranking. Export the current trace as
local JSON for comparison; tracing sends no telemetry and excludes search text
and file paths. Nested stages and parallel retrieval lanes overlap.
For headless diagnostics, append `--performance --json` for basic metrics or
`--debug-performance --json` for the detailed response. Plain `--json` retains
the existing results-array format. See [timing boundaries, trace schema and
reproducible measurements](docs/PERFORMANCE.md).
## Development and release
The Rust engine, service, media adapter and CLI live under `crates/`; the Tauri
shell is `apps/desktop`. SQLite, image/jpeg-decoder, USearch/NumKong, and Tauri
provide established storage, decoding, ANN/SIMD and desktop facilities.
Extraction and inference use bounded isolated work with explicit errors.
```sh
cargo test --workspace --locked
cargo fmt --all --check
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo run -p searchi-cli -- --help
```
Desktop development instructions live in `apps/desktop/README.md`.
Agent constraints and workflows live in [AGENTS.md](AGENTS.md) and
[SKILLS.md](SKILLS.md). Use Node.js 24+ for frontend state tests.
The approved visual identity and desktop direction live in [the brand guide](docs/BRAND.md).
See [desktop setup](apps/desktop/README.md), [build/platform constraints](docs/BUILDING.md),
[semantic bounds](docs/SEMANTIC.md), [executed validation](docs/VALIDATION.md),
and [brand direction](docs/BRAND.md). Native visual QA, operating-system builds,
macOS decoder packaging, installers, licensing distribution, signatures,
upgrades and representative production quality remain release gates.
Cross-platform source support does not mean signed macOS installers can be
built on Windows. CI declarations are not evidence of executed platform jobs.
For a headless build, run `cargo build --release --locked -p searchi-cli`.
No Node, Tauri, or WebView is needed. Semantic commands additionally need the
native worker and Google assets; `searchi setup` provisions the model cache.
Platform and CI details live in [the build guide](docs/BUILDING.md).
Native CI builds must pass on all supported systems before release. Cross-platform
source support does not imply signed macOS installers can be built on Windows.
## Release policy
Do not publish until the roadmap's release gates have evidence. Record benchmark
hardware, corpus size, cold/warm state and p95 latency. Signing credentials are
supplied through CI secrets, never source control.
Licensed under [MIT](LICENSE). Repository author: Karl.
Record benchmark hardware, corpus, cold/warm state and p95 latency. Never ship
signing credentials in source control. Licensed under [MIT](LICENSE).
Repository author: Karl.
SKILLS.md
+4 -1
View File
@@ -42,7 +42,7 @@ Use stable Rust and Node.js 24+. Before Windows VM tests or launch:
```powershell
$env:SEARCHI_FORCE_CPU = '1'
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu'
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
```
From the repository root:
@@ -74,6 +74,9 @@ cargo test -p searchi-semantic --test gemma_cpu -- --ignored --nocapture
Record actual outputs, hardware, corpus size, background preparation, first-input
latency and warm latency in the relevant validation report. Unexecuted platform
checks stay open. Compilation alone is not UI interaction verification.
Use the opt-in measured search APIs and docs/PERFORMANCE.md for granular traces.
Keep basic and debug overhead separate; do not sum nested stages or parallel
lanes, or label DOM updates as actual paint. Retain raw measurement samples.
## Commit and hand off
apps/desktop/README.md
+52 -4
View File
@@ -1,11 +1,24 @@
# Searchi desktop
This Tauri 2 shell calls the same Rust core as the CLI and the real Google EmbeddingGemma 2 service. It provides folder indexing, native folder selection, progressive filename/content/semantic search, folder/extension/size/date filters, keyboard navigation, escaped text previews, native open/reveal actions, persisted selected folders/resource settings, and a global shortcut. There are no mocked results or embeddings. Watchers and release installers remain roadmap work.
This Tauri 2 shell calls the same Rust core as the CLI and the real Google EmbeddingGemma 2 service. It provides folder indexing, native folder selection, progressive filename/content/semantic search, folder/extension/size/date/media-duration filters, keyboard navigation, escaped text and scoped image/audio/video previews, native open/reveal actions, persisted selected folders/resource settings, and a global shortcut. There are no mocked results or embeddings. Watchers and release installers remain roadmap work.
## Develop
Install Node.js LTS, Rust stable, and the platform prerequisites documented at https://v2.tauri.app/start/prerequisites/. Windows requires MSVC build tools and WebView2; Linux requires WebKitGTK and system development packages; macOS requires Xcode command line tools.
Install Node.js 24 LTS, Rust stable, and the platform prerequisites documented at https://v2.tauri.app/start/prerequisites/. Windows requires MSVC build tools and WebView2; Linux requires WebKitGTK and system development packages; macOS requires Xcode command line tools.
Before launching from a checkout, build and stage both native helpers from the
repository root. These explicit setup commands access the network and verify
pinned artifacts; they do not install tools globally:
```sh
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
python tools/embeddinggemma2/stage_runtime.py --backend cpu
python tools/media-runtime/provision.py --stage
```
The decoder manifest currently has Windows/Linux x86-64 and ARM64 artifacts;
a vetted macOS decoder package remains a release gate. Build `cpu` in the
Windows VM. Production acceleration requires separate platform/hardware QA.
```sh
cd apps/desktop
npm ci
@@ -25,7 +38,7 @@ The desktop index is stored in the platform app-data directory under `dev.search
On first launch, the native app downloads and verifies the pinned Google model artifacts (310 MB text model plus 555 MB media projector, approximately 865 MB total) into app-managed storage and prepares the packaged embedding worker. No model paths, runtime paths, or terminal setup are required. The UI shows native download bytes, verification/warming states, the actual worker device, and explicit errors with a retry control. The first download requires network access; file extraction, embeddings, and queries stay local.
After a folder is indexed, semantic indexing starts automatically when the model is ready. Existing keyword results stay searchable during setup/indexing. Progress is indeterminate during embedding because the native command does not expose a chunk completion count. Failed automatic builds retry only after a changed index/status revision or an explicit rebuild, avoiding repeated failure loops. Status polls run once per second while setup/indexing/warming is active and stop when idle.
After a folder is indexed, unified document/media semantic indexing starts automatically when the model and packaged media helpers are ready. Images use actual pixels; video frames and full primary-stream audio windows use the Google media projector. Audio/music files also retain artist/title metadata for keyword retrieval. The UI reports unavailable decoder/runtime support and per-file failures while keeping text search available. Existing keyword results stay searchable during setup/indexing. Progress is indeterminate during embedding because the native command does not expose a chunk completion count. Failed automatic builds retry only after a changed index/status revision or an explicit rebuild, avoiding repeated failure loops. Status polls run once per second while setup/indexing/warming is active and stop when idle.
Each query captures one immutable filter snapshot. Keyword results appear first; real hybrid results follow after the query settles. `src/search-state.ts` bounds each lane to one active native request and one latest pending request, discards old revisions, and retries a busy request at most once. Escape clears both pending lanes and their timers; already-running native work finishes without changing cleared results. Hybrid updates preserve the selected file and keyboard focus where possible.
@@ -38,7 +51,7 @@ node --test tests/search-state.test.mjs
For Windows VM UI testing, disable WebView2 GPU rendering as well, before starting Tauri in PowerShell:
```powershell
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu'
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
$env:SEARCHI_FORCE_CPU = '1'
npm.cmd run tauri dev
```
@@ -56,3 +69,38 @@ Visual fidelity remains subject to native screenshot QA. Compilation and state t
## Verification and packaging
`npm run build` checks strict TypeScript and creates production frontend assets. `npm run tauri build -- --no-bundle` compiles the native shell when platform prerequisites are installed. Installer bundling is intentionally disabled until release icons, signing and native platform checks are complete. Native Windows, macOS, and Linux checks must be recorded separately; a frontend build does not prove the native shell works.
## Media preview and quality
The result pane renders images and audio/video controls through Tauri's scoped
asset protocol. Native preparation verifies the selected path belongs to a
current indexed file before exposing it. There is no general filesystem access
or local HTTP server. Players use metadata preload, never autoplay, and seek
matched millisecond timestamps after metadata loads. Selection changes and
Escape pause/unload playback; stale preparation responses cannot replace the
current preview. Unsupported WebView codecs show an Open file fallback.
Opening the default application does not guarantee exact-location navigation.
Preview seeking uses the returned window/frame start, which may differ from the
actual utterance. The [independent media quality report](../../docs/MEDIA-QUALITY.md)
records real file recall, speech alignment failures, music hard negatives and
unmet no-answer handling. Genuine multimodal support is implemented; production
quality and native playback/visual interaction QA remain open gates.
## Performance details
Normal results show the actual native search duration and latest result-display
latency from input. The **Performance details** checkbox in Settings is opt-in
and captured per request. Its expandable panel separates input debounce,
intentional semantic delay, pending-lane queue, failed attempts and bounded retry
wait, invocation time, native stage hierarchy and synchronous result DOM updates.
First response and hybrid-final latency are distinct; keyword-only results do not
claim semantic completion. Native spans use a command-relative clock and nested
or parallel spans must not be summed. Invocation minus native total is explicitly
an estimated transport/serialization/scheduling residual, not an exact IPC cost.
DOM update timings do not claim actual screen paint or deferred media loading.
The local clipboard report excludes queries, paths, filenames, snippets and error
strings. It captures available model/device/thread/runtime context without
guessing missing values. Histories are bounded and dropped counts remain visible.
Clipboard failure is explicit; no diagnostic telemetry is uploaded.
apps/desktop/src-tauri/Cargo.lock
Generated
+101
View File
@@ -412,6 +412,12 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "byteorder-lite"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
[[package]]
name = "bytes"
version = "1.12.1"
@@ -1845,6 +1851,12 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "http-range"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573"
[[package]]
name = "httparse"
version = "1.10.1"
@@ -2063,6 +2075,32 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "image"
version = "0.25.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
dependencies = [
"bytemuck",
"byteorder-lite",
"image-webp",
"moxcms",
"num-traits",
"png 0.18.1",
"zune-core",
"zune-jpeg",
]
[[package]]
name = "image-webp"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
dependencies = [
"byteorder-lite",
"quick-error",
]
[[package]]
name = "indexmap"
version = "1.9.3"
@@ -2257,6 +2295,12 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "jpeg-decoder"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
[[package]]
name = "js-sys"
version = "0.3.106"
@@ -2522,6 +2566,16 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "moxcms"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
dependencies = [
"num-traits",
"pxfm",
]
[[package]]
name = "muda"
version = "0.20.0"
@@ -3157,6 +3211,18 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "pxfm"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
[[package]]
name = "quick-error"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
[[package]]
name = "quick-xml"
version = "0.42.0"
@@ -3652,8 +3718,11 @@ dependencies = [
name = "searchi-desktop"
version = "0.1.0"
dependencies = [
"anyhow",
"image",
"searchi-assets",
"searchi-core",
"searchi-media",
"searchi-service",
"serde",
"serde_json",
@@ -3665,11 +3734,26 @@ dependencies = [
"tempfile",
]
[[package]]
name = "searchi-media"
version = "0.1.0"
dependencies = [
"anyhow",
"image",
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
]
[[package]]
name = "searchi-semantic"
version = "0.1.0"
dependencies = [
"anyhow",
"base64 0.22.1",
"image",
"jpeg-decoder",
"serde",
"serde_json",
"sha2 0.10.9",
@@ -3683,6 +3767,7 @@ dependencies = [
"fs2",
"rusqlite",
"searchi-core",
"searchi-media",
"searchi-semantic",
"serde",
"sha2 0.10.9",
@@ -4218,6 +4303,7 @@ dependencies = [
"gtk",
"heck 0.5.0",
"http",
"http-range",
"jni",
"libc",
"log",
@@ -5993,6 +6079,21 @@ dependencies = [
"simd-adler32",
]
[[package]]
name = "zune-core"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b"
[[package]]
name = "zune-jpeg"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
dependencies = [
"zune-core",
]
[[package]]
name = "zvariant"
version = "5.15.0"
apps/desktop/src-tauri/Cargo.toml
+4 -1
View File
@@ -14,7 +14,8 @@ custom-protocol = ["tauri/custom-protocol"]
tauri-build = { version = "2", features = [] }
[dependencies]
tauri = { version = "2", features = [] }
anyhow = "1"
tauri = { version = "2", features = ["protocol-asset"] }
tauri-plugin-dialog = "2"
tauri-plugin-opener = "2"
tauri-plugin-global-shortcut = "2"
@@ -24,3 +25,5 @@ tempfile = "3"
searchi-core = { path = "../../../crates/searchi-core" }
searchi-service = { path = "../../../crates/searchi-service" }
searchi-assets = { path = "../../../crates/searchi-assets" }
searchi-media = { path = "../../../crates/searchi-media" }
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp", "bmp"] }
apps/desktop/src-tauri/resources/media-runtime/licenses/LICENSE.txt
+165
View File
@@ -0,0 +1,165 @@
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
apps/desktop/src-tauri/resources/media-runtime/licenses/ffmpeg-n9.0.2-27-g7e93014c5b-win64-lgpl-9.0/LICENSE.txt
+165
View File
@@ -0,0 +1,165 @@
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
apps/desktop/src-tauri/resources/media-runtime/manifest.json
+11
View File
@@ -0,0 +1,11 @@
{
"provider": "BtbN/FFmpeg-Builds",
"release": "autobuild-2026-10-12-13-12",
"version": "9.0.2-27-g7e93014c5b",
"platforms": {
"windows-x86_64": {"archive": "ffmpeg-n9.0.2-27-g7e93014c5b-win64-lgpl-9.0.zip", "bytes": 171476229, "sha256": "104a019fe34c0747b208bde4842a0bac8b70a307cc2bb33aa4648fc15ddb2e39"},
"windows-aarch64": {"archive": "ffmpeg-n9.0.2-27-g7e93014c5b-winarm64-lgpl-9.0.zip", "bytes": 118006941, "sha256": "68d3bac585b80b6f17e459eb4a203ed1f01b7635d4c611f1685de1ae65ca0987"},
"linux-x86_64": {"archive": "ffmpeg-n9.0.2-27-g7e93014c5b-linux64-lgpl-9.0.tar.xz", "bytes": 137901812, "sha256": "12445b38a4700ba1ae7c9c7e57062f602c927dc4187555bfadc9f3e2f75f05b0"},
"linux-aarch64": {"archive": "ffmpeg-n9.0.2-27-g7e93014c5b-linuxarm64-lgpl-9.0.tar.xz", "bytes": 117008524, "sha256": "c9160ae8440c17c016fb2f6b3fcce4bb9d633a15845e908b2303948a7724f227"}
}
}
apps/desktop/src-tauri/src/main.rs
+197 -3
View File
@@ -4,9 +4,11 @@ mod model_download;
mod semantic_commands;
mod settings;
use searchi_core::metrics::{MetricsMode, SearchMetrics, SearchResponse, TraceCollector};
use searchi_core::{IndexReport, IndexStats, SearchIndex, SearchOptions, SearchResult};
use semantic_commands::{
hybrid_search, index_embeddings, semantic_status, setup_model, SemanticState,
hybrid_search, hybrid_search_measured, index_embeddings, semantic_status, setup_model,
SemanticState,
};
use serde::{Deserialize, Serialize};
use settings::{Settings, SettingsStore};
@@ -31,6 +33,46 @@ struct AppState {
semantic: Arc<SemanticState>,
}
#[derive(Serialize)]
struct MeasuredFailure {
message: String,
metrics: SearchMetrics,
}
impl MeasuredFailure {
fn new(message: impl Into<String>, trace: TraceCollector) -> Self {
Self {
message: message.into(),
metrics: trace.finish(0),
}
}
fn task(message: impl Into<String>, started: std::time::Instant) -> Self {
Self {
message: message.into(),
metrics: SearchMetrics {
total_ms: started.elapsed().as_secs_f64() * 1000.0,
result_count: 0,
trace: None,
dropped_spans: 1,
},
}
}
}
fn measured_outcome(
result: Result<Vec<SearchResult>, String>,
trace: TraceCollector,
) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
match result {
Ok(results) => Ok(SearchResponse {
metrics: trace.finish(results.len()),
results,
}),
Err(message) => Err(MeasuredFailure::new(message, trace)),
}
}
#[derive(Serialize)]
struct DesktopStatus {
settings: Settings,
@@ -78,6 +120,8 @@ struct Filters {
extension: Option<String>,
min_size: Option<u64>,
max_size: Option<u64>,
min_duration_ms: Option<u32>,
max_duration_ms: Option<u32>,
modified_after_unix_ms: Option<i64>,
modified_before_unix_ms: Option<i64>,
}
@@ -89,6 +133,8 @@ impl From<Filters> for SearchOptions {
extension: value.extension,
min_size: value.min_size,
max_size: value.max_size,
min_duration_ms: value.min_duration_ms,
max_duration_ms: value.max_duration_ms,
modified_after_unix_ms: value.modified_after_unix_ms,
modified_before_unix_ms: value.modified_before_unix_ms,
}
@@ -114,6 +160,46 @@ async fn search(
.map_err(|e| e.to_string())?
}
/// Measured command preserves the legacy array-returning API for integrations.
#[tauri::command]
async fn search_measured(
query: String,
filters: Option<Filters>,
debug_performance: bool,
state: State<'_, AppState>,
) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
let started = std::time::Instant::now();
let mut trace = TraceCollector::new(if debug_performance {
MetricsMode::Debug
} else {
MetricsMode::Basic
});
let queue = trace.begin("desktop.worker_queue", None);
let lexical = state.lexical.clone();
tauri::async_runtime::spawn_blocking(move || {
trace.end(queue, Default::default());
let result = (|| {
let index = trace.timed("desktop.lexical_lock", None, |_, _| {
lexical
.try_lock()
.map_err(|_| "Keyword worker is busy; try the latest query again.".to_owned())
})?;
index
.search_with_options_traced(
&query,
50,
&filters.unwrap_or_default().into(),
&mut trace,
None,
)
.map_err(|error| error.to_string())
})();
measured_outcome(result, trace)
})
.await
.map_err(|error| MeasuredFailure::task(error.to_string(), started))?
}
#[tauri::command]
async fn pick_folder(app: tauri::AppHandle) -> Result<Option<String>, String> {
tauri::async_runtime::spawn_blocking(move || {
@@ -148,6 +234,68 @@ fn validated_result(index: &SearchIndex, path: &str) -> Result<PathBuf, String>
Ok(canonical)
}
#[derive(Serialize)]
struct MediaPreview {
path: String,
kind: &'static str,
}
// The asset protocol starts with an empty scope. Only a validated indexed
// media file is granted access; the webview receives no general filesystem API.
#[tauri::command]
async fn prepare_media_preview(
path: String,
app: tauri::AppHandle,
state: State<'_, AppState>,
) -> Result<MediaPreview, String> {
let database = state.database.clone();
tauri::async_runtime::spawn_blocking(move || {
let index = SearchIndex::open(database).map_err(|error| error.to_string())?;
let canonical = validated_result(&index, &path)?;
let kind = match searchi_media::classify(&canonical) {
Some(searchi_media::MediaKind::Image) => "image",
Some(searchi_media::MediaKind::Audio) => "audio",
Some(searchi_media::MediaKind::Video) => "video",
None => return Err("This file is not a supported media preview.".into()),
};
if kind == "image" {
let bytes = std::fs::metadata(&canonical)
.map_err(|error| error.to_string())?
.len();
if bytes > 64 * 1024 * 1024 {
return Err("Image exceeds the 64 MiB preview limit.".into());
}
let (width, height) = image::ImageReader::open(&canonical)
.map_err(|error| error.to_string())?
.with_guessed_format()
.map_err(|error| error.to_string())?
.into_dimensions()
.map_err(|error| format!("Image preview is unavailable: {error}"))?;
let jpeg = canonical
.extension()
.and_then(|value| value.to_str())
.is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "jpg" | "jpeg"));
let max_pixels = if jpeg {
64 * 1024 * 1024
} else {
16 * 1024 * 1024
};
if width == 0 || height == 0 || u64::from(width) * u64::from(height) > max_pixels {
return Err("Image exceeds the supported preview dimensions.".into());
}
}
app.asset_protocol_scope()
.allow_file(&canonical)
.map_err(|error| error.to_string())?;
Ok(MediaPreview {
path: canonical.to_string_lossy().into_owned(),
kind,
})
})
.await
.map_err(|error| error.to_string())?
}
#[tauri::command]
async fn open_result(
path: String,
@@ -279,7 +427,24 @@ fn main() {
app.path().resource_dir().unwrap_or_else(|_| directory.clone()).join(worker_name)
}
});
let semantic = Arc::new(SemanticState::new(model_directory, worker));
let media_tools = (|| -> Result<searchi_media::FfmpegTools, String> {
if let Some(tools) = searchi_media::FfmpegTools::from_environment()
.map_err(|error| error.to_string())? {
return Ok(tools);
}
let development_media = development_root.join(".searchi/media-runtime/bin");
let media_directory = if cfg!(debug_assertions) && development_media.is_dir() {
development_media
} else {
app.path().resource_dir().map_err(|error| error.to_string())?.join("media-runtime/bin")
};
let ffmpeg = media_directory.join(if cfg!(windows) { "ffmpeg.exe" } else { "ffmpeg" });
let ffprobe = media_directory.join(if cfg!(windows) { "ffprobe.exe" } else { "ffprobe" });
searchi_media::FfmpegTools::new(ffmpeg, ffprobe).map_err(|_| {
"The media decoder is missing from this installation. Repair or reinstall searchi.".into()
})
})();
let semantic = Arc::new(SemanticState::new(model_directory, worker, media_tools));
semantic.begin_automatic_setup(database.clone(), settings.value.cpu_threads);
#[cfg(target_os = "macos")]
let shortcut = "Shift+Super+Space";
@@ -307,16 +472,19 @@ fn main() {
})
.invoke_handler(tauri::generate_handler![
search,
search_measured,
stats,
index_folder,
pick_folder,
open_result,
prepare_media_preview,
desktop_status,
set_cpu_threads,
semantic_status,
setup_model,
index_embeddings,
hybrid_search
hybrid_search,
hybrid_search_measured
])
.run(tauri::generate_context!())
.expect("Searchi desktop could not start");
@@ -326,6 +494,32 @@ fn main() {
mod tests {
use super::*;
#[test]
fn measured_failure_retains_closed_trace_without_inventing_results() {
let mut trace = TraceCollector::new(MetricsMode::Debug);
let parent = trace.begin("desktop.request", None);
trace.begin("desktop.lexical_lock", parent);
let failed = measured_outcome(Err("Keyword worker is busy".into()), trace)
.err()
.unwrap();
assert_eq!(failed.message, "Keyword worker is busy");
assert_eq!(failed.metrics.result_count, 0);
let spans = failed.metrics.trace.unwrap();
assert_eq!(spans.len(), 2);
assert_eq!(spans[1].parent, Some(0));
assert!(spans[1].start_ms + spans[1].duration_ms <= failed.metrics.total_ms + 0.001);
}
#[test]
fn measured_empty_success_is_distinct_from_failure_and_basic_has_no_trace() {
let response = measured_outcome(Ok(Vec::new()), TraceCollector::new(MetricsMode::Basic))
.ok()
.unwrap();
assert!(response.results.is_empty());
assert_eq!(response.metrics.result_count, 0);
assert!(response.metrics.trace.is_none());
}
#[test]
fn indexing_gate_is_released_when_worker_unwinds() {
let indexing = Arc::new(AtomicBool::new(true));
apps/desktop/src-tauri/src/semantic_commands.rs
+134 -4
View File
@@ -1,5 +1,6 @@
//! Desktop-owned orchestration: managed model setup and persistent hybrid queries.
use crate::{model_download, AppState, Filters};
use crate::{measured_outcome, model_download, AppState, Filters, MeasuredFailure};
use searchi_core::metrics::{MetricsMode, SearchResponse, TraceCollector};
use searchi_core::SearchResult;
use searchi_service::{EmbeddingReport, HybridSearch};
use serde::Serialize;
@@ -24,10 +25,20 @@ pub struct SemanticState {
downloaded_bytes: AtomicU64,
error: Mutex<Option<String>>,
service: Mutex<Option<HybridSearch>>,
media_tools: Option<searchi_media::FfmpegTools>,
media_setup_error: Option<String>,
}
impl SemanticState {
pub fn new(model_directory: PathBuf, worker: PathBuf) -> Self {
pub fn new(
model_directory: PathBuf,
worker: PathBuf,
media_tools: Result<searchi_media::FfmpegTools, String>,
) -> Self {
let (media_tools, media_setup_error) = match media_tools {
Ok(tools) => (Some(tools), None),
Err(error) => (None, Some(error)),
};
Self {
model_directory,
worker,
@@ -40,6 +51,8 @@ impl SemanticState {
downloaded_bytes: AtomicU64::new(0),
error: Mutex::new(None),
service: Mutex::new(None),
media_tools,
media_setup_error,
}
}
@@ -118,6 +131,9 @@ pub struct SemanticStatus {
pending: usize,
warming: bool,
worker_info: Option<searchi_service::WorkerInfo>,
media_tools_ready: bool,
media_setup_error: Option<String>,
media_failures: Vec<searchi_service::MediaFailure>,
}
struct WorkGuard<'a>(&'a AtomicBool);
@@ -158,6 +174,9 @@ pub fn semantic_status(state: State<'_, AppState>) -> Result<SemanticStatus, Str
.try_lock()
.ok()
.and_then(|service| service.as_ref().and_then(HybridSearch::worker_info)),
media_tools_ready: semantic.media_tools.is_some(),
media_setup_error: semantic.media_setup_error.clone(),
media_failures: index.failures,
})
}
@@ -217,13 +236,42 @@ pub async fn index_embeddings(state: State<'_, AppState>) -> Result<EmbeddingRep
semantic.record_error(None);
// Indexing uses its own isolated worker; interactive queries keep their own session.
let result = HybridSearch::open(
database,
&database,
&semantic.model_directory,
&semantic.worker,
threads,
)
.and_then(|mut service| service.index_embeddings())
.and_then(|mut service| {
let tools = semantic.media_tools.as_ref().ok_or_else(|| {
anyhow::anyhow!("The media decoder is missing from this installation. Repair or reinstall searchi.")
})?;
service.index_embeddings_with_media(tools, &searchi_media::MediaLimits::default())
})
.map_err(|error| error.to_string());
// Prepare the newly activated graph before advertising semantic readiness.
// Keyword search stays available while the cached query session catches up.
if result.is_ok() {
semantic.warmed.store(false, Ordering::Release);
semantic.warming.store(true, Ordering::Release);
let _warming_guard = WorkGuard(&semantic.warming);
let preparation = (|| {
let mut cached = semantic
.service
.lock()
.map_err(|_| "Semantic session is unavailable.".to_string())?;
let service = cached
.as_mut()
.ok_or_else(|| "Semantic query session has not been prepared.".to_string())?;
service.warmup().map_err(|error| error.to_string())
})();
semantic.warmed.store(preparation.is_ok(), Ordering::Release);
if let Err(error) = preparation {
semantic.record_error(Some(format!(
"The index was saved, but search preparation failed: {error}"
)));
return result;
}
}
semantic.record_error(result.as_ref().err().cloned());
result
})
@@ -283,3 +331,85 @@ pub async fn hybrid_search(
.await
.map_err(|error| error.to_string())?
}
#[tauri::command]
pub async fn hybrid_search_measured(
query: String,
filters: Option<Filters>,
debug_performance: bool,
state: State<'_, AppState>,
) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
let started = std::time::Instant::now();
let mut trace = TraceCollector::new(if debug_performance {
MetricsMode::Debug
} else {
MetricsMode::Basic
});
let semantic = state.semantic.clone();
if !semantic.warmed.load(Ordering::Acquire) {
return Err(MeasuredFailure::new(
"Semantic search is preparing in the background; keyword results are available.",
trace,
));
}
let threads = trace.timed("desktop.settings_lock", None, |_, _| {
state
.settings
.lock()
.map(|settings| settings.value.cpu_threads)
.map_err(|_| "Settings are unavailable.")
});
let threads = match threads {
Ok(threads) => threads,
Err(error) => return Err(MeasuredFailure::new(error, trace)),
};
if semantic
.querying
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return Err(MeasuredFailure::new(
"Semantic worker is busy; keyword results remain available.",
trace,
));
}
let database = state.database.clone();
let queue = trace.begin("desktop.worker_queue", None);
tauri::async_runtime::spawn_blocking(move || {
let _guard = WorkGuard(&semantic.querying);
trace.end(queue, Default::default());
let result = (|| {
let mut cached = trace.timed("desktop.semantic_lock", None, |_, _| {
semantic
.service
.lock()
.map_err(|_| "Semantic session is unavailable.")
})?;
if cached.is_none() {
*cached = Some(trace.timed("desktop.session_initialize", None, |_, _| {
HybridSearch::open(
database,
&semantic.model_directory,
&semantic.worker,
threads,
)
.map_err(|error| error.to_string())
})?);
}
cached
.as_mut()
.ok_or("Semantic session is unavailable.")?
.search_traced(
&query,
50,
&filters.unwrap_or_default().into(),
&mut trace,
None,
)
.map_err(|error| error.to_string())
})();
measured_outcome(result, trace)
})
.await
.map_err(|error| MeasuredFailure::task(error.to_string(), started))?
}
apps/desktop/src-tauri/tauri.conf.json
+2 -1
View File
@@ -20,7 +20,8 @@
}
],
"security": {
"csp": "default-src 'self'; connect-src ipc: http://ipc.localhost; style-src 'self' 'unsafe-inline'; img-src 'self' data:"
"csp": "default-src 'self'; connect-src ipc: http://ipc.localhost; style-src 'self' 'unsafe-inline'; img-src 'self' data: asset: http://asset.localhost https://asset.localhost; media-src 'self' asset: http://asset.localhost https://asset.localhost",
"assetProtocol": { "enable": true, "scope": [] }
}
},
"bundle": {
apps/desktop/src/icons.ts
+9
View File
@@ -14,6 +14,10 @@ import arrowDown from "@fluentui/svg-icons/icons/arrow_down_16_regular.svg?raw";
import open from "@fluentui/svg-icons/icons/open_20_regular.svg?raw";
import chevron from "@fluentui/svg-icons/icons/chevron_down_16_regular.svg?raw";
import sparkles from "@fluentui/svg-icons/icons/sparkle_20_regular.svg?raw";
import image from "@fluentui/svg-icons/icons/image_24_regular.svg?raw";
import video from "@fluentui/svg-icons/icons/video_24_regular.svg?raw";
import audio from "@fluentui/svg-icons/icons/music_note_2_24_regular.svg?raw";
import { mediaKind } from "./media-display";
const assets = {
search,
@@ -31,11 +35,16 @@ const assets = {
open,
chevron,
sparkles,
image,
video,
audio,
};
export function icon(name: keyof typeof assets) {
return `<span class="icon" aria-hidden="true">${assets[name]}</span>`;
}
export function fileIcon(filename: string) {
const media = mediaKind(filename);
if (media) return { html: icon(media), type: media };
const extension = filename.split(".").pop()?.toLowerCase();
if (extension === "pdf") return { html: icon("pdf"), type: "pdf" };
if (
apps/desktop/src/main.ts
+333 -28
View File
@@ -1,4 +1,4 @@
import { invoke, isTauri } from "@tauri-apps/api/core";
import { invoke, isTauri, convertFileSrc } from "@tauri-apps/api/core";
import { listen } from "@tauri-apps/api/event";
import {
AutomaticIndexGate,
@@ -7,8 +7,22 @@ import {
} from "./search-state";
import type { SemanticPhase } from "./search-state";
import { fileIcon, icon } from "./icons";
import {
SearchMeasurement,
measuredFailure,
spanDepth,
briefMs,
} from "./search-performance";
import type { MeasuredResponse, SearchLane } from "./search-performance";
import "@fontsource-variable/inter/wght.css";
import "./style.css";
import {
mediaKind,
matchLabel,
sourceLabel,
seekSeconds,
PreviewRevision,
} from "./media-display";
interface SearchResult {
path: string;
@@ -16,7 +30,7 @@ interface SearchResult {
snippet: string;
score: number;
match_kind: string;
source?: { kind: string; number: number };
source?: { kind: string; number: number; matched_query?: string };
extraction_error?: string;
}
interface IndexReport {
@@ -55,11 +69,16 @@ interface SemanticStatus {
stale: boolean;
pending: number;
warming: boolean;
media_tools_ready?: boolean;
media_setup_error?: string | null;
media_failures?: { path: string; message: string }[];
worker_info?: {
model?: string;
device?: string;
runtime?: string;
threads?: number;
dimensions?: number;
media_supported?: boolean;
} | null;
}
interface SearchRequest {
@@ -67,15 +86,16 @@ interface SearchRequest {
query: string;
filters: Record<string, string | number | null>;
started: number;
measurement: SearchMeasurement;
}
document.querySelector<HTMLDivElement>("#app")!.innerHTML = `
<header><div class="brand"><img class="brand-symbol" src="/searchi-symbol.png" alt=""><span class="wordmark">searchi</span></div><span id="index-status" class="index-status">Loading your index…</span><div class="header-actions"><button id="add-folder" class="quiet-button" type="button">${icon("folderAdd")}<span>Add folder</span></button><details id="filters" class="popover"><summary>${icon("filter")}<span>All files</span>${icon("chevron")}</summary><div class="popover-panel"><h2>Filter your search</h2><div class="filters-grid"><label>Within folder<input id="filter-folder" placeholder="Absolute folder path"></label><label>Extension<input id="filter-extension" placeholder="pdf, md, txt…"></label><label>Minimum size (bytes)<input id="filter-min" type="number" min="0" step="1"></label><label>Maximum size (bytes)<input id="filter-max" type="number" min="0" step="1"></label><label>Modified from<input id="filter-after" type="date"></label><label>Modified through<input id="filter-before" type="date"></label></div></div></details><details id="settings" class="popover"><summary class="icon-button" aria-label="Settings" title="Settings">${icon("settings")}</summary><div class="popover-panel"><h2>Folders & resources</h2><p id="shortcut-status" class="hint"></p><label class="resource-label" for="cpu-threads">CPU thread budget</label><select id="cpu-threads">${Array.from({ length: 8 }, (_, i) => `<option value="${i + 1}">${i + 1} ${i ? "threads" : "thread (default)"}</option>`).join("")}</select><p class="hint">Search automatically uses supported acceleration with CPU fallback.</p><ul id="roots" aria-label="Selected folders"></ul></div></details></div></header>
<header><div class="brand"><img class="brand-symbol" src="/searchi-symbol.png" alt=""><span class="wordmark">searchi</span></div><span id="index-status" class="index-status">Loading your index…</span><div class="header-actions"><button id="add-folder" class="quiet-button" type="button">${icon("folderAdd")}<span>Add folder</span></button><details id="filters" class="popover"><summary>${icon("filter")}<span>All files</span>${icon("chevron")}</summary><div class="popover-panel"><h2>Filter your search</h2><div class="filters-grid"><label>Within folder<input id="filter-folder" placeholder="Absolute folder path"></label><label>Extension<input id="filter-extension" placeholder="pdf, md, txt…"></label><label>Minimum size (bytes)<input id="filter-min" type="number" min="0" step="1"></label><label>Maximum size (bytes)<input id="filter-max" type="number" min="0" step="1"></label><label>Minimum duration (seconds)<input id="filter-duration-min" type="number" min="0" step="0.1"></label><label>Maximum duration (seconds)<input id="filter-duration-max" type="number" min="0" step="0.1"></label><label>Modified from<input id="filter-after" type="date"></label><label>Modified through<input id="filter-before" type="date"></label></div></div></details><details id="settings" class="popover"><summary class="icon-button" aria-label="Settings" title="Settings">${icon("settings")}</summary><div class="popover-panel"><h2>Folders & resources</h2><p id="shortcut-status" class="hint"></p><label class="resource-label" for="cpu-threads">CPU thread budget</label><select id="cpu-threads">${Array.from({ length: 8 }, (_, i) => `<option value="${i + 1}">${i + 1} ${i ? "threads" : "thread (default)"}</option>`).join("")}</select><p class="hint">Search automatically uses supported acceleration with CPU fallback.</p><ul id="roots" aria-label="Selected folders"></ul><label class="performance-toggle"><input id="performance-toggle" type="checkbox">Performance details</label><p class="hint">Local timing diagnostics for new searches. Queries and file paths are excluded from copied reports.</p></div></details></div></header>
<main>
<form id="search-form" role="search"><label class="sr-only" for="query">Search your files</label>${icon("search")}<input id="query" type="search" placeholder="Search a filename, phrase, or what you remember…" autocomplete="off" spellcheck="false" maxlength="1000"><kbd id="focus-shortcut">Ctrl K</kbd><button id="clear-query" class="icon-button" type="button" aria-label="Clear search" title="Clear search">${icon("dismiss")}</button></form>
<details id="semantic-panel"><summary>${icon("sparkles")}<span id="semantic-status" role="status" aria-live="polite">Preparing local semantic search…</span><span id="semantic-device" class="kind">Local model</span>${icon("chevron")}</summary><div class="semantic-content"><strong>Google EmbeddingGemma 2</strong><progress id="semantic-progress" aria-label="Local model setup progress"></progress><p id="semantic-detail" class="hint">The verified local model downloads automatically on first launch (865 MB). Your files stay on this device.</p><div class="semantic-actions"><button id="retry-model" type="button" hidden>Retry model setup</button><button id="rebuild-semantic" type="button" hidden>Rebuild semantic index</button></div></div></details>
<form id="folder-form" hidden><label for="folder">Folder to index</label><div class="folder-row"><input id="folder" placeholder="Absolute folder path" required><button id="browse-folder" type="button">Browse…</button><button type="submit">Index folder</button><button id="cancel-folder" type="button">Cancel</button></div><p class="hint">Only choose folders you want searched. Large folders may take a while.</p></form>
<p id="notice" role="status" aria-live="polite"></p><div class="workspace"><section class="results-pane" aria-label="Search results"><p id="result-summary" class="eyebrow">READY WHEN YOU ARE</p><ul id="results" aria-label="Matching files"></ul><div id="empty">${icon("search")}<h2>Find what matters. Locally.</h2><p>Index a folder, then search a filename, phrase, or what you remember.</p></div></section><aside id="preview" aria-label="Result preview" hidden><div class="preview-header"><span id="preview-icon" class="file-icon"></span><div class="preview-heading"><h2 id="preview-title"></h2><p id="preview-path" class="path"></p></div></div><div class="preview-toolbar"><span id="preview-kind"></span><div class="preview-actions"><button id="open-result" type="button">${icon("open")}<span>Open file</span></button><button id="reveal-result" type="button" title="Show in folder" aria-label="Show in folder">${icon("folder")}</button></div></div><div class="passage"><p class="eyebrow">EXTRACTED PASSAGE</p><pre id="preview-text"></pre></div></aside></div>
<p id="notice" role="status" aria-live="polite"></p><details id="performance-panel" hidden><summary>Performance details</summary><div id="performance-detail"></div><button id="copy-performance" type="button">Copy local timing report</button><span id="copy-performance-status" role="status"></span></details><div class="workspace"><section class="results-pane" aria-label="Search results"><p id="result-summary" class="eyebrow">READY WHEN YOU ARE</p><ul id="results" aria-label="Matching files"></ul><div id="empty">${icon("search")}<h2>Find what matters. Locally.</h2><p>Index a folder, then search a filename, phrase, or what you remember.</p></div></section><aside id="preview" aria-label="Result preview" hidden><div class="preview-header"><span id="preview-icon" class="file-icon"></span><div class="preview-heading"><h2 id="preview-title"></h2><p id="preview-path" class="path"></p></div></div><div class="preview-toolbar"><span id="preview-kind"></span><div class="preview-actions"><button id="open-result" type="button">${icon("open")}<span>Open file</span></button><button id="reveal-result" type="button" title="Show in folder" aria-label="Show in folder">${icon("folder")}</button></div></div><div id="media-preview" class="media-preview" hidden></div><p id="media-preview-status" class="hint" role="status" hidden></p><div class="passage"><p id="passage-label" class="eyebrow">EXTRACTED PASSAGE</p><pre id="preview-text"></pre></div></aside></div>
</main><footer><span class="footer-privacy">${icon("lock")}<span>Private search. Everything stays on your device.</span><span id="privacy" class="privacy">Local</span></span><span class="keyboard-hints"><kbd>${icon("arrowUp")}</kbd><kbd>${icon("arrowDown")}</kbd><span>Navigate</span><kbd>Enter</kbd><span>Open</span><kbd>Esc</kbd><span>Clear</span></span></footer>`;
const element = <T extends HTMLElement>(id: string) =>
@@ -83,6 +103,12 @@ const element = <T extends HTMLElement>(id: string) =>
const query = element<HTMLInputElement>("query");
const list = element<HTMLUListElement>("results");
const notice = element<HTMLParagraphElement>("notice");
try {
element<HTMLInputElement>("performance-toggle").checked =
localStorage.getItem("searchi-performance-details") === "true";
} catch {
/* Diagnostics remain opt-in when storage is unavailable. */
}
let results: SearchResult[] = [];
let selected = -1;
let requestId = 0;
@@ -99,6 +125,89 @@ let folderRevision = 0;
let localSemanticError: string | null = null;
let querySemanticFailed = false;
let previewPath: string | null = null;
const previewRevision = new PreviewRevision();
interface PreviewRequest {
revision: number;
result: SearchResult;
}
function unloadMedia() {
element("media-preview")
.querySelectorAll<HTMLMediaElement>("audio,video")
.forEach((player) => {
player.pause();
player.removeAttribute("src");
player.load();
});
element("media-preview").replaceChildren();
element("media-preview").hidden = true;
element("media-preview-status").hidden = true;
}
function clearMedia() {
previewRevision.clear();
mediaLane.clear();
unloadMedia();
}
const mediaLane = new LatestLane<PreviewRequest>(
async (request) => {
const prepared = await invoke<{
path: string;
kind: "image" | "audio" | "video";
}>("prepare_media_preview", { path: request.result.path });
if (!previewRevision.accepts(request.revision)) return;
const host = element("media-preview");
const status = element("media-preview-status");
const failed = () => {
if (!previewRevision.accepts(request.revision)) return;
status.hidden = false;
status.textContent =
"This format cannot be previewed here. Use Open file to view it in your default application.";
};
if (prepared.kind === "image") {
const image = document.createElement("img");
image.alt = request.result.filename;
image.addEventListener("error", failed, { once: true });
image.src = convertFileSrc(prepared.path);
host.append(image);
} else {
const player = document.createElement(prepared.kind);
player.controls = true;
player.preload = "metadata";
player.addEventListener("error", failed, { once: true });
player.addEventListener(
"loadedmetadata",
() => {
if (!previewRevision.accepts(request.revision)) return;
const target = seekSeconds(request.result.source);
if (target !== undefined) {
if (!Number.isFinite(player.duration) || target > player.duration) {
status.hidden = false;
status.textContent =
"The matched timestamp is unavailable in this preview. Use Open file to check the recording.";
} else {
try {
player.currentTime = target;
} catch {
failed();
}
}
}
},
{ once: true },
);
player.src = convertFileSrc(prepared.path);
host.append(player);
}
host.hidden = false;
status.hidden = true;
},
(request) => previewRevision.accepts(request.revision),
(request, error) => {
if (!previewRevision.accepts(request.revision)) return;
element("media-preview-status").hidden = false;
element("media-preview-status").textContent =
`Preview unavailable: ${String(error)}. Use Open file.`;
},
);
let displayedRequestId: number | null = null;
let expandedError: string | null = null;
const reducedMotion = matchMedia("(prefers-reduced-motion: reduce)");
@@ -168,7 +277,7 @@ async function buildSemanticIndex() {
"index_embeddings",
);
element("semantic-status").textContent =
`Semantic search indexed ${report.chunks.toLocaleString()} passages.`;
`Semantic search indexed ${report.chunks.toLocaleString()} content segments.`;
} catch (error) {
localSemanticError = String(error);
} finally {
@@ -229,7 +338,7 @@ async function refreshSemanticStatus() {
message =
"Building your semantic index… Keyword search remains available.";
else if (status.ready && status.chunks > 0)
message = `Semantic search ready · ${status.chunks.toLocaleString()} passages${status.stale ? " · Updates pending" : ""}`;
message = `Semantic search ready · ${status.chunks.toLocaleString()} content segments${status.stale ? " · Updates pending" : ""}${status.media_failures?.length ? ` · ${status.media_failures.length} files could not be indexed` : ""}`;
else if (status.ready)
message = "Google model ready. Index a folder to search by meaning.";
else
@@ -240,6 +349,18 @@ async function refreshSemanticStatus() {
element("semantic-detail").textContent = status.worker_info
? `${status.model} · ${status.worker_info.runtime || "Local inference"} · Your files stay on this device.`
: "The verified Google model downloads automatically once (865 MB). After setup, inference runs locally.";
const mediaDetail = status.media_setup_error
? `Media decoder setup needs attention: ${status.media_setup_error}. Text search remains available.`
: status.worker_info && !status.worker_info.media_supported
? "The installed worker needs a media-capable runtime update. Text search remains available."
: status.media_tools_ready && status.worker_info?.media_supported
? "Image, audio, and video encoders are available locally."
: "Preparing local image, audio, and video support.";
element("semantic-detail").textContent += ` ${mediaDetail}`;
if (status.media_failures?.length) {
element("semantic-detail").textContent +=
` ${status.media_failures.length} files could not be indexed. The first issue: ${status.media_failures[0].path}: ${status.media_failures[0].message}. Filename search remains available; rebuild after fixing the file.`;
}
const retry = element<HTMLButtonElement>("retry-model");
retry.hidden = !error || status.ready;
retry.disabled = working;
@@ -266,7 +387,7 @@ async function refreshSemanticStatus() {
canSearchSemantically() &&
latestSearch?.id === requestId
)
hybridLane.submit(latestSearch);
submitSearch(latestSearch, "hybrid");
if (working || (!status.ready && !error)) queueSemanticRefresh();
} catch (error) {
element("semantic-status").textContent =
@@ -349,15 +470,33 @@ function select(index: number, focus = false) {
element("preview-title").textContent = result.filename;
element("preview-path").textContent = result.path;
element("preview-kind").textContent =
`${result.match_kind} match${result.source ? ` · ${result.source.kind} ${result.source.number}` : ""}`;
`${matchLabel(result.match_kind)}${result.source ? ` · ${sourceLabel(result.source)}` : ""}`;
const media = mediaKind(result.filename);
element("passage-label").textContent = media
? "MATCHED CONTENT"
: "EXTRACTED PASSAGE";
element("preview-text").textContent =
result.snippet ||
(result.extraction_error
? `Content could not be extracted: ${result.extraction_error}`
: "No extracted text is available for this file.");
: media
? result.source?.matched_query || "Preview the matched media below."
: "No extracted text is available for this file.");
element<HTMLButtonElement>("open-result").title = result.source
? `Opens this file in your default application. ${result.source.kind} ${result.source.number} is shown here; exact-location opening is not supported by the default application interface.`
? `Open in your default application. The matched location ${sourceLabel(result.source)} is shown in Searchi; default application opening starts at its own location.`
: "Open in your default application";
const selection = previewRevision.select(
JSON.stringify([result.path, result.source?.kind, result.source?.number]),
);
if (selection.changed) {
mediaLane.clear();
unloadMedia();
if (media && isTauri()) {
element("media-preview-status").hidden = false;
element("media-preview-status").textContent = "Preparing media preview…";
mediaLane.submit({ revision: selection.revision, result });
}
}
if (previewPath !== result.path) {
const passage = document.querySelector<HTMLElement>(".passage")!;
passage.scrollTop = 0;
@@ -388,6 +527,7 @@ function render() {
if (!results.length) {
element("preview").hidden = true;
previewPath = null;
clearMedia();
}
results.forEach((result, i) => {
let entry = resultRows.get(result.path);
@@ -419,9 +559,7 @@ function render() {
entry.title.textContent = result.filename;
entry.path.textContent = result.path;
entry.snippet.textContent = result.snippet;
entry.kind.textContent = result.source
? `${result.source.kind} ${result.source.number}`
: result.match_kind;
entry.kind.textContent = `${matchLabel(result.match_kind)}${result.source ? ` · ${sourceLabel(result.source)}` : ""}`;
const asset = fileIcon(result.filename);
if (entry.file.dataset.type !== asset.type) {
entry.file.innerHTML = asset.html;
@@ -443,6 +581,14 @@ function captureFilters() {
throw new Error("File size must be a nonnegative whole number of bytes.");
return value;
};
const duration = (id: string) => {
const text = input(id);
if (!text) return null;
const ms = Math.round(Number(text) * 1000);
if (!Number.isFinite(ms) || ms < 0 || ms > 4_294_967_295)
throw new Error("Enter a valid nonnegative media duration in seconds.");
return ms;
};
// Date controls represent the user's local calendar day, including its end.
const date = (id: string, end: boolean) => {
const text = input(id);
@@ -457,6 +603,8 @@ function captureFilters() {
return {
folder: input("filter-folder") || null,
extension: input("filter-extension") || null,
min_duration_ms: duration("filter-duration-min"),
max_duration_ms: duration("filter-duration-max"),
min_size: size("filter-min"),
max_size: size("filter-max"),
modified_after_unix_ms: date("filter-after", false),
@@ -474,6 +622,7 @@ function applyResults(
(lane === "lexical" && shownLane === "hybrid")
)
return;
const renderStarted = performance.now();
const selectedPath = results[selected]?.path;
const resultsHadFocus = list.contains(document.activeElement);
shownLane = lane;
@@ -484,18 +633,131 @@ function applyResults(
const preserved = results.findIndex((result) => result.path === selectedPath);
if (preserved >= 0) select(preserved, resultsHadFocus);
else if (results.length) select(0, resultsHadFocus);
element("result-summary").textContent =
`${found.length} RESULTS · ${lane === "hybrid" ? "KEYWORD + MEANING" : "KEYWORD"} · ${Math.round(performance.now() - request.started)} MS`;
if (!found.length) refreshEmptySearchState();
request.measurement.displayed(
lane,
renderStarted,
performance.now(),
found.length,
);
const metric = request.measurement.latestNative(lane);
const trace = request.measurement.export();
element("result-summary").textContent =
`${found.length} RESULTS · ${lane === "hybrid" ? "KEYWORD + MEANING" : "KEYWORD"} · NATIVE ${briefMs(metric?.total_ms ?? 0)} MS · ${lane === "hybrid" ? "FINAL" : "FIRST RESPONSE"} ${Math.round(lane === "hybrid" ? (trace.hybrid_final_results_ms ?? 0) : (trace.first_results_ms ?? 0))} MS`;
refreshPerformance(request);
}
function refreshPerformance(request: SearchRequest) {
if (request.id !== requestId) return;
const panel = element<HTMLDetailsElement>("performance-panel");
panel.hidden = !request.measurement.debug;
if (!request.measurement.debug) return;
const body = element("performance-detail");
body.replaceChildren();
const report = request.measurement.export();
const append = (text: string, className = "") => {
const row = document.createElement("p");
row.textContent = text;
row.className = className;
body.append(row);
return row;
};
append(
"Times below use separate UI and native clocks. Nested spans and parallel lanes must not be added together. DOM update measures synchronous changes, not paint or deferred media preview loading.",
"hint",
);
append(
`Input debounce ${report.debounce_ms.toFixed(1)} ms · First displayed response ${report.first_results_ms?.toFixed(1) ?? "pending"} ms · Latest displayed ${report.latest_displayed_ms?.toFixed(1) ?? "pending"} ms · Hybrid final ${report.hybrid_final_results_ms?.toFixed(1) ?? "pending / unavailable"} ms`,
);
for (const [lane, delay] of Object.entries(report.lane_schedule_ms))
append(
`${lane}: scheduled ${delay.toFixed(1)} ms after debounce${lane === "hybrid" ? " (includes intentional 350 ms semantic delay or waiting for readiness)" : ""}`,
);
for (const attempt of report.attempts) {
append(
`${attempt.lane} attempt ${attempt.attempt}${attempt.failed ? " · FAILED" : ""}: pending queue ${attempt.queued_ms.toFixed(1)} ms · retry wait ${attempt.retry_wait_ms.toFixed(1)} ms · invoke ${attempt.invoke_ms.toFixed(1)} ms`,
"performance-attempt",
);
if (!attempt.native) {
append("Native timing unavailable for this failure.", "hint");
continue;
}
append(
`Native command ${attempt.native.total_ms.toFixed(1)} ms · ${attempt.native.result_count} results · estimated transport / serialization / scheduling ${attempt.transport_estimate_ms?.toFixed(1)} ms`,
"hint",
);
if (attempt.native.trace === null)
append("Detailed native trace unavailable for this attempt.", "hint");
const spans = attempt.native.trace ?? [];
spans.forEach((span, index) => {
const counts = Object.entries(span.counters)
.map(([name, value]) =>
name.endsWith("_us")
? `${name.slice(0, -3).replaceAll("_", " ")}: ${(value / 1000).toFixed(3)} ms`
: `${name.replaceAll("_", " ")}: ${value}`,
)
.join(" · ");
const row = append(
`${span.name}: ${span.duration_ms.toFixed(2)} ms · starts +${span.start_ms.toFixed(2)} ms${counts ? ` · ${counts}` : ""}`,
"performance-span",
);
row.style.paddingLeft = `${spanDepth(spans, index) * 14}px`;
});
if (attempt.native.dropped_spans)
append(`${attempt.native.dropped_spans} native spans dropped.`, "hint");
}
for (const display of report.displays)
append(
`${display.lane} synchronous DOM update ${display.dom_update_ms.toFixed(2)} ms · ${display.count} displayed results`,
"hint",
);
if (report.dropped_attempts || report.dropped_displays)
append(
`History capped: ${report.dropped_attempts} attempts and ${report.dropped_displays} displays dropped.`,
"hint",
);
}
async function invokeMeasured(request: SearchRequest, lane: SearchLane) {
const attempt = request.measurement.begin(lane, performance.now());
try {
const response = await invoke<MeasuredResponse<SearchResult>>(
lane === "lexical" ? "search_measured" : "hybrid_search_measured",
{
query: request.query,
filters: request.filters,
debugPerformance: request.measurement.debug,
},
);
request.measurement.completed(
attempt,
performance.now(),
response.metrics,
false,
);
return response.results;
} catch (error) {
const failure = measuredFailure(error);
request.measurement.completed(
attempt,
performance.now(),
failure?.metrics ?? null,
true,
);
refreshPerformance(request);
throw failure ? new Error(failure.message) : error;
}
}
function submitSearch(request: SearchRequest, lane: SearchLane) {
request.measurement.submitted(lane, performance.now());
(lane === "lexical" ? lexicalLane : hybridLane).submit(request);
}
const isCurrent = (request: SearchRequest) => request.id === requestId;
const lexicalLane = new LatestLane<SearchRequest>(
async (request) => {
const found = await invoke<SearchResult[]>("search", {
query: request.query,
filters: request.filters,
});
const found = await invokeMeasured(request, "lexical");
applyResults(found, request, "lexical");
},
isCurrent,
@@ -509,10 +771,7 @@ const lexicalLane = new LatestLane<SearchRequest>(
const hybridLane = new LatestLane<SearchRequest>(
async (request) => {
if (!canSearchSemantically()) return;
const found = await invoke<SearchResult[]>("hybrid_search", {
query: request.query,
filters: request.filters,
});
const found = await invokeMeasured(request, "hybrid");
applyResults(found, request, "hybrid");
},
isCurrent,
@@ -534,12 +793,13 @@ function invalidateSearch() {
latestSearch = null;
}
async function search() {
async function search(inputAt = performance.now()) {
invalidateSearch();
querySemanticFailed = false;
shownLane = "lexical";
const value = query.value.trim();
if (!value) {
element("performance-panel").hidden = true;
results = [];
displayedRequestId = null;
selected = -1;
@@ -557,13 +817,28 @@ async function search() {
query: value,
filters: captureFilters(),
started: performance.now(),
measurement: new SearchMeasurement(
inputAt,
performance.now(),
element<HTMLInputElement>("performance-toggle").checked,
{
model: semanticStatus?.worker_info?.model ?? null,
device: semanticStatus?.worker_info?.device ?? null,
runtime: semanticStatus?.worker_info?.runtime ?? null,
threads: semanticStatus?.worker_info?.threads ?? null,
dimensions: semanticStatus?.worker_info?.dimensions ?? null,
semantic_ready: canSearchSemantically(),
indexed_chunks: semanticStatus?.chunks ?? null,
},
),
};
latestSearch = request;
refreshPerformance(request);
element("result-summary").textContent = "SEARCHING…";
lexicalLane.submit(request);
submitSearch(request, "lexical");
hybridTimer = setTimeout(() => {
if (isCurrent(request) && canSearchSemantically())
hybridLane.submit(request);
submitSearch(request, "hybrid");
}, 350);
} catch (error) {
showError(error);
@@ -573,12 +848,41 @@ async function search() {
async function refreshStats() {
const stats = await invoke<IndexStats>("stats");
element("index-status").textContent =
`${stats.files.toLocaleString()} files · ${stats.content_files.toLocaleString()} with searchable content`;
`${stats.files.toLocaleString()} files · ${stats.content_files.toLocaleString()} with extracted text`;
}
query.addEventListener("input", () => {
const inputAt = performance.now();
invalidateSearch();
timer = setTimeout(search, 60);
timer = setTimeout(() => void search(inputAt), 60);
});
element("performance-toggle").addEventListener("change", () => {
const enabled = element<HTMLInputElement>("performance-toggle").checked;
try {
localStorage.setItem("searchi-performance-details", String(enabled));
} catch {
/* Session preference still works. */
}
void search();
});
element("copy-performance").addEventListener("click", () => {
if (!latestSearch?.measurement.debug) return;
if (!navigator.clipboard) {
element("copy-performance-status").textContent =
"Clipboard unavailable. Timings remain visible here.";
return;
}
const report = JSON.stringify(latestSearch.measurement.export(), null, 2);
void navigator.clipboard.writeText(report).then(
() => {
element("copy-performance-status").textContent =
"Timing report copied. No query or file paths included.";
},
() => {
element("copy-performance-status").textContent =
"Clipboard unavailable. Timings remain visible here.";
},
);
});
element("search-form").addEventListener("submit", (event) => {
event.preventDefault();
@@ -714,6 +1018,7 @@ document.addEventListener("keydown", (event) => {
query.select();
}
if (event.key === "Escape") {
clearMedia();
document
.querySelectorAll<HTMLDetailsElement>(".popover[open]")
.forEach((popover) => {
apps/desktop/src/media-display.ts
+98
View File
@@ -0,0 +1,98 @@
/** Media formatting and selection revisions shared by the preview and tests. */
export interface MediaSource {
kind: string;
number: number;
}
export type MediaKind = "image" | "audio" | "video";
export function timestampLabel(milliseconds: number): string {
if (!Number.isFinite(milliseconds) || milliseconds < 0) return "Unknown time";
const seconds = Math.floor(milliseconds / 1000);
return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
}
export function seekSeconds(source?: MediaSource): number | undefined {
return source?.kind === "timestamp_ms" &&
Number.isFinite(source.number) &&
source.number >= 0
? source.number / 1000
: undefined;
}
export function sourceLabel(source?: MediaSource): string {
if (!source) return "";
if (source.kind === "image") return "Image";
if (source.kind === "timestamp_ms") return timestampLabel(source.number);
return `${source.kind} ${source.number}`;
}
export function matchLabel(kind: string): string {
const labels: Record<string, string> = {
semantic_image: "Image meaning",
semantic_video: "Video frame meaning",
semantic_audio: "Audio meaning",
semantic: "Meaning",
hybrid: "Keyword + meaning",
exact_filename: "Exact filename",
exact_identifier: "Exact identifier",
filename: "Filename",
keyword: "Content",
};
return labels[kind] ?? kind.replaceAll("_", " ");
}
export function mediaKind(filename: string): MediaKind | undefined {
const extension = filename.split(".").pop()?.toLowerCase() ?? "";
if (
[
"png",
"jpg",
"jpeg",
"webp",
"bmp",
"gif",
"tiff",
"tif",
"heic",
"avif",
].includes(extension)
)
return "image";
if (
["mp4", "mkv", "mov", "webm", "avi", "m4v", "mpeg", "mpg"].includes(
extension,
)
)
return "video";
if (
[
"mp3",
"wav",
"flac",
"ogg",
"m4a",
"aac",
"opus",
"wma",
"aiff",
"aif",
].includes(extension)
)
return "audio";
return undefined;
}
/** Even selecting the same file at a new timestamp invalidates old responses. */
export class PreviewRevision {
private revision = 0;
private key = "";
select(key: string): { revision: number; changed: boolean } {
const changed = this.key !== key;
if (changed) {
this.key = key;
this.revision++;
}
return { revision: this.revision, changed };
}
accepts(revision: number) {
return revision === this.revision && this.key !== "";
}
clear() {
this.key = "";
this.revision++;
}
}
apps/desktop/src/search-performance.ts
+203
View File
@@ -0,0 +1,203 @@
/** UI times use performance.now(); native spans use their own command clock. */
export interface NativeSpan {
name: string;
start_ms: number;
duration_ms: number;
parent: number | null;
counters: Record<string, number>;
}
export interface NativeMetrics {
total_ms: number;
result_count: number;
trace: NativeSpan[] | null;
dropped_spans: number;
}
export interface MeasuredResponse<T> {
results: T[];
metrics: NativeMetrics;
}
export type SearchLane = "lexical" | "hybrid";
export interface AttemptStart {
lane: SearchLane;
startedAt: number;
submittedAt: number;
submission: number;
}
interface Attempt {
lane: SearchLane;
submission: number;
attempt: number;
queued_ms: number;
retry_wait_ms: number;
invoke_ms: number;
native: NativeMetrics | null;
transport_estimate_ms: number | null;
failed: boolean;
ended_at: number;
}
export class SearchMeasurement {
debug: boolean;
inputAt: number;
scheduledAt: number;
context: Record<string, string | number | boolean | null>;
submissions = new Map<SearchLane, number>();
latestSubmission = new Map<SearchLane, number>();
submissionSequence = new Map<SearchLane, number>();
laneAttempts = new Map<SearchLane, number>();
attempts: Attempt[] = [];
droppedAttempts = 0;
droppedDisplays = 0;
firstDisplayMs: number | null = null;
displays: {
lane: SearchLane;
elapsed_ms: number;
dom_update_ms: number;
count: number;
}[] = [];
constructor(
inputAt: number,
scheduledAt: number,
debug: boolean,
context: Record<string, string | number | boolean | null> = {},
) {
this.inputAt = inputAt;
this.scheduledAt = scheduledAt;
this.debug = debug;
this.context = { ...context };
}
submitted(lane: SearchLane, at: number) {
if (!this.submissions.has(lane)) this.submissions.set(lane, at);
this.latestSubmission.set(lane, at);
this.submissionSequence.set(
lane,
(this.submissionSequence.get(lane) ?? 0) + 1,
);
}
begin(lane: SearchLane, startedAt: number): AttemptStart {
return {
lane,
startedAt,
submittedAt: this.latestSubmission.get(lane) ?? startedAt,
submission: this.submissionSequence.get(lane) ?? 0,
};
}
completed(
token: AttemptStart,
end: number,
native: NativeMetrics | null,
failed: boolean,
) {
const { lane, startedAt: start, submittedAt, submission } = token;
const prior = this.attempts.filter((attempt) => attempt.lane === lane);
const previous = prior.at(-1);
const retry = previous?.failed && previous.submission === submission;
const attemptNumber = (this.laneAttempts.get(lane) ?? 0) + 1;
this.laneAttempts.set(lane, attemptNumber);
const elapsed = Math.max(0, end - start);
this.attempts.push({
lane,
submission,
attempt: attemptNumber,
queued_ms: retry ? 0 : Math.max(0, start - submittedAt),
retry_wait_ms: retry ? Math.max(0, start - previous.ended_at) : 0,
invoke_ms: elapsed,
native,
transport_estimate_ms: native
? Math.max(0, elapsed - native.total_ms)
: null,
failed,
ended_at: end,
});
if (this.attempts.length > (this.debug ? 32 : 4)) {
this.attempts.shift();
this.droppedAttempts++;
}
}
displayed(lane: SearchLane, start: number, end: number, count: number) {
if (this.firstDisplayMs === null)
this.firstDisplayMs = Math.max(0, end - this.inputAt);
this.displays.push({
lane,
elapsed_ms: Math.max(0, end - this.inputAt),
dom_update_ms: Math.max(0, end - start),
count,
});
if (this.displays.length > (this.debug ? 16 : 2)) {
this.displays.shift();
this.droppedDisplays++;
}
}
latestNative(lane: SearchLane) {
return (
this.attempts
.filter((attempt) => attempt.lane === lane && !attempt.failed)
.at(-1)?.native ?? null
);
}
/** Deliberately exclude query text, filenames, paths, snippets and errors. */
export() {
return {
version: 1,
context: this.context,
clock_note:
"UI monotonic elapsed times; native command-relative spans. Nested spans and parallel lanes must not be summed.",
debounce_ms: Math.max(0, this.scheduledAt - this.inputAt),
lane_schedule_ms: Object.fromEntries(
[...this.submissions].map(([lane, at]) => [
lane,
Math.max(0, at - this.scheduledAt),
]),
),
attempts: this.attempts.map(({ ended_at: _end, ...attempt }) => attempt),
displays: this.displays,
first_results_ms: this.firstDisplayMs,
latest_displayed_ms: this.displays.at(-1)?.elapsed_ms ?? null,
hybrid_final_results_ms:
[...this.displays]
.reverse()
.find((display) => display.lane === "hybrid")?.elapsed_ms ?? null,
dropped_attempts: this.droppedAttempts,
dropped_displays: this.droppedDisplays,
};
}
}
export function measuredFailure(
error: unknown,
): { message: string; metrics: NativeMetrics } | null {
if (!error || typeof error !== "object") return null;
const value = error as { message?: unknown; metrics?: NativeMetrics };
return typeof value.message === "string" &&
value.metrics &&
Number.isFinite(value.metrics.total_ms)
? { message: value.message, metrics: value.metrics }
: null;
}
/** Malformed/cyclic parent chains cannot trap the debug renderer. */
export function spanDepth(spans: NativeSpan[], index: number) {
let depth = 0;
const visited = new Set([index]);
let parent = spans[index]?.parent;
while (
parent != null &&
spans[parent] &&
!visited.has(parent) &&
depth < 16
) {
visited.add(parent);
depth++;
parent = spans[parent].parent;
}
return depth;
}
export function briefMs(value: number) {
return value < 1
? "<1"
: value < 10
? value.toFixed(1)
: String(Math.round(value));
}
apps/desktop/src/style.css
+70
View File
@@ -766,3 +766,73 @@ footer {
outline-color: Highlight;
}
}
/* Keep one scoped media player within the existing result preview. */
.media-preview {
margin: 0 20px 14px;
overflow: hidden;
border-radius: 10px;
background: #f3f4f7;
}
.media-preview img,
.media-preview video {
display: block;
width: 100%;
max-height: 340px;
object-fit: contain;
}
.media-preview audio {
display: block;
width: 100%;
}
#media-preview-status {
margin: 0 20px 14px;
}
.file-icon[data-type="image"] {
color: #6c62b5;
}
.file-icon[data-type="video"] {
color: #a16a48;
}
.file-icon[data-type="audio"] {
color: #608271;
}
.performance-toggle {
display: flex;
align-items: center;
gap: 8px;
margin-top: 14px;
}
.performance-toggle input {
width: auto;
accent-color: var(--brand);
}
#performance-panel {
margin: 0 20px 12px;
padding: 10px 14px;
border: 1px solid var(--line);
border-radius: 10px;
background: var(--surface);
font-size: 12px;
}
#performance-panel summary {
cursor: pointer;
font-weight: 600;
}
#performance-detail {
max-height: 280px;
overflow: auto;
font-variant-numeric: tabular-nums;
}
#performance-detail p {
margin: 8px 0;
}
.performance-attempt {
font-weight: 600;
}
.performance-span {
border-left: 2px solid var(--line);
}
#copy-performance-status {
margin-left: 8px;
color: var(--muted);
}
apps/desktop/tests/media-display.test.mjs
+83
View File
@@ -0,0 +1,83 @@
import test from "node:test";
import assert from "node:assert/strict";
import {
timestampLabel,
seekSeconds,
sourceLabel,
matchLabel,
PreviewRevision,
} from "../src/media-display.ts";
import { LatestLane } from "../src/search-state.ts";
const settle = () => new Promise((resolve) => setImmediate(resolve));
test("millisecond seeks preserve fractions and readable minute labels", () => {
assert.equal(timestampLabel(125450), "2:05");
assert.equal(timestampLabel(3600000), "60:00");
assert.equal(seekSeconds({ kind: "timestamp_ms", number: 125450 }), 125.45);
assert.equal(seekSeconds({ kind: "page", number: 4 }), undefined);
assert.equal(seekSeconds({ kind: "timestamp_ms", number: -1 }), undefined);
assert.equal(timestampLabel(Infinity), "Unknown time");
assert.equal(sourceLabel({ kind: "image", number: 0 }), "Image");
assert.equal(matchLabel("semantic_audio"), "Audio meaning");
assert.equal(matchLabel("semantic_video"), "Video frame meaning");
});
test("another timestamp in the same movie invalidates older preparations", () => {
const gate = new PreviewRevision();
const first = gate.select("movie:1000");
assert.equal(gate.select("movie:1000").changed, false);
const second = gate.select("movie:3000");
assert.equal(gate.accepts(first.revision), false);
assert.equal(gate.accepts(second.revision), true);
gate.clear();
assert.equal(gate.accepts(second.revision), false);
});
test("one active preview and only latest pending reject stale output", async () => {
const gate = new PreviewRevision();
const executed = [];
const accepted = [];
let release;
const lane = new LatestLane(
async (request) => {
executed.push(request.path);
if (request.path === "first")
await new Promise((resolve) => {
release = resolve;
});
if (gate.accepts(request.revision)) accepted.push(request.path);
},
(request) => gate.accepts(request.revision),
() => assert.fail("Unexpected preview failure"),
);
lane.submit({ path: "first", revision: gate.select("first").revision });
lane.submit({ path: "middle", revision: gate.select("middle").revision });
lane.submit({ path: "latest", revision: gate.select("latest").revision });
release();
await settle();
assert.deepEqual(executed, ["first", "latest"]);
assert.deepEqual(accepted, ["latest"]);
});
test("Escape clears queued preparation and rejects active output", async () => {
const gate = new PreviewRevision();
let release;
const accepted = [];
const lane = new LatestLane(
async (request) => {
await new Promise((resolve) => {
release = resolve;
});
if (gate.accepts(request.revision)) accepted.push(request.path);
},
(request) => gate.accepts(request.revision),
() => assert.fail("Unexpected preview failure"),
);
lane.submit({ path: "one", revision: gate.select("one").revision });
lane.submit({ path: "two", revision: gate.select("two").revision });
gate.clear();
lane.clear();
release();
await settle();
assert.deepEqual(accepted, []);
});
apps/desktop/tests/search-performance.test.mjs
+102
View File
@@ -0,0 +1,102 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { SearchMeasurement, measuredFailure, spanDepth, briefMs } from '../src/search-performance.ts';
const finish = (trace, lane, start, end, metrics, failed) => trace.completed(trace.begin(lane, start), end, metrics, failed);
const native = { total_ms: 12, result_count: 2, trace: [], dropped_spans: 0 };
test('separates debounce, intentional semantic delay, queue, invoke and DOM update', () => {
const trace = new SearchMeasurement(100, 165, true);
trace.submitted('lexical', 165);
finish(trace, 'lexical', 180, 200, native, false);
trace.displayed('lexical', 200, 203, 2);
trace.submitted('hybrid', 515);
finish(trace, 'hybrid', 520, 550, native, false);
trace.displayed('hybrid', 550, 554, 3);
const report = trace.export();
assert.equal(report.debounce_ms, 65);
assert.equal(report.lane_schedule_ms.hybrid, 350);
assert.equal(report.attempts[0].queued_ms, 15);
assert.equal(report.attempts[0].invoke_ms, 20);
assert.equal(report.attempts[0].transport_estimate_ms, 8);
assert.equal(report.displays[0].dom_update_ms, 3);
assert.equal(report.first_results_ms, 103);
assert.equal(report.hybrid_final_results_ms, 454);
});
test('failed attempt timings survive retry and lexical response does not pretend hybrid completion', () => {
const trace = new SearchMeasurement(0, 0, true);
trace.submitted('lexical', 0);
finish(trace, 'lexical', 0, 2, native, true);
finish(trace, 'lexical', 402, 414, native, false);
trace.displayed('lexical', 414, 415, 2);
const report = trace.export();
assert.equal(report.attempts[1].retry_wait_ms, 400);
assert.equal(report.attempts[0].failed, true);
assert.equal(report.attempts[0].transport_estimate_ms, 0);
assert.equal(report.hybrid_final_results_ms, null);
assert.equal(report.latest_displayed_ms, 415);
assert.equal(measuredFailure({message: 'worker busy', metrics: native}).message, 'worker busy');
assert.equal(measuredFailure('legacy failure'), null);
});
test('debug history is bounded and first response survives dropped displays', () => {
const context = { device: 'cpu', threads: 1 };
const trace = new SearchMeasurement(0, 0, true, context);
context.threads = 8;
for (let i = 0; i < 40; i++) {
finish(trace, 'hybrid', i, i + 1, native, false);
trace.displayed('hybrid', i, i + 1, 1);
}
const report = trace.export();
assert.equal(report.attempts.length, 32);
assert.equal(report.displays.length, 16);
assert.equal(report.dropped_attempts, 8);
assert.equal(report.dropped_displays, 24);
assert.equal(report.first_results_ms, 1);
assert.equal(report.context.threads, 1);
assert.doesNotMatch(JSON.stringify(report), /query|filename|snippet|ended_at/);
});
test('cyclic or invalid native span parents cannot hang hierarchy rendering', () => {
const spans = [{parent: 1}, {parent: 0}, {parent: 99}];
assert.equal(spanDepth(spans, 0), 1);
assert.equal(spanDepth(spans, 2), 0);
});
test('a later refresh has queue timing rather than being misreported as a busy retry', () => {
const trace = new SearchMeasurement(0, 0, true);
trace.submitted('hybrid', 350);
finish(trace, 'hybrid', 350, 360, native, false);
trace.submitted('hybrid', 1000);
finish(trace, 'hybrid', 1010, 1020, native, false);
assert.equal(trace.export().attempts[1].queued_ms, 10);
assert.equal(trace.export().attempts[1].retry_wait_ms, 0);
assert.equal(briefMs(.44), '<1');
assert.equal(briefMs(3.14), '3.1');
});
test('new submission after a failure measures pending queue rather than retry wait', () => {
const trace = new SearchMeasurement(0, 0, true);
trace.submitted('hybrid', 350);
finish(trace, 'hybrid', 350, 360, native, true);
trace.submitted('hybrid', 1000);
finish(trace, 'hybrid', 1010, 1020, native, false);
assert.equal(trace.export().attempts[1].queued_ms, 10);
assert.equal(trace.export().attempts[1].retry_wait_ms, 0);
});
test('an in-flight attempt keeps its captured submission when a refresh queues', () => {
const trace = new SearchMeasurement(0, 0, true);
trace.submitted('hybrid', 350);
const active = trace.begin('hybrid', 355);
trace.submitted('hybrid', 1000);
trace.completed(active, 1005, native, true);
finish(trace, 'hybrid', 1010, 1020, native, false);
const attempts = trace.export().attempts;
assert.equal(attempts[0].queued_ms, 5);
assert.equal(attempts[1].queued_ms, 10);
assert.equal(attempts[1].retry_wait_ms, 0);
assert.notEqual(attempts[0].submission, attempts[1].submission);
});
crates/searchi-cli/src/main.rs
+109 -15
View File
@@ -1,9 +1,10 @@
//! Thin command-line adapter around the shared, CPU-only search engine.
//! Thin headless adapter around the shared local search engine.
use std::path::PathBuf;
use anyhow::{bail, Context, Result};
use clap::{Parser, Subcommand};
use searchi_core::metrics::{MetricsMode, SearchResponse, TraceCollector};
use searchi_core::{IndexOptions, SearchIndex, SearchOptions};
use searchi_service::HybridSearch;
@@ -46,6 +47,12 @@ enum Command {
/// Emit a JSON array for scripting.
#[arg(long)]
json: bool,
/// Include basic measured statistics in JSON output.
#[arg(long)]
performance: bool,
/// Include bounded granular stage timings and diagnostic counters.
#[arg(long)]
debug_performance: bool,
/// Restrict results to this folder and descendants.
#[arg(long)]
folder: Option<PathBuf>,
@@ -56,6 +63,11 @@ enum Command {
min_size: Option<u64>,
#[arg(long)]
max_size: Option<u64>,
/// Inclusive media duration limit in milliseconds.
#[arg(long)]
min_duration_ms: Option<u32>,
#[arg(long)]
max_duration_ms: Option<u32>,
/// Inclusive modification time in Unix milliseconds.
#[arg(long)]
modified_after_ms: Option<i64>,
@@ -80,6 +92,14 @@ enum Command {
embedding_worker: PathBuf,
#[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u8).range(1..=8))]
cpu_threads: u8,
/// Standalone decoder; normally discovered beside the installed CLI.
#[arg(long, requires = "ffprobe")]
ffmpeg: Option<PathBuf>,
#[arg(long, requires = "ffmpeg")]
ffprobe: Option<PathBuf>,
/// Explicitly build only document vectors, without media.
#[arg(long)]
text_only: bool,
},
/// Show the indexed file and content counts.
Stats {
@@ -122,10 +142,14 @@ fn main() -> Result<()> {
query,
limit,
json,
performance,
debug_performance,
folder,
extension,
min_size,
max_size,
min_duration_ms,
max_duration_ms,
modified_after_ms,
modified_before_ms,
model_dir,
@@ -141,30 +165,56 @@ fn main() -> Result<()> {
extension,
min_size,
max_size,
min_duration_ms,
max_duration_ms,
modified_after_unix_ms: modified_after_ms,
modified_before_unix_ms: modified_before_ms,
};
let mut trace = TraceCollector::new(if debug_performance {
MetricsMode::Debug
} else {
MetricsMode::Basic
});
let results = if let (Some(model), Some(runtime)) = (model_dir, embedding_worker) {
HybridSearch::open(&db, model, runtime, usize::from(cpu_threads))?
.search(&query, usize::from(limit), &options)
let mut service = trace.timed("cli.session_initialize", None, |_, _| {
HybridSearch::open(&db, model, runtime, usize::from(cpu_threads))
})?;
service
.search_traced(&query, usize::from(limit), &options, &mut trace, None)
.context("Hybrid search failed")?
} else {
SearchIndex::open(&db)
.with_context(|| format!("Cannot open index {}", db.display()))?
.search_with_options(&query, usize::from(limit), &options)
let index = trace
.timed("cli.index_open", None, |_, _| SearchIndex::open(&db))
.with_context(|| format!("Cannot open index {}", db.display()))?;
index
.search_with_options_traced(
&query,
usize::from(limit),
&options,
&mut trace,
None,
)
.context("Search failed")?
};
let response = SearchResponse {
metrics: trace.finish(results.len()),
results,
};
if json {
println!("{}", serde_json::to_string_pretty(&results)?);
} else if results.is_empty() {
if performance || debug_performance {
println!("{}", serde_json::to_string_pretty(&response)?);
} else {
println!("{}", serde_json::to_string_pretty(&response.results)?);
}
} else if response.results.is_empty() {
println!("No matching files.");
} else {
for result in results {
for result in &response.results {
println!("{} [{}]", result.path, result.match_kind);
if let Some(source) = result.source {
if let Some(source) = &result.source {
println!(" {} {}", source.kind, source.number);
}
if let Some(error) = result.extraction_error {
if let Some(error) = &result.extraction_error {
println!(" Text extraction unavailable: {error}");
}
if !result.snippet.is_empty() {
@@ -172,18 +222,36 @@ fn main() -> Result<()> {
}
}
}
if !json {
eprintln!(
"{} results · {:.2} ms (includes CLI initialization)",
response.metrics.result_count, response.metrics.total_ms
);
if debug_performance {
eprintln!("{}", serde_json::to_string_pretty(&response.metrics)?);
}
}
}
Command::Embed {
db,
model_dir,
embedding_worker,
cpu_threads,
ffmpeg,
ffprobe,
text_only,
} => {
require_existing_index(&db)?;
let report =
HybridSearch::open(&db, model_dir, embedding_worker, usize::from(cpu_threads))?
.index_embeddings()
.context("Semantic indexing failed; prior generation remains active")?;
let mut service =
HybridSearch::open(&db, model_dir, embedding_worker, usize::from(cpu_threads))?;
let report = if text_only {
service.index_embeddings()
} else {
let tools = resolve_media_tools(ffmpeg, ffprobe)?;
service
.index_embeddings_with_media(&tools, &searchi_service::MediaLimits::default())
}
.context("Semantic indexing failed; prior generation remains active")?;
println!("{}", serde_json::to_string_pretty(&report)?);
}
Command::Stats { db, json } => {
@@ -204,6 +272,32 @@ fn main() -> Result<()> {
Ok(())
}
fn resolve_media_tools(
ffmpeg: Option<PathBuf>,
ffprobe: Option<PathBuf>,
) -> Result<searchi_service::FfmpegTools> {
if let (Some(ffmpeg), Some(ffprobe)) = (ffmpeg, ffprobe) {
return searchi_service::FfmpegTools::new(ffmpeg, ffprobe);
}
if let Some(tools) = searchi_service::FfmpegTools::from_environment()? {
return Ok(tools);
}
let development =
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.searchi/media-runtime/bin");
let directory = if cfg!(debug_assertions) && development.is_dir() {
development
} else {
std::env::current_exe()?
.parent()
.context("CLI installation directory is unavailable")?
.join("media-runtime/bin")
};
searchi_service::FfmpegTools::new(
directory.join(if cfg!(windows) { "ffmpeg.exe" } else { "ffmpeg" }),
directory.join(if cfg!(windows) { "ffprobe.exe" } else { "ffprobe" }),
).context("Media tools are missing from this installation; repair the headless package or explicitly use --text-only")
}
/// Read commands must not silently create a new empty database after a typo.
fn require_existing_index(db: &std::path::Path) -> Result<()> {
if !db.is_file() {
crates/searchi-cli/tests/commands.rs
+30
View File
@@ -68,6 +68,36 @@ fn indexes_persists_and_returns_search_and_stats_json() {
.unwrap()
.contains("termination"));
for flag in ["--performance", "--debug-performance"] {
let measured = fixture.run(&["search", "--db", "index.db", "termination", "--json", flag]);
assert!(
measured.status.success(),
"{}",
String::from_utf8_lossy(&measured.stderr)
);
let envelope: serde_json::Value = serde_json::from_slice(&measured.stdout).unwrap();
assert_eq!(
envelope["results"],
serde_json::Value::Array(results.clone())
);
assert_eq!(envelope["metrics"]["result_count"], 1);
let total = envelope["metrics"]["total_ms"].as_f64().unwrap();
assert!(total.is_finite() && total >= 0.0);
if flag == "--performance" {
assert!(envelope["metrics"]["trace"].is_null());
} else {
let spans = envelope["metrics"]["trace"].as_array().unwrap();
assert!(spans.iter().any(|span| span["name"] == "cli.index_open"));
assert!(spans.len() > 2);
for span in spans {
let start = span["start_ms"].as_f64().unwrap();
let duration = span["duration_ms"].as_f64().unwrap();
assert!(start >= 0.0 && duration >= 0.0 && start + duration <= total + 0.001);
assert!(!span["name"].as_str().unwrap().contains("termination"));
}
}
}
let stats = fixture.run(&["stats", "--db", "index.db", "--json"]);
assert!(stats.status.success());
let stats: serde_json::Value = serde_json::from_slice(&stats.stdout).unwrap();
crates/searchi-core/examples/trace_overhead.rs
+80
View File
@@ -0,0 +1,80 @@
//! Small prepared SQLite instrumentation diagnostic; not a production benchmark.
use anyhow::{ensure, Result};
use searchi_core::SearchIndex;
use std::{fs, path::Path, time::Instant};
fn distribution(samples: &[f64]) -> serde_json::Value {
let mut sorted = samples.to_vec();
sorted.sort_by(f64::total_cmp);
let percentile = |fraction: f64| {
sorted[((sorted.len() as f64 * fraction).ceil() as usize).saturating_sub(1)]
};
serde_json::json!({"samples":sorted.len(),"p50_ms":percentile(0.50),"p95_ms":percentile(0.95),"mean_ms":sorted.iter().sum::<f64>()/sorted.len() as f64})
}
fn main() -> Result<()> {
let fixture = tempfile::tempdir()?;
let root = fixture.path().join("files");
fs::create_dir(&root)?;
for index in 0..1000 {
fs::write(
root.join(format!("record-{index:04}.txt")),
format!(
"Supplier agreement {index}. Cancellation needs written notice. Group {}.",
index % 31
),
)?;
}
let mut index = SearchIndex::open(fixture.path().join("index.sqlite"))?;
let report = index.index_folder(&root)?;
index.prewarm()?;
ensure!(report.indexed == 1000, "Unexpected fixture indexing count");
let queries: Vec<String> = (0..24)
.map(|index| format!("record-{:04}.txt", index * 37))
.collect();
for query in &queries {
index.search_measured(query, 10, false)?;
index.search_measured(query, 10, true)?;
}
let mut basic = Vec::new();
let mut debug = Vec::new();
let mut trace_spans = Vec::new();
// Alternate order to reduce monotonic cache/CPU-order effects.
for repetition in 0..20 {
for query in &queries {
for detailed in if repetition % 2 == 0 {
[false, true]
} else {
[true, false]
} {
let started = Instant::now();
let response = index.search_measured(query, 10, detailed)?;
let elapsed = started.elapsed().as_secs_f64() * 1000.0;
ensure!(
response
.results
.first()
.is_some_and(|result| &result.filename == query),
"Fixture retrieval changed"
);
if detailed {
trace_spans.push(response.metrics.trace.as_ref().unwrap().len());
debug.push(elapsed);
} else {
ensure!(
response.metrics.trace.is_none(),
"Basic unexpectedly stored traces"
);
basic.push(elapsed);
}
}
}
}
let output = serde_json::json!({"kind":"prepared_sqlite_trace_overhead","production_claim":false,"profile":if cfg!(debug_assertions){"debug"}else{"release"},"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"files":1000,"queries":24,"repetitions":20,"measurement":"outer Rust call including metrics assembly; excludes fixture setup/JSON serialization","basic":distribution(&basic),"debug":distribution(&debug),"basic_samples_ms":basic,"debug_samples_ms":debug,"debug_span_count_min":trace_spans.iter().min(),"debug_span_count_max":trace_spans.iter().max(),"limits":"tiny warm lexical corpus; harness serial/no model/media/GPU; other VM work including native photo inference may contend; not isolated production performance"});
let serialized = serde_json::to_string_pretty(&output)?;
if let Some(path) = std::env::args_os().nth(1) {
fs::write(Path::new(&path), serialized)?;
} else {
println!("{serialized}");
}
Ok(())
}
crates/searchi-core/src/lib.rs
+255 -6
View File
@@ -4,7 +4,10 @@
//! Text and document extraction have byte limits, but still run in this process.
//! Worker isolation remains a production milestone.
mod extraction;
pub mod metrics;
use anyhow::{bail, Context, Result};
pub use metrics::{MetricsMode, SearchMetrics, SearchResponse, SearchSpan};
use metrics::{SpanToken, TraceCollector};
use rusqlite::{params, params_from_iter, types::Value, Connection, OptionalExtension};
use serde::Serialize;
use std::{
@@ -16,7 +19,7 @@ use std::{
};
use walkdir::WalkDir;
const SCHEMA_VERSION: i64 = 9;
const SCHEMA_VERSION: i64 = 10;
const MAX_RESULTS: usize = 1000;
#[derive(Debug, Default, Serialize)]
@@ -53,6 +56,8 @@ pub struct SearchOptions {
pub max_size: Option<u64>,
pub modified_after_unix_ms: Option<i64>,
pub modified_before_unix_ms: Option<i64>,
pub min_duration_ms: Option<u32>,
pub max_duration_ms: Option<u32>,
}
#[derive(Debug, Serialize)]
@@ -66,7 +71,8 @@ pub struct SearchResult {
pub extraction_error: Option<String>,
}
/// A verified matching passage location; numbers are one based.
/// Verified source location. Document numbers are one based; timestamp_ms
/// numbers are milliseconds from media start, and image numbers are zero.
#[derive(Debug, Clone, Serialize)]
pub struct SourceLocation {
pub kind: String,
@@ -227,6 +233,15 @@ impl SearchIndex {
UPDATE name_statistics_state SET dirty=1 WHERE singleton=1; END;
PRAGMA user_version=9;",
)?;
initialization.execute_batch("CREATE INDEX IF NOT EXISTS files_extension ON files(extension,id);
CREATE TABLE IF NOT EXISTS media_metadata(path TEXT PRIMARY KEY, file_id INTEGER NOT NULL UNIQUE, tags TEXT NOT NULL,duration_ms INTEGER NOT NULL);
CREATE VIRTUAL TABLE IF NOT EXISTS media_tags USING fts5(tags,content='media_metadata',content_rowid='file_id');
CREATE TRIGGER IF NOT EXISTS media_metadata_insert AFTER INSERT ON media_metadata BEGIN INSERT INTO media_tags(rowid,tags) VALUES(new.file_id,new.tags); END;
CREATE TRIGGER IF NOT EXISTS media_metadata_delete AFTER DELETE ON media_metadata BEGIN INSERT INTO media_tags(media_tags,rowid,tags) VALUES('delete',old.file_id,old.tags); END;
CREATE TRIGGER IF NOT EXISTS media_metadata_update AFTER UPDATE ON media_metadata BEGIN INSERT INTO media_tags(media_tags,rowid,tags) VALUES('delete',old.file_id,old.tags); INSERT INTO media_tags(rowid,tags) VALUES(new.file_id,new.tags); END;
CREATE TRIGGER IF NOT EXISTS files_media_delete AFTER DELETE ON files BEGIN DELETE FROM media_metadata WHERE path=old.path; END;
CREATE TRIGGER IF NOT EXISTS files_media_update AFTER UPDATE ON files WHEN new.path!=old.path OR new.size!=old.size OR new.mtime!=old.mtime BEGIN DELETE FROM media_metadata WHERE path=old.path; END;
PRAGMA user_version=10;")?;
initialization.commit()?;
}
Ok(Self {
@@ -236,6 +251,32 @@ impl SearchIndex {
})
}
/// Persist genuine extracted media tags, guarded against filesystem changes.
pub fn set_media_metadata(
&mut self,
path: &str,
tags: &str,
duration_ms: u32,
size: u64,
mtime_ns: i64,
) -> Result<()> {
if tags.len() > 32768 {
bail!("Media tags exceed 32 KiB bound");
}
let tx = self.connection.transaction()?;
let id: Option<i64> = tx
.query_row(
"SELECT id FROM files WHERE path=?1 AND size=?2 AND mtime=?3",
params![path, i64::try_from(size)?, mtime_ns],
|r| r.get(0),
)
.optional()?;
let id = id.context("Media metadata file changed since indexing")?;
tx.execute("INSERT INTO media_metadata(path,file_id,tags,duration_ms) VALUES(?1,?2,?3,?4) ON CONFLICT(path) DO UPDATE SET tags=excluded.tags,duration_ms=excluded.duration_ms",params![path,id,tags,duration_ms])?;
tx.commit()?;
Ok(())
}
/// Refresh candidate-frequency statistics off the foreground search path.
/// Stale/missing estimates can affect speed, never literal-match correctness.
/// The app invokes this during background startup and after reconciliation.
@@ -465,6 +506,56 @@ impl SearchIndex {
limit: usize,
options: &SearchOptions,
) -> Result<Vec<SearchResult>> {
let mut trace = TraceCollector::new(MetricsMode::Basic);
self.search_with_options_traced(query, limit, options, &mut trace, None)
}
pub fn search_measured(
&self,
query: &str,
limit: usize,
debug: bool,
) -> Result<SearchResponse<SearchResult>> {
self.search_with_options_measured(query, limit, &SearchOptions::default(), debug)
}
pub fn search_with_options_measured(
&self,
query: &str,
limit: usize,
options: &SearchOptions,
debug: bool,
) -> Result<SearchResponse<SearchResult>> {
let mut trace = TraceCollector::new(if debug {
MetricsMode::Debug
} else {
MetricsMode::Basic
});
let results = self.search_with_options_traced(query, limit, options, &mut trace, None)?;
let metrics = trace.finish(results.len());
Ok(SearchResponse { results, metrics })
}
/// Share one monotonic request collector with downstream retrieval lanes.
pub fn search_with_options_traced(
&self,
query: &str,
limit: usize,
options: &SearchOptions,
trace: &mut TraceCollector,
parent: Option<SpanToken>,
) -> Result<Vec<SearchResult>> {
trace.timed("lexical.search", parent, |trace, parent| {
self.search_inner(query, limit, options, trace, parent)
})
}
fn search_inner(
&self,
query: &str,
limit: usize,
options: &SearchOptions,
trace: &mut TraceCollector,
parent: Option<SpanToken>,
) -> Result<Vec<SearchResult>> {
let normalization = trace.begin("lexical.normalization", parent);
if options
.min_size
.zip(options.max_size)
@@ -479,6 +570,13 @@ impl SearchIndex {
{
bail!("modified-after exceeds modified-before");
}
if options
.min_duration_ms
.zip(options.max_duration_ms)
.is_some_and(|(a, b)| a > b)
{
bail!("Minimum duration exceeds maximum duration");
}
let folder = options
.folder
.as_ref()
@@ -518,11 +616,16 @@ impl SearchIndex {
.map(|term| format!("\"{}\"", term.replace('"', "\"\"")))
.collect::<Vec<_>>()
.join(" AND ");
trace.counter(normalization, "query_bytes", query.len() as u64);
trace.end(normalization, Default::default());
let planning = trace.begin("lexical.candidate_plan", parent);
let limit = limit.min(MAX_RESULTS);
let filters = " AND (?3 IS NULL OR substr(f.path,1,length(?3))=?3)
AND (?4 IS NULL OR f.extension=?4)
AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6)
AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8)";
AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8)
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms>=?10))
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))";
let filter_values = [
folder_prefix.map_or(Value::Null, Value::Text),
extension.map_or(Value::Null, Value::Text),
@@ -535,7 +638,9 @@ impl SearchIndex {
.modified_before_unix_ms
.map_or(Value::Null, Value::Integer),
];
let has_filters = filter_values.iter().any(|value| *value != Value::Null);
let has_filters = filter_values.iter().any(|value| *value != Value::Null)
|| options.min_duration_ms.is_some()
|| options.max_duration_ms.is_some();
let mut results = Vec::new();
let mut selected = Vec::<i64>::new();
// Each lane filters its entire candidate set before LIMIT. There is no
@@ -561,13 +666,17 @@ impl SearchIndex {
if cache.len() + grams.len() > 4096 {
cache.clear();
}
let mut frequency_hits = 0u64;
let mut frequency_misses = 0u64;
let mut rare_first = grams
.into_iter()
.map(|gram| {
let term = gram.to_lowercase();
let frequency = if let Some(frequency) = cache.get(&term) {
frequency_hits += 1;
*frequency
} else {
frequency_misses += 1;
let frequency = vocabulary
.query_row([&term], |row| row.get::<_, i64>(0))
.optional()?
@@ -581,6 +690,9 @@ impl SearchIndex {
rare_first.sort();
drop(cache);
let anchor = substring_anchor(&characters, &rare_first);
trace.counter(planning, "frequency_cache_hits", frequency_hits);
trace.counter(planning, "frequency_cache_misses", frequency_misses);
trace.counter(planning, "trigram_index", u64::from(trigram));
let grams = format!("\"{}\"", anchor.replace('"', "\"\""));
let mut lanes = vec![
(
@@ -648,10 +760,30 @@ impl SearchIndex {
"rank,f.id",
));
}
lanes.push((
"media_metadata",
"media_tags CROSS JOIN files f ON f.id=media_tags.rowid".into(),
"media_tags MATCH ?1".into(),
fts.clone(),
"rank,f.id",
));
trace.end(planning, Default::default());
for (kind, from, condition, expression, ordering) in lanes {
if results.len() == limit {
break;
}
let lane = trace.begin(
match kind {
"exact_filename" => "lexical.exact_filename",
"filename" => "lexical.filename_substring",
"path" => "lexical.path_substring",
"name_terms" => "lexical.filename_terms",
"content" => "lexical.content",
"media_metadata" => "lexical.media_metadata",
_ => "lexical.other",
},
parent,
);
let exclusions = if selected.is_empty() {
String::new()
} else {
@@ -664,12 +796,14 @@ impl SearchIndex {
.join(",")
)
};
let rank = if matches!(kind, "content" | "name_terms") {
let rank = if matches!(kind, "content" | "name_terms" | "media_metadata") {
"rank"
} else {
"0.0"
};
let snippet = if kind == "content" {
let snippet = if kind == "media_metadata" {
"snippet(media_tags,0,'','',' … ',24)"
} else if kind == "content" {
"snippet(content_search,0,'','',' … ',24)"
} else if kind == "name_terms" {
"snippet(search,2,'','',' … ',24)"
@@ -698,6 +832,20 @@ impl SearchIndex {
let mut values = vec![Value::Text(expression), Value::Text(query.into())];
values.extend(filter_values.iter().cloned());
values.push(Value::Integer((limit - results.len()) as i64));
if sql.contains("?10") {
values.push(
options
.min_duration_ms
.map_or(Value::Null, |v| Value::Integer(i64::from(v))),
);
values.push(
options
.max_duration_ms
.map_or(Value::Null, |v| Value::Integer(i64::from(v))),
);
}
let sql_span = trace.begin("lexical.filtered_sql", lane);
trace.counter(sql_span, "filters_active", u64::from(has_filters));
let mut statement = self.connection.prepare(&sql)?;
let matches = statement
.query_map(params_from_iter(values), |row| {
@@ -715,12 +863,16 @@ impl SearchIndex {
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
trace.counter(sql_span, "rows", matches.len() as u64);
trace.end(sql_span, Default::default());
for (id, result) in matches {
selected.push(id);
results.push(result);
}
trace.end(lane, Default::default());
}
if !results.is_empty() {
let source_span = trace.begin("lexical.source_locations", parent);
// Ownership has a B-tree index. The rowid IN constraint intersects
// the FTS postings with only the actual finalists' passage IDs.
let placeholders = (0..results.len())
@@ -762,6 +914,9 @@ impl SearchIndex {
result.snippet = snippet.clone();
}
}
trace.counter(source_span, "files", results.len() as u64);
trace.counter(source_span, "matched_sources", sources.len() as u64);
trace.end(source_span, Default::default());
}
Ok(results)
}
@@ -856,6 +1011,100 @@ fn substring_anchor(characters: &[char], frequencies: &[(i64, String)]) -> Strin
#[cfg(test)]
mod tests {
#[test]
fn measured_search_preserves_results_and_contains_private_free_nested_spans(
) -> anyhow::Result<()> {
let directory = tempfile::tempdir()?;
let root = directory.path().join("sensitive-private-location");
std::fs::create_dir(&root)?;
std::fs::write(
root.join("agreement.txt"),
"confidentialneedle cancellation policy",
)?;
let db = directory.path().join("index.db");
let mut index = super::SearchIndex::open(&db)?;
index.index_folder(&root)?;
let basic = index.search_measured("confidentialneedle", 10, false)?;
let debug = index.search_measured("confidentialneedle", 10, true)?;
assert_eq!(
serde_json::to_value(&basic.results)?,
serde_json::to_value(&debug.results)?
);
assert_eq!(basic.metrics.result_count, 1);
assert!(basic.metrics.trace.is_none());
let spans = debug.metrics.trace.as_ref().unwrap();
assert!(spans.iter().any(|s| s.name == "lexical.content"));
assert!(spans.iter().any(|s| s.name == "lexical.source_locations"));
for span in spans {
assert!(span.start_ms >= 0.0 && span.duration_ms >= 0.0);
assert!(span.start_ms + span.duration_ms <= debug.metrics.total_ms + 1e-6);
if let Some(parent) = span.parent {
let parent = &spans[parent];
assert!(span.start_ms >= parent.start_ms);
assert!(
span.start_ms + span.duration_ms <= parent.start_ms + parent.duration_ms + 1e-6
);
}
}
let trace = serde_json::to_string(spans)?;
assert!(!trace.contains("confidentialneedle"));
assert!(!trace.contains("sensitive-private-location"));
Ok(())
}
#[test]
fn genuine_media_tags_search_duration_filter_and_lifecycle() -> anyhow::Result<()> {
let directory = tempfile::tempdir()?;
let root = directory.path().join("music");
std::fs::create_dir(&root)?;
let file = root.join("track.ogg");
std::fs::write(&file, b"storage fixture")?;
let db = directory.path().join("index.db");
let mut index = super::SearchIndex::open(&db)?;
index.index_folder(&root)?;
let path = file.canonicalize()?.to_string_lossy().into_owned();
let (size, mtime): (u64, i64) = index.connection.query_row(
"SELECT size,mtime FROM files WHERE path=?1",
[&path],
|r| Ok((r.get(0)?, r.get(1)?)),
)?;
index.set_media_metadata(
&path,
"artist: Kimiko Ishizaka\ntitle: Variatio 13 a 2 Clav.",
180000,
size,
mtime,
)?;
let results = index.search("Kimiko Ishizaka", 10)?;
assert_eq!(results.len(), 1);
assert_eq!(results[0].match_kind, "media_metadata");
assert!(results[0].snippet.contains("Kimiko"));
let too_short = super::SearchOptions {
max_duration_ms: Some(100000),
..Default::default()
};
assert!(index
.search_with_options("track.ogg", 10, &too_short)?
.is_empty());
let correct = super::SearchOptions {
min_duration_ms: Some(170000),
max_duration_ms: Some(190000),
..Default::default()
};
assert_eq!(index.search_with_options("Kimiko", 10, &correct)?.len(), 1);
drop(index);
let mut index = super::SearchIndex::open(&db)?;
assert_eq!(index.search("Variatio", 10)?.len(), 1);
std::fs::write(&file, b"changed file bytes")?;
index.index_folder(&root)?;
assert!(index.search("Kimiko", 10)?.is_empty());
assert!(index
.set_media_metadata(&path, "stale", 180000, size, mtime)
.is_err());
std::fs::remove_file(&file)?;
index.index_folder(&root)?;
assert!(index.search("Variatio", 10)?.is_empty());
Ok(())
}
use super::*;
#[test]
fn persistence_updates_deletes_and_exact_priority() -> Result<()> {
crates/searchi-core/src/metrics.rs
+292
View File
@@ -0,0 +1,292 @@
//! Monotonic request metrics. Stage names are static labels, never queries/paths.
use serde::{Deserialize, Serialize};
use std::{
collections::BTreeMap,
sync::atomic::{AtomicU64, Ordering},
time::Instant,
};
const MAX_SPANS: usize = 512;
static COLLECTOR_ID: AtomicU64 = AtomicU64::new(1);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SearchResponse<T> {
pub results: Vec<T>,
pub metrics: SearchMetrics,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SearchMetrics {
pub total_ms: f64,
pub result_count: usize,
pub trace: Option<Vec<SearchSpan>>,
#[serde(default)]
pub dropped_spans: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SearchSpan {
pub name: String,
pub start_ms: f64,
pub duration_ms: f64,
pub parent: Option<usize>,
pub counters: BTreeMap<String, u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MetricsMode {
Basic,
Debug,
}
#[derive(Debug, Clone, Copy)]
pub struct SpanToken {
collector: u64,
index: usize,
}
impl SpanToken {
pub fn index(self) -> usize {
self.index
}
}
struct OpenSpan {
value: SearchSpan,
started: Instant,
ended: bool,
}
struct DebugTrace {
id: u64,
spans: Vec<OpenSpan>,
dropped: u64,
}
/// Basic mode stores only one Instant: no span Vec or stage/counter allocation.
pub struct TraceCollector {
started: Instant,
debug: Option<DebugTrace>,
}
impl TraceCollector {
pub fn new(mode: MetricsMode) -> Self {
Self {
started: Instant::now(),
debug: match mode {
MetricsMode::Basic => None,
MetricsMode::Debug => Some(DebugTrace {
id: COLLECTOR_ID.fetch_add(1, Ordering::Relaxed),
spans: Vec::new(),
dropped: 0,
}),
},
}
}
pub fn elapsed_ms(&self) -> f64 {
self.started.elapsed().as_secs_f64() * 1000.0
}
pub fn is_debug(&self) -> bool {
self.debug.is_some()
}
pub fn begin(&mut self, name: &'static str, parent: Option<SpanToken>) -> Option<SpanToken> {
let debug = self.debug.as_mut()?;
let parent_index = match parent {
Some(token)
if token.collector == debug.id
&& debug.spans.get(token.index).is_some_and(|span| !span.ended) =>
{
Some(token.index)
}
Some(_) => {
debug.dropped = debug.dropped.saturating_add(1);
return None;
}
None => None,
};
if debug.spans.len() >= MAX_SPANS {
debug.dropped = debug.dropped.saturating_add(1);
return None;
}
let started = Instant::now();
let index = debug.spans.len();
debug.spans.push(OpenSpan {
value: SearchSpan {
name: name.chars().take(96).collect(),
start_ms: started.duration_since(self.started).as_secs_f64() * 1000.0,
duration_ms: 0.0,
parent: parent_index,
counters: BTreeMap::new(),
},
started,
ended: false,
});
Some(SpanToken {
collector: debug.id,
index,
})
}
/// Invalid/previously ended tokens are harmless. Closing a parent also
/// closes open descendants at the same instant, preserving containment.
pub fn end(&mut self, token: Option<SpanToken>, counters: BTreeMap<String, u64>) {
let Some(token) = token else {
return;
};
let Some(debug) = self.debug.as_mut() else {
return;
};
if token.collector != debug.id
|| token.index >= debug.spans.len()
|| debug.spans[token.index].ended
{
return;
}
let now = Instant::now();
for index in token.index..debug.spans.len() {
let mut ancestor = Some(index);
let mut contained = false;
while let Some(current) = ancestor {
if current == token.index {
contained = true;
break;
}
ancestor = debug.spans[current].value.parent;
}
if contained && !debug.spans[index].ended {
let span = &mut debug.spans[index];
span.value.duration_ms = now.duration_since(span.started).as_secs_f64() * 1000.0;
span.ended = true;
}
}
// At most32 distinct bounded keys; callers should use static diagnostic
// names and counts, never source text or paths.
let stored = &mut debug.spans[token.index].value.counters;
for (key, value) in counters.into_iter().take(32) {
let key: String = key.chars().take(96).collect();
if stored.len() < 32 || stored.contains_key(&key) {
stored.insert(key, value);
}
}
}
pub fn counter(&mut self, token: Option<SpanToken>, name: &'static str, value: u64) {
let Some(token) = token else {
return;
};
let Some(debug) = self.debug.as_mut() else {
return;
};
if token.collector != debug.id {
return;
}
let Some(span) = debug.spans.get_mut(token.index) else {
return;
};
if span.value.counters.len() < 32 || span.value.counters.contains_key(name) {
span.value
.counters
.insert(name.chars().take(96).collect(), value);
}
}
/// Return the closure's value unchanged, including any Result error.
/// The closure receives its parent token for safe nested instrumentation.
pub fn timed<R>(
&mut self,
name: &'static str,
parent: Option<SpanToken>,
run: impl FnOnce(&mut Self, Option<SpanToken>) -> R,
) -> R {
let token = self.begin(name, parent);
let value = run(self, token);
self.end(token, BTreeMap::new());
value
}
pub fn finish(mut self, result_count: usize) -> SearchMetrics {
let now = Instant::now();
if let Some(debug) = self.debug.as_mut() {
for span in &mut debug.spans {
if !span.ended {
span.value.duration_ms =
now.duration_since(span.started).as_secs_f64() * 1000.0;
span.ended = true;
}
}
}
let (trace, dropped_spans) = match self.debug {
Some(debug) => (
Some(debug.spans.into_iter().map(|span| span.value).collect()),
debug.dropped,
),
None => (None, 0),
};
SearchMetrics {
total_ms: now.duration_since(self.started).as_secs_f64() * 1000.0,
result_count,
trace,
dropped_spans,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn nesting_contains_children_and_preserves_error() {
let mut trace = TraceCollector::new(MetricsMode::Debug);
let result: Result<(), &str> = trace.timed("request", None, |trace, parent| {
trace.timed("index", parent, |trace, child| {
trace.counter(child, "rows", 7);
Err("failure")
})
});
assert_eq!(result, Err("failure"));
let metrics = trace.finish(0);
let spans = metrics.trace.unwrap();
assert_eq!(spans[1].parent, Some(0));
assert_eq!(spans[1].counters["rows"], 7);
assert!(spans[1].start_ms >= spans[0].start_ms);
assert!(
spans[1].start_ms + spans[1].duration_ms
<= spans[0].start_ms + spans[0].duration_ms + 1e-6
);
assert!(metrics.total_ms >= spans[0].start_ms + spans[0].duration_ms - 1e-6);
}
#[test]
fn basic_disables_stage_storage_and_preserves_counts() {
let mut trace = TraceCollector::new(MetricsMode::Basic);
assert!(!trace.is_debug());
assert!(trace.begin("unused", None).is_none());
assert_eq!(trace.timed("unused", None, |_, _| 42), 42);
let metrics = trace.finish(3);
assert_eq!(metrics.result_count, 3);
assert!(metrics.trace.is_none());
assert_eq!(metrics.dropped_spans, 0);
}
#[test]
fn trace_is_capped_and_reports_dropped_spans() {
let mut trace = TraceCollector::new(MetricsMode::Debug);
for index in 0..520 {
let token = trace.begin("stage", None);
if index == 0 {
trace.counter(token, "existing", 99);
}
let counters = if index == 0 {
(0..40).map(|key| (format!("counter_{key}"), key)).collect()
} else {
BTreeMap::new()
};
trace.end(token, counters);
}
let metrics = trace.finish(2);
let spans = metrics.trace.unwrap();
assert_eq!(spans.len(), 512);
assert_eq!(spans[0].counters.len(), 32);
assert_eq!(spans[0].counters["existing"], 99);
assert_eq!(metrics.dropped_spans, 8);
}
#[test]
fn foreign_tokens_and_early_parent_close_are_safe() {
let mut one = TraceCollector::new(MetricsMode::Debug);
let mut two = TraceCollector::new(MetricsMode::Debug);
let root = one.begin("root", None);
let child = one.begin("child", root);
assert!(two.begin("foreign", root).is_none());
one.end(root, BTreeMap::new());
one.end(child, BTreeMap::new());
let spans = one.finish(0).trace.unwrap();
assert!(
spans[1].start_ms + spans[1].duration_ms
<= spans[0].start_ms + spans[0].duration_ms + 1e-6
);
}
}
crates/searchi-media/Cargo.toml
+14
View File
@@ -0,0 +1,14 @@
[package]
name = "searchi-media"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Bounded local CPU media probing and timestamp-preserving sampling"
[dependencies]
anyhow.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2 = "0.10"
tempfile = "3"
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "bmp"] }
crates/searchi-media/README.md
+70
View File
@@ -0,0 +1,70 @@
# Local media preparation
`searchi-media` prepares actual files for the Google multimodal embedder. It
does not download programs or models and has no UI dependency. Callers supply
verified packaged FFmpeg/ffprobe paths through `FfmpegTools::new`, or explicitly
set both `SEARCHI_FFMPEG_PATH` and `SEARCHI_FFPROBE_PATH`.
FFmpeg handles maintained codecs and container timestamps; serde parses its
bounded JSON, tempfile owns frame cleanup, and sha2 validates file identity.
The embedding crate uses the Rust image library for bounded raster decoding.
USearch supplies the service's persisted HNSW index; no codec or vector index
is reimplemented here.
Every FFmpeg/ffprobe media input restricts protocols to `file,pipe`. A selected
local playlist cannot fetch HTTP/HTTPS/RTSP references. A real loopback-listener
regression verifies probing and both decoding paths never connect. This is a
protocol restriction, not a filesystem sandbox: explicitly selected files on
network-mounted drives still use their operating system's filesystem access.
Windows VM tests use CPU only. Decoder commands specify one thread and
`-hwaccel none`; subprocess windows are hidden. Portable argument arrays avoid
shell interpretation. Production hardware decode and reliable CPU fallback
remain unimplemented; this crate does not silently enable accelerators.
Supported images are PNG, JPEG, WebP and BMP (actual bounded embedding-library
decode, including EXIF orientation). AVIF, HEIC, TIFF, animated GIF and embedded
album artwork are not supported yet. Movie containers include MP4, MKV, MOV,
AVI, WebM, M4V, MPEG/MPG; audio includes MP3, FLAC, WAV, M4A, Ogg, Opus, AAC,
AIFF. Actual codec support comes from the packaged FFmpeg build. Unknown or
corrupt files remain filename-searchable with explicit indexing errors.
Default limits: 16 GiB media input; four-hour known duration; 16 million source
video pixels; 45 seconds per decoder command; 64 KiB ffprobe JSON; 256 KiB
frame diagnostic output; 8 MiB encoded PNG frame. Images are separately limited
to 64 MiB input. Captured pipe output is bounded and timed processes are killed.
Process address-space sandboxing remains a release gate; these limits do not
guarantee a codec's internal allocations fit a fixed RAM ceiling.
Video samples span the whole timeline at two-second intervals, becoming coarser
when the default 240 distributed-frame budget is reached. Up to 120 additional
scene-cut frames are captured within 1.5-second windows around sample anchors.
This is **partial visual coverage**, not exhaustive scene detection. Actual
decoded presentation timestamps are retained relative to container start. The
primary non-cover-art video stream is selected. Full images use model budget
280; video frames use 70. No fabricated captions are stored.
The primary audio stream is decoded to mono 16 kHz PCM, embedded in 30-second
windows with two seconds of overlap. The complete stream is covered or the
600-window budget returns an explicit failure. Standalone music uses the same
real audio embedding; artist/title/album/genre tags are merged from genuine
stream and format metadata and stored separately for lexical search. Audio
embeddings are not transcripts; exact spoken-word retrieval is not provided.
Alternative language tracks and isolated channels remain a release gate.
The service publishes text, image, video and audio vectors as one generation.
It persists origin and page/timestamp locations, policy/model fingerprints,
full file SHA at indexing, and size/mtime guards. Corrupt-file failures discard
that file's partial vectors and persist reasons. Model inference failure aborts
the generation. At query time only actual media candidate hits are stat-checked;
no corpus-wide hash/stat scan occurs. Changes preserving both size and mtime
cannot be detected by this fast guard; periodic content-hash reconciliation is
a release gate. Builds currently rebuild a complete generation, with a
100,000-vector / 10,000-media-file bound; unchanged-vector reuse is future work.
Offline tests cover complete audio windows, frame budgets and identity changes.
The ignored `real_decoder_preserves_frame_timestamps_and_audio` test has been
executed successfully in the CPU Windows VM with provisioned FFmpeg 9.0.2.
It creates genuine FFV1/PCM media, validates decoded two-second PTS and actual
mono sine samples. Linux/macOS execution and installer codec/license delivery
have not been validated.
crates/searchi-media/src/lib.rs
+789
View File
@@ -0,0 +1,789 @@
//! Local bounded media preparation. No network, model, executable installer, or
//! fabricated caption/transcript lives here. FFmpeg paths come from the caller.
//!
//! Video coverage is distributed sampling, not a guarantee that every scene is
//! represented. Audio windows cover the complete primary stream or return an
//! explicit budget error. Decoder process memory is not sandbox-limited yet.
use anyhow::{bail, ensure, Context, Result};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
collections::BTreeMap,
fs::{self, File},
io::Read,
path::{Path, PathBuf},
process::{Command, Stdio},
thread,
time::{Duration, Instant, UNIX_EPOCH},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum MediaKind {
Image,
Video,
Audio,
}
#[cfg(test)]
mod tests {
use super::*;
fn fixture(duration_ms: u32) -> MediaProbe {
MediaProbe {
duration_ms,
start_ms: 0,
video_stream_index: 0,
width: 64,
height: 64,
has_video: true,
has_audio: true,
audio_start_ms: 0,
audio_end_ms: duration_ms,
tags: BTreeMap::new(),
}
}
#[test]
fn audio_covers_tail_with_overlap_and_rejects_budget() -> Result<()> {
let probe = fixture(65_000);
let limits = MediaLimits::default();
let windows = audio_windows(&probe, &limits)?;
assert_eq!(
windows.iter().map(|w| w.start_ms).collect::<Vec<_>>(),
vec![0, 28_000, 56_000]
);
assert_eq!(windows.last().unwrap().duration_ms, 9_000);
let tight = MediaLimits {
max_audio_windows: 2,
..limits
};
assert!(audio_windows(&probe, &tight).is_err());
Ok(())
}
#[test]
fn finer_audio_windows_cover_long_movie_or_explicitly_reject() -> Result<()> {
let probe = fixture(3 * 60 * 60 * 1000);
let limits = MediaLimits {
audio_window_ms: 10_000,
audio_overlap_ms: 2000,
max_audio_windows: 1800,
..Default::default()
};
let windows = audio_windows(&probe, &limits)?;
assert_eq!(windows.first().unwrap().start_ms, 0);
let last = windows.last().unwrap();
assert_eq!(last.start_ms + last.duration_ms, probe.duration_ms);
assert!(windows.len() >= 1300 && windows.len() <= 1440);
let too_small = MediaLimits {
max_audio_windows: 1000,
..limits
};
assert!(audio_windows(&probe, &too_small).is_err());
Ok(())
}
#[test]
fn frame_budget_spans_entire_movie() -> Result<()> {
let probe = fixture(3_600_000);
let times = frame_times(&probe, &MediaLimits::default())?;
assert!(times.len() <= 240);
assert_eq!(times.first(), Some(&0));
assert_eq!(times.last(), Some(&3_599_500));
Ok(())
}
#[test]
fn identity_rejects_changes_and_oversized_input() -> Result<()> {
let temp = tempfile::tempdir()?;
let file = temp.path().join("audio.wav");
fs::write(&file, b"first")?;
assert!(MediaIdentity::read(&file, 4).is_err());
let identity = MediaIdentity::read(&file, 10)?;
fs::write(&file, b"changed size")?;
assert!(identity.ensure_unchanged().is_err());
Ok(())
}
#[test]
#[ignore = "Requires explicitly provisioned CPU FFmpeg/ffprobe executables"]
fn decoder_rejects_network_references_inside_local_media() -> Result<()> {
let tools = FfmpegTools::from_environment()?.context("Set CPU decoder paths")?;
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
listener.set_nonblocking(true)?;
let temp = tempfile::tempdir()?;
let file = temp.path().join("local.m3u8");
fs::write(&file,format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:2\n#EXTINF:2.0,\nhttp://{}/segment.ts\n#EXT-X-ENDLIST\n",listener.local_addr()?))?;
let limits = MediaLimits {
command_timeout: Duration::from_secs(3),
..Default::default()
};
let error = super::probe(&tools, &file, &limits).unwrap_err();
assert!(format!("{error:#}").contains("whitelist"), "{error:#}");
assert!(decode_frames(&tools, &file, &fixture(2000), 0, false, &limits).is_err());
assert!(decode_audio(
&tools,
&file,
AudioWindow {
start_ms: 0,
duration_ms: 2000
},
&limits
)
.is_err());
match listener.accept() {
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {}
other => panic!("Decoder attempted network connection: {other:?}"),
}
Ok(())
}
#[test]
#[ignore = "Requires explicitly provisioned CPU FFmpeg/ffprobe executables"]
fn real_decoder_preserves_frame_timestamps_and_audio() -> Result<()> {
let tools = FfmpegTools::from_environment()?.context("Set CPU decoder paths")?;
let temp = tempfile::tempdir()?;
let video = temp.path().join("fixture.mkv");
let mut command = Command::new(&tools.ffmpeg);
command
.args([
"-nostdin",
"-v",
"error",
"-threads",
"1",
"-f",
"lavfi",
"-i",
"color=c=red:s=64x64:r=10:d=4",
"-f",
"lavfi",
"-i",
"sine=frequency=440:sample_rate=16000:duration=4",
"-c:v",
"ffv1",
"-c:a",
"pcm_s16le",
"-threads",
"1",
"-y",
])
.arg(&video);
run(command, Duration::from_secs(30), 4096, 65536)?;
let limits = MediaLimits::default();
let probe = probe(&tools, &video, &limits)?;
assert!(probe.has_video && probe.has_audio);
let frames = decode_frames(&tools, &video, &probe, 2000, false, &limits)?;
assert_eq!(frames.frames.len(), 1);
assert!((1900..=2100).contains(&frames.frames[0].timestamp_ms));
let audio = decode_audio(
&tools,
&video,
AudioWindow {
start_ms: 1000,
duration_ms: 2000,
},
&limits,
)?;
assert!((31_000..=32_000).contains(&audio.len()));
assert!(audio.iter().any(|x| x.abs() > 0.01));
assert!(audio.iter().all(|x| x.is_finite() && x.abs() <= 1.0));
let ogg = temp.path().join("overshoot.ogg");
let mut command = Command::new(&tools.ffmpeg);
command
.args([
"-nostdin",
"-v",
"error",
"-threads",
"1",
"-f",
"lavfi",
"-i",
"sine=frequency=440:sample_rate=44100:duration=31.123",
"-c:a",
"libvorbis",
"-metadata",
"artist=CPU decoder fixture",
"-threads",
"1",
"-y",
])
.arg(&ogg);
run(command, Duration::from_secs(30), 4096, 65536)?;
let music_probe = super::probe(&tools, &ogg, &limits)?;
assert_eq!(
music_probe.tags.get("artist").map(String::as_str),
Some("CPU decoder fixture")
);
let pcm = decode_audio(
&tools,
&ogg,
AudioWindow {
start_ms: 0,
duration_ms: 30000,
},
&limits,
)?;
assert_eq!(
pcm.len(),
480000,
"Vorbis packets must not overshoot model limit"
);
Ok(())
}
}
pub fn classify(path: &Path) -> Option<MediaKind> {
match path.extension()?.to_str()?.to_ascii_lowercase().as_str() {
"png" | "jpg" | "jpeg" | "webp" | "bmp" => Some(MediaKind::Image),
"mp4" | "mkv" | "mov" | "avi" | "webm" | "m4v" | "mpeg" | "mpg" => Some(MediaKind::Video),
"mp3" | "flac" | "wav" | "m4a" | "ogg" | "opus" | "aac" | "aiff" | "aif" => {
Some(MediaKind::Audio)
}
_ => None,
}
}
/// Validate bounded raster decoding before treating model errors as fatal.
/// The provider still performs its own bounds and EXIF-aware preprocessing.
pub fn validate_image(path: &Path) -> Result<()> {
ensure!(
fs::metadata(path)?.len() <= 64 * 1024 * 1024,
"Image exceeds 64 MiB"
);
let reader = image::ImageReader::open(path)?.with_guessed_format()?;
let (width, height) = reader.into_dimensions()?;
ensure!(
width <= 16384 && height <= 16384 && u64::from(width) * u64::from(height) <= 16_000_000,
"Image exceeds 16M pixels"
);
let mut reader = image::ImageReader::open(path)?.with_guessed_format()?;
let mut limits = image::Limits::default();
limits.max_alloc = Some(256 * 1024 * 1024);
limits.max_image_width = Some(16384);
limits.max_image_height = Some(16384);
reader.limits(limits);
use image::ImageDecoder;
let mut decoder = reader.into_decoder()?;
decoder
.orientation()
.context("Invalid image EXIF orientation")?;
image::DynamicImage::from_decoder(decoder).context("Image decoder rejected corrupt content")?;
Ok(())
}
#[derive(Debug, Clone)]
pub struct FfmpegTools {
pub ffmpeg: PathBuf,
pub ffprobe: PathBuf,
}
impl FfmpegTools {
pub fn new(ffmpeg: impl AsRef<Path>, ffprobe: impl AsRef<Path>) -> Result<Self> {
ensure!(
ffmpeg.as_ref().is_file() && ffprobe.as_ref().is_file(),
"FFmpeg/ffprobe executable is missing"
);
Ok(Self {
ffmpeg: fs::canonicalize(ffmpeg)?,
ffprobe: fs::canonicalize(ffprobe)?,
})
}
pub fn from_environment() -> Result<Option<Self>> {
match (
std::env::var_os("SEARCHI_FFMPEG_PATH"),
std::env::var_os("SEARCHI_FFPROBE_PATH"),
) {
(None, None) => Ok(None),
(Some(ffmpeg), Some(ffprobe)) => Self::new(ffmpeg, ffprobe).map(Some),
_ => bail!("Set both SEARCHI_FFMPEG_PATH and SEARCHI_FFPROBE_PATH"),
}
}
}
#[derive(Debug, Clone)]
pub struct MediaLimits {
pub max_file_bytes: u64,
pub max_duration_ms: u32,
pub max_frames: usize,
pub frame_interval_ms: u32,
pub max_scene_frames: usize,
pub audio_window_ms: u32,
pub audio_overlap_ms: u32,
pub max_audio_windows: usize,
pub command_timeout: Duration,
}
impl Default for MediaLimits {
fn default() -> Self {
Self {
max_file_bytes: 16 * 1024 * 1024 * 1024,
max_duration_ms: 4 * 60 * 60 * 1000,
max_frames: 240,
frame_interval_ms: 2000,
max_scene_frames: 120,
audio_window_ms: 30_000,
audio_overlap_ms: 2000,
max_audio_windows: 600,
command_timeout: Duration::from_secs(45),
}
}
}
#[derive(Debug, Clone, Serialize)]
pub struct MediaIdentity {
pub path: PathBuf,
pub bytes: u64,
pub modified_ns: i64,
pub sha256: String,
}
impl MediaIdentity {
pub fn read(path: &Path, maximum: u64) -> Result<Self> {
let file = File::open(path)?;
let before = file.metadata()?;
ensure!(
before.len() <= maximum,
"Media file exceeds {maximum} bytes"
);
let modified_ns = modified_ns(&before)?;
let mut reader = file.take(maximum.saturating_add(1));
let mut buffer = [0u8; 65536];
let mut hash = Sha256::new();
let mut count = 0u64;
loop {
let read = reader.read(&mut buffer)?;
if read == 0 {
break;
}
count += read as u64;
ensure!(count <= maximum, "Media grew past file budget");
hash.update(&buffer[..read]);
}
let identity = Self {
path: fs::canonicalize(path)?,
bytes: count,
modified_ns,
sha256: format!("{:x}", hash.finalize()),
};
ensure!(count == before.len(), "Media changed while fingerprinting");
identity.ensure_unchanged()?;
Ok(identity)
}
pub fn ensure_unchanged(&self) -> Result<()> {
let metadata = fs::metadata(&self.path)?;
ensure!(
metadata.len() == self.bytes && modified_ns(&metadata)? == self.modified_ns,
"Media changed during indexing"
);
Ok(())
}
}
fn modified_ns(metadata: &fs::Metadata) -> Result<i64> {
i64::try_from(metadata.modified()?.duration_since(UNIX_EPOCH)?.as_nanos())
.context("Unsupported media timestamp")
}
#[derive(Debug, Clone, Serialize)]
pub struct MediaProbe {
pub duration_ms: u32,
pub start_ms: i64,
pub video_stream_index: u32,
pub width: u32,
pub height: u32,
pub has_video: bool,
pub has_audio: bool,
pub audio_start_ms: u32,
pub audio_end_ms: u32,
pub tags: BTreeMap<String, String>,
}
#[derive(Deserialize)]
struct ProbeJson {
#[serde(default)]
streams: Vec<ProbeStream>,
format: ProbeFormat,
}
#[derive(Deserialize)]
struct ProbeFormat {
duration: String,
#[serde(default)]
start_time: Option<String>,
#[serde(default)]
tags: BTreeMap<String, String>,
}
#[derive(Deserialize)]
struct ProbeStream {
index: u32,
codec_type: String,
#[serde(default)]
width: u32,
#[serde(default)]
height: u32,
#[serde(default)]
duration: Option<String>,
#[serde(default)]
start_time: Option<String>,
#[serde(default)]
tags: BTreeMap<String, String>,
#[serde(default)]
disposition: BTreeMap<String, i64>,
}
fn milliseconds(value: &str) -> Result<i64> {
let value: f64 = value.parse()?;
ensure!(
value.is_finite() && value.abs() < 100_000_000.0,
"Invalid media duration/timestamp"
);
Ok((value * 1000.0).round() as i64)
}
pub fn probe(tools: &FfmpegTools, path: &Path, limits: &MediaLimits) -> Result<MediaProbe> {
let mut command = Command::new(&tools.ffprobe);
command.args(["-protocol_whitelist", "file,pipe"]);
command
.args([
"-v",
"error",
"-threads",
"1",
"-probesize",
"5000000",
"-analyzeduration",
"5000000",
"-show_streams",
"-show_format",
"-of",
"json",
])
.arg(path);
let output = run(command, limits.command_timeout, 65536, 65536)?;
let parsed: ProbeJson =
serde_json::from_slice(&output.stdout).context("Invalid ffprobe JSON")?;
let duration = milliseconds(&parsed.format.duration)?;
ensure!(
duration > 0 && duration <= i64::from(limits.max_duration_ms),
"Media duration exceeds policy or is unknown"
);
let start = parsed
.format
.start_time
.as_deref()
.map(milliseconds)
.transpose()?
.unwrap_or(0);
let video = parsed.streams.iter().find(|stream| {
stream.codec_type == "video" && stream.disposition.get("attached_pic") != Some(&1)
});
if let Some(video) = video {
ensure!(
u64::from(video.width) * u64::from(video.height) <= 16_000_000,
"Video dimensions exceed 16M pixels"
);
}
let audio = parsed
.streams
.iter()
.find(|stream| stream.codec_type == "audio");
let audio_start = audio
.and_then(|stream| stream.start_time.as_deref())
.map(milliseconds)
.transpose()?
.unwrap_or(start)
- start;
let audio_length = audio
.and_then(|stream| stream.duration.as_deref())
.map(milliseconds)
.transpose()?
.unwrap_or(duration);
let mut tags = parsed.format.tags;
for stream in &parsed.streams {
for (key, value) in &stream.tags {
tags.entry(key.clone()).or_insert_with(|| value.clone());
}
}
tags = tags
.into_iter()
.map(|(key, value)| {
(
key.to_ascii_lowercase(),
value.chars().take(2048).collect::<String>(),
)
})
.filter(|(key, _)| {
matches!(
key.as_str(),
"title"
| "artist"
| "album"
| "album_artist"
| "genre"
| "comment"
| "date"
| "composer"
| "description"
| "track"
)
})
.take(16)
.collect();
Ok(MediaProbe {
duration_ms: duration as u32,
start_ms: start,
video_stream_index: video.map_or(0, |stream| stream.index),
width: video.map_or(0, |stream| stream.width),
height: video.map_or(0, |stream| stream.height),
has_video: video.is_some(),
has_audio: audio.is_some(),
audio_start_ms: audio_start.max(0) as u32,
audio_end_ms: (audio_start + audio_length).clamp(0, duration) as u32,
tags,
})
}
/// Uniform coverage is distributed over the entire movie; this is not a prefix
/// truncation. Scene probes may add extra local cuts around these anchors.
pub fn frame_times(probe: &MediaProbe, limits: &MediaLimits) -> Result<Vec<u32>> {
if !probe.has_video {
return Ok(Vec::new());
}
ensure!(
limits.max_frames >= 2 && limits.max_frames <= 2000,
"Frame budget must be 2..2000"
);
let last = probe.duration_ms.saturating_sub(500);
let interval = limits
.frame_interval_ms
.max(last.div_ceil((limits.max_frames - 1) as u32))
.max(1);
let mut times = Vec::new();
let mut time = 0;
while time < last {
times.push(time);
time = time.saturating_add(interval);
}
times.push(last);
ensure!(
times.len() <= limits.max_frames,
"Frame sampling budget exceeded"
);
Ok(times)
}
#[derive(Debug, Clone, Copy, Serialize)]
pub struct AudioWindow {
pub start_ms: u32,
pub duration_ms: u32,
}
pub fn audio_windows(probe: &MediaProbe, limits: &MediaLimits) -> Result<Vec<AudioWindow>> {
if !probe.has_audio {
return Ok(Vec::new());
}
ensure!(
limits.audio_window_ms >= 5_000
&& limits.audio_window_ms <= 30_000
&& limits.audio_overlap_ms < limits.audio_window_ms,
"Invalid audio window policy"
);
let step = limits.audio_window_ms - limits.audio_overlap_ms;
let mut windows = Vec::new();
let mut start = probe.audio_start_ms;
while start < probe.audio_end_ms {
ensure!(
windows.len() < limits.max_audio_windows,
"Whole audio coverage exceeds window budget; no partial index produced"
);
windows.push(AudioWindow {
start_ms: start,
duration_ms: limits.audio_window_ms.min(probe.audio_end_ms - start),
});
if start + limits.audio_window_ms >= probe.audio_end_ms {
break;
}
start = start.saturating_add(step);
}
Ok(windows)
}
pub struct DecodedFrame {
pub path: PathBuf,
pub timestamp_ms: u32,
}
pub struct FrameBatch {
directory: tempfile::TempDir,
pub frames: Vec<DecodedFrame>,
}
impl FrameBatch {
pub fn directory(&self) -> &Path {
self.directory.path()
}
}
pub fn decode_frames(
tools: &FfmpegTools,
path: &Path,
probe: &MediaProbe,
start_ms: u32,
include_scene: bool,
limits: &MediaLimits,
) -> Result<FrameBatch> {
let directory = tempfile::tempdir()?;
let mut command = Command::new(&tools.ffmpeg);
command.args(["-protocol_whitelist", "file,pipe"]);
command.args(["-nostdin","-hide_banner","-loglevel","info","-threads","1","-hwaccel","none","-copyts","-ss"])
.arg(format!("{:.3}",f64::from(start_ms)/1000.0)).args(["-t",if include_scene {"1.5"} else {"0.5"},"-i"]).arg(path)
.arg("-map").arg(format!("0:{}",probe.video_stream_index)).args(["-an","-filter_threads","1","-vf"])
.arg(if include_scene {"select=eq(n\\,0)+gt(scene\\,0.32),showinfo,scale=960:960:force_original_aspect_ratio=decrease"}
else {"showinfo,scale=960:960:force_original_aspect_ratio=decrease"})
.args(["-frames:v",if include_scene {"2"} else {"1"},"-fps_mode","vfr","-threads","1","-y"])
.arg(directory.path().join("frame-%02d.png"));
let output = run(command, limits.command_timeout, 4096, 256 * 1024)?;
let mut timestamps = Vec::new();
for line in String::from_utf8_lossy(&output.stderr).lines() {
if line.contains("showinfo") {
if let Some((_, tail)) = line.split_once("pts_time:") {
let value = tail
.split_whitespace()
.next()
.context("Missing frame PTS")?;
let actual = (milliseconds(value)? - probe.start_ms).max(0);
ensure!(
actual <= i64::from(probe.duration_ms) + 1000,
"Frame PTS lies beyond media timeline"
);
timestamps.push(actual as u32);
}
}
}
let mut frames = Vec::new();
for ordinal in 1..=if include_scene { 2 } else { 1 } {
let file = directory.path().join(format!("frame-{ordinal:02}.png"));
if file.is_file() {
ensure!(
fs::metadata(&file)?.len() <= 8 * 1024 * 1024,
"Decoded frame exceeds 8MiB"
);
let timestamp_ms = *timestamps
.get(ordinal - 1)
.context("Decoder did not report genuine frame PTS")?;
frames.push(DecodedFrame {
path: file,
timestamp_ms,
});
}
}
ensure!(
!frames.is_empty(),
"Decoder produced no frame at requested timeline position"
);
Ok(FrameBatch { directory, frames })
}
pub fn decode_audio(
tools: &FfmpegTools,
path: &Path,
window: AudioWindow,
limits: &MediaLimits,
) -> Result<Vec<f32>> {
ensure!(
window.duration_ms > 0 && window.duration_ms <= 30_000,
"Audio decode window exceeds 30 seconds"
);
let mut command = Command::new(&tools.ffmpeg);
command.args(["-protocol_whitelist", "file,pipe"]);
command
.args([
"-nostdin",
"-hide_banner",
"-v",
"error",
"-threads",
"1",
"-ss",
])
.arg(format!("{:.3}", f64::from(window.start_ms) / 1000.0))
.args(["-t"])
.arg(format!("{:.3}", f64::from(window.duration_ms) / 1000.0))
.arg("-i")
.arg(path)
.arg("-af")
.arg(format!(
"aresample=16000,atrim=end_sample={},asetpts=PTS-STARTPTS",
u64::from(window.duration_ms) * 16
))
.args([
"-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000", "-threads", "1", "-f", "s16le",
"pipe:1",
]);
let output = run(command, limits.command_timeout, 960_000, 65536)?;
ensure!(
!output.stdout.is_empty() && output.stdout.len() % 2 == 0,
"Empty or invalid decoded PCM audio"
);
Ok(output
.stdout
.as_chunks::<2>()
.0
.iter()
.map(|bytes| f32::from(i16::from_le_bytes([bytes[0], bytes[1]])) / 32768.0)
.collect())
}
struct Output {
stdout: Vec<u8>,
stderr: Vec<u8>,
}
fn run(
mut command: Command,
timeout: Duration,
stdout_limit: usize,
stderr_limit: usize,
) -> Result<Output> {
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
command.creation_flags(0x08000000);
}
let mut child = command
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("Launch configured FFmpeg tool")?;
let stdout = child.stdout.take().context("Missing decoder stdout")?;
let stderr = child.stderr.take().context("Missing decoder stderr")?;
let read = |reader: Box<dyn Read + Send>, limit: usize| {
thread::spawn(move || -> Result<Vec<u8>> {
let mut bytes = Vec::new();
reader.take(limit as u64 + 1).read_to_end(&mut bytes)?;
ensure!(bytes.len() <= limit, "Decoder output exceeded byte budget");
Ok(bytes)
})
};
let out = read(Box::new(stdout), stdout_limit);
let err = read(Box::new(stderr), stderr_limit);
let start = Instant::now();
let status = loop {
if let Some(status) = child.try_wait()? {
break Some(status);
}
if start.elapsed() > timeout {
child.kill()?;
child.wait()?;
break None;
}
thread::sleep(Duration::from_millis(10));
};
let stdout = out
.join()
.map_err(|_| anyhow::anyhow!("Decoder stdout reader panicked"))??;
let stderr = err
.join()
.map_err(|_| anyhow::anyhow!("Decoder stderr reader panicked"))??;
let status = status.context("Decoder exceeded time budget")?;
ensure!(
status.success(),
"Decoder failed: {}",
String::from_utf8_lossy(&stderr)
.chars()
.take(2048)
.collect::<String>()
);
Ok(Output { stdout, stderr })
}
crates/searchi-semantic/Cargo.toml
+3
View File
@@ -11,6 +11,9 @@ minilm-proof = ["dep:ort", "dep:tokenizers"]
[dependencies]
anyhow.workspace = true
base64 = "0.22"
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "bmp"] }
jpeg-decoder = { version = "0.3.2", default-features = false }
ort = { version = "=2.0.0-rc.13", optional = true, default-features = false, features = ["std", "load-dynamic", "api-27"] }
sha2 = "0.10"
tokenizers = { version = "=0.22.2", optional = true, default-features = false, features = ["fancy-regex"] }
crates/searchi-semantic/README.md
+44
View File
@@ -0,0 +1,44 @@
# Google worker image decoding
The production adapter verifies model/projector SHA256 and performs genuine
Google MTMD image/audio inference. Images are decoded with established Rust
libraries, not filename captions. `WorkerInfo.media_supported` comes from the
native handshake; absent fields on older text workers mean false. Media APIs
return an explicit runtime repair error for such workers while text remains
compatible.
JPEG sources up to 64 Mi pixels (including 48 MP camera photos), 16,384 pixels
per side, and 64 MiB compressed size are supported. For JPEG above 16 Mi pixels,
`image-rs/jpeg-decoder` 0.3.2 applies scaled IDCT before final model resizing.
Its default features are disabled, so this path adds no Rayon worker pool.
The source dimensions are checked before pixels are decoded. The secondary
header must match the first to detect file changes. RGB24 and grayscale camera
JPEGs are supported on this scaled path; unusual large CMYK/16-bit JPEGs fail
explicitly rather than silently taking an unbounded full-resolution fallback.
The published decoder offers 1/8, 1/4, 1/2, and full-scale decoding. It is in
maintenance mode; the current image 0.25 zune JPEG adapter exposes no scaled
IDCT interface. See the [decoder API](https://docs.rs/jpeg-decoder/0.3.2/jpeg_decoder/struct.Decoder.html)
and [upstream maintenance note](https://github.com/image-rs/jpeg-decoder).
Small JPEGs retain the existing image/zune path and pinned parity behavior.
PNG, WebP, and BMP remain capped at 16 Mi pixels with explicit diagnostics.
Both paths enforce a 256 MiB advertised decoding-buffer ceiling and apply
EXIF orientation. That ceiling is a buffer/output bound, not an absolute
process RSS guarantee: compressed data, codec working state, and resizing
have additional allocations. Extreme/progressive files need broader resource
profiling. The media fingerprint includes `largejpeg-scaled-v2`; generations
must additionally include visual token budget. Text fingerprints are unchanged.
An executed Windows CPU release test encodes and decodes an actual 8000x6000
JPEG with EXIF rotation, verifies orientation and region colors, and resizes it
to a 280-token grid. Decode plus resize took 176–220 ms and produced 672x912
RGB (1,838,592 bytes). A sampled whole-test peak was 146.86 MiB, including
144 MB of source fixture construction; it is not a decoder-only measurement
or a semantic quality benchmark. An 81 MP JPEG header is rejected before
pixel decode in the default tests.
```powershell
cargo test --release -p searchi-semantic large_phone_jpeg -- --ignored --nocapture
cargo test -p searchi-semantic --lib
```
crates/searchi-semantic/native/CMakeLists.txt
+3 -2
View File
@@ -41,11 +41,12 @@ set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "No runtime tools" FORCE)
set(LLAMA_BUILD_SERVER OFF CACHE BOOL "No HTTP server" FORCE)
set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "No examples" FORCE)
set(LLAMA_BUILD_TESTS OFF CACHE BOOL "No upstream tests in worker build" FORCE)
set(LLAMA_BUILD_MTMD OFF CACHE BOOL "Text worker; media encoder follows separately" FORCE)
set(LLAMA_BUILD_MTMD ON CACHE BOOL "Google image/audio projector" FORCE)
set(MTMD_VIDEO OFF CACHE BOOL "Frames decoded by bounded media adapter" FORCE)
set(LLAMA_OPENSSL OFF CACHE BOOL "No network runtime dependencies" FORCE)
add_subdirectory("${LLAMA_CPP_DIR}" llama-build)
add_executable(searchi-gemma-worker worker.cpp)
target_include_directories(searchi-gemma-worker PRIVATE "${LLAMA_CPP_DIR}/vendor")
target_link_libraries(searchi-gemma-worker PRIVATE llama)
target_link_libraries(searchi-gemma-worker PRIVATE llama mtmd)
target_compile_definitions(searchi-gemma-worker PRIVATE SEARCHI_LLAMA_TAG="b11461-no-logits-v1")
install(TARGETS searchi-gemma-worker RUNTIME DESTINATION bin)
crates/searchi-semantic/native/worker.cpp
+108 -6
View File
@@ -1,11 +1,14 @@
// Local NDJSON helper around the pinned llama.cpp encoder. stdout is exclusively
// protocol output; llama diagnostics go to stderr. No model download or HTTP.
#include "llama.h"
#include "mtmd.h"
#include "nlohmann/json.hpp"
#include <algorithm>
#include <cmath>
#include <chrono>
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <iostream>
#include <memory>
#include <stdexcept>
@@ -13,7 +16,7 @@
#include <vector>
using json = nlohmann::json;
constexpr size_t max_request_bytes = 2 * 1024 * 1024;
constexpr size_t max_request_bytes = 16 * 1024 * 1024;
constexpr size_t max_passage_bytes = 1024 * 1024;
constexpr size_t max_text_bytes = 65536;
constexpr size_t max_windows = 256;
@@ -34,12 +37,14 @@ class Encoder {
llama_context * context = nullptr;
llama_batch_ext * batch = nullptr;
const llama_vocab * vocab = nullptr;
mtmd_context * media = nullptr;
std::string model_path;
void release() {
if (media) mtmd_free(media);
if (batch) llama_batch_ext_free(batch);
if (context) llama_free(context);
if (model) llama_model_free(model);
batch = nullptr; context = nullptr; model = nullptr; vocab = nullptr;
batch = nullptr; context = nullptr; model = nullptr; vocab = nullptr; media = nullptr;
}
bool open(bool gpu) {
auto mp = llama_model_default_params();
@@ -68,6 +73,9 @@ public:
llama_log_set([](ggml_log_level level, const char * text, void *) {
if (level >= GGML_LOG_LEVEL_WARN) std::fputs(text,stderr);
},nullptr);
mtmd_log_set([](ggml_log_level level, const char * text, void *) {
if (level >= GGML_LOG_LEVEL_WARN) std::fputs(text,stderr);
},nullptr);
llama_backend_init();
bool use_gpu = want_gpu && llama_supports_gpu_offload();
require(open(use_gpu) || (use_gpu && open(false)),"Cannot initialize local EmbeddingGemma 2 model/context");
@@ -77,6 +85,52 @@ public:
llama_backend_free();
}
bool healthy() const {return context != nullptr && batch != nullptr;}
void ensure_media(const std::string & projector) {
if (media) return;
auto params = mtmd_context_params_default();
params.use_gpu = device == "gpu"; params.n_threads = threads;
params.print_timings = false; params.warmup = false;
params.image_min_tokens = 1; params.image_max_tokens = 1120;
media = mtmd_init_from_file(projector.c_str(),model,params);
require_inference(media != nullptr,"Cannot initialize Google media projector");
}
std::vector<float> embed_bitmap(mtmd_bitmap * bitmap) {
std::unique_ptr<mtmd_input_chunks,decltype(&mtmd_input_chunks_free)> chunks(mtmd_input_chunks_init(),mtmd_input_chunks_free);
require_inference(chunks != nullptr,"Cannot allocate media chunks");
const char * marker = mtmd_default_marker();
mtmd_input_text text{marker,std::strlen(marker),true,true};
const mtmd_bitmap * bitmaps[] = {bitmap};
require_inference(mtmd_tokenize(media,chunks.get(),&text,bitmaps,1) == 0,"Media tokenization failed");
llama_batch_ext_clear(batch); size_t position = 0;
auto place = [&](int index) {
require_inference(index >= 0 && position < context_tokens,"Media batch exceeds context");
llama_pos pos[4] = {llama_pos(position++),0,0,0};
require_inference(llama_batch_ext_set_pos(batch,index,pos) && llama_batch_ext_set_output_embd(batch,index,true),"Cannot place media row");
};
for (size_t i=0;i<mtmd_input_chunks_size(chunks.get());++i) {
const auto * chunk = mtmd_input_chunks_get(chunks.get(),i);
if (mtmd_input_chunk_get_type(chunk) == MTMD_INPUT_CHUNK_TYPE_TEXT) {
size_t count=0; const auto * tokens=mtmd_input_chunk_get_tokens_text(chunk,&count);
for(size_t j=0;j<count;++j) place(llama_batch_ext_add_token(batch,0,tokens[j]));
} else {
require_inference(mtmd_encode_chunk(media,chunk)==0,"Google media encoder failed");
const float * rows=mtmd_get_output_embd(media);
require_inference(rows != nullptr,"No projected media rows");
size_t width=llama_model_n_embd_inp(model);
for(size_t j=0;j<mtmd_input_chunk_get_n_tokens(chunk);++j) {
llama_embd embd{rows+j*width,1,width};
place(llama_batch_ext_add_embd(batch,0,embd));
}
}
}
require_inference(llama_process(context,LLAMA_PROCESS_TYPE_ENCODE,batch)==0,"Media text encoder failed");
const float * pooled=llama_get_embeddings_seq(context,0);
require_inference(pooled != nullptr,"No pooled media vector");
double norm=0; for(int i=0;i<dimensions;++i){require_inference(std::isfinite(pooled[i]),"Nonfinite media vector");norm+=double(pooled[i])*pooled[i];}
require_inference(norm>0 && std::isfinite(norm),"Invalid media vector norm");
std::vector<float> vector(dimensions);for(int i=0;i<dimensions;++i)vector[i]=float(pooled[i]/std::sqrt(norm));
return vector;
}
void retry_cpu() {
require(device == "gpu","Only a GPU inference failure can request a CPU retry");
release();
@@ -103,10 +157,21 @@ public:
require(title.size() <= 512, "Document title exceeds 512 UTF-8 bytes");
return "title: " + (title.empty() ? "none" : title) + " | text: ";
}
std::vector<float> embed(const std::string & text, int cap) {
std::vector<float> embed(const std::string & text, int cap, json * timings = nullptr) {
using clock = std::chrono::steady_clock;
auto started = timings ? clock::now() : clock::time_point{};
auto mark = [&](const char * name) {
if (timings) {
auto now = clock::now();
(*timings)[name] = std::chrono::duration_cast<std::chrono::microseconds>(now-started).count();
started = now;
}
};
require(!text.empty() && text.size() <= max_text_bytes, "Embedding text must be nonempty and at most 64 KiB");
auto tokens = tokenize(text,true);
require(!tokens.empty() && tokens.size() <= size_t(cap), "Prompt exceeds model token budget; split document windows first");
if (timings) (*timings)["token_count"] = tokens.size();
mark("tokenization_us");
llama_batch_ext_clear(batch);
for (size_t i = 0; i < tokens.size(); ++i) {
int index = llama_batch_ext_add_token(batch,0,tokens[i]);
@@ -115,7 +180,9 @@ public:
require(llama_batch_ext_set_pos(batch,index,position), "Cannot set encoder position");
require(llama_batch_ext_set_output_embd(batch,index,true), "Cannot select embedding output");
}
mark("batch_prepare_us");
require_inference(llama_process(context,LLAMA_PROCESS_TYPE_ENCODE,batch) == 0, "llama_process encoder failed");
mark("inference_us");
const float * pooled = llama_get_embeddings_seq(context,0);
require_inference(pooled != nullptr, "Encoder returned no pooled vector");
double norm = 0;
@@ -123,6 +190,7 @@ public:
require_inference(norm > 0 && std::isfinite(norm), "Zero or nonfinite embedding norm");
std::vector<float> vector(dimensions);
for (int i = 0; i < dimensions; ++i) vector[i] = float(pooled[i]/std::sqrt(norm));
mark("pool_normalize_us");
return vector;
}
json windows(const std::string & text, const std::string & title) const {
@@ -157,8 +225,42 @@ public:
}
json handle(const json & request) {
auto op = request.at("op").get<std::string>();
if(op=="image" || op=="audio") {
auto encoded=request.at("data").get<std::string>();
require(encoded.size()<=12000000 && encoded.size()%4==0,"Invalid media payload length");
static const std::string alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
std::vector<unsigned char> bytes; bytes.reserve(encoded.size()/4*3);
unsigned accumulator=0;int bits=0;bool padding=false;
for(char c:encoded){if(c=='='){padding=true;continue;} require(!padding,"Invalid base64 padding");auto n=alphabet.find(c);require(n!=std::string::npos,"Invalid base64");accumulator=(accumulator<<6)|unsigned(n);bits+=6;if(bits>=8){bits-=8;bytes.push_back(static_cast<unsigned char>(accumulator>>bits));}}
auto projector=request.at("projector").get<std::string>();
require(!projector.empty() && projector.size()<32768,"Invalid projector path");
ensure_media(projector);
mtmd_bitmap * raw=nullptr;
if(op=="image") {
int w=request.at("width"),h=request.at("height");
require(w>=48 && h>=48 && w%48==0 && h%48==0 && uint64_t(w)*h<=1120*48*48,"Invalid bounded image dimensions");
require(bytes.size()==size_t(w)*h*3,"RGB image length mismatch");
raw=mtmd_bitmap_init(w,h,bytes.data());
} else {
require(bytes.size()%4==0 && !bytes.empty() && bytes.size()/4<=30*16000,"Audio must be at most30seconds mono16kHz PCM");
std::vector<float> pcm(bytes.size()/4);
std::memcpy(pcm.data(),bytes.data(),bytes.size());
for(float f:pcm)require(std::isfinite(f) && std::abs(f)<=1,"Audio PCM outside normalized range");
raw=mtmd_bitmap_init_from_audio(pcm.size(),pcm.data());
}
std::unique_ptr<mtmd_bitmap,decltype(&mtmd_bitmap_free)> bitmap(raw,mtmd_bitmap_free);
require_inference(bitmap != nullptr,"Cannot allocate media bitmap");
return {{"vectors",json::array({embed_bitmap(bitmap.get())})}};
}
if (op == "windows") return {{"windows",windows(request.at("text").get<std::string>(),request.value("title",std::string()))}};
if (op == "query") return {{"vectors",json::array({embed("task: search result | query: "+request.at("text").get<std::string>(),512)})}};
if (op == "query") {
bool debug = request.value("debug_performance",false);
json timings;
auto vector = embed("task: search result | query: "+request.at("text").get<std::string>(),512,debug ? &timings : nullptr);
json response = {{"vectors",json::array({vector})}};
if (debug) response["performance"] = timings;
return response;
}
if (op == "documents") {
const auto & texts = request.at("texts");
require(texts.is_array() && !texts.empty() && texts.size() <= 8,"Batch size must be between 1 and 8");
@@ -187,11 +289,11 @@ int main(int argc, char ** argv) {
bool force_cpu = std::getenv("SEARCHI_FORCE_CPU") && std::string(std::getenv("SEARCHI_FORCE_CPU")) == "1";
Encoder encoder(path,threads,device == "auto" && !force_cpu);
std::cout << json({{"ready",true},{"model","google/embeddinggemma-2"},{"dimensions",dimensions},
{"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG}}).dump() << '\n' << std::flush;
{"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true}}).dump() << '\n' << std::flush;
for (;;) {
std::string line; char c;
while (std::cin.get(c) && c != '\n') {
require(line.size() < max_request_bytes,"Request exceeds 2 MiB"); line.push_back(c);
require(line.size() < max_request_bytes,"Request exceeds 16 MiB"); line.push_back(c);
}
if (!std::cin && line.empty()) break;
json response;
crates/searchi-semantic/src/gemma.rs
+390 -6
View File
@@ -1,24 +1,35 @@
use crate::TextWindow;
use anyhow::{ensure, Context, Result};
use base64::Engine;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use std::{
collections::BTreeMap,
fs::File,
io::{BufRead, BufReader, Read, Write},
path::{Path, PathBuf},
process::{Child, ChildStdin, Command, Stdio},
sync::mpsc::{self, Receiver},
thread,
time::Duration,
time::{Duration, Instant},
};
pub const GEMMA_DIMENSIONS: usize = 768;
pub const GEMMA_FINGERPRINT: &str = "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1";
pub const GEMMA_MEDIA_FINGERPRINT: &str = "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mtmd:mean:l2:rgb-catmullrom48-exif-largejpeg-scaled-v2:pcm16k30s-v1";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ImageEmbedding {
pub vector: Vec<f32>,
pub encoded_width: u32,
pub encoded_height: u32,
pub token_budget: u32,
}
const MODEL_FILE: &str = "embeddinggemma-2-Q8_0.gguf";
const MODEL_BYTES: u64 = 309855456;
const MODEL_SHA256: &str = "2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135";
const MAX_REQUEST: usize = 2 * 1024 * 1024;
const MAX_REQUEST: usize = 16 * 1024 * 1024;
const MAX_RESPONSE: usize = 4 * 1024 * 1024;
#[derive(Debug, Clone, Copy)]
@@ -35,6 +46,8 @@ pub struct WorkerInfo {
pub device: String,
pub threads: usize,
pub runtime: String,
#[serde(default)]
pub media_supported: bool,
}
/// Persistent isolated native worker. No HTTP or remote model fallback.
@@ -47,6 +60,8 @@ pub struct GemmaEmbedder {
force_cpu: bool,
model_dir: PathBuf,
worker_path: PathBuf,
projector_verified: bool,
query_profile: Option<BTreeMap<String, u64>>,
}
impl Drop for GemmaEmbedder {
@@ -161,6 +176,8 @@ impl GemmaEmbedder {
force_cpu: cpu,
model_dir,
worker_path: worker,
projector_verified: false,
query_profile: None,
})
}
@@ -169,10 +186,12 @@ impl GemmaEmbedder {
self.healthy,
"Gemma worker session failed; reopen it explicitly"
);
let encode_started = self.query_profile.as_ref().map(|_| Instant::now());
let encoded = serde_json::to_vec(&request)?;
self.record_query_duration("request_encode_us", encode_started);
ensure!(
encoded.len() < MAX_REQUEST,
"Gemma request exceeds 2 MiB protocol budget"
"Gemma request exceeds 16 MiB protocol budget"
);
let response = match self.exchange(&encoded) {
Ok(response) => response,
@@ -180,6 +199,7 @@ impl GemmaEmbedder {
// GPU runtimes can abort rather than return an encoder error.
// Restart once as CPU and retry this same bounded request.
let _ = self.child.wait();
let restart_started = self.query_profile.as_ref().map(|_| Instant::now());
let replacement = Self::load_with_device(
&self.model_dir,
&self.worker_path,
@@ -189,7 +209,13 @@ impl GemmaEmbedder {
.with_context(|| {
format!("CPU worker restart failed after GPU transport error: {error}")
})?;
let diagnostics = self.query_profile.take();
*self = replacement;
self.query_profile = diagnostics;
self.record_query_duration("cpu_restart_us", restart_started);
if let Some(profile) = &mut self.query_profile {
profile.insert("cpu_restart_count".into(), 1);
}
self.exchange(&encoded)?
}
Err(error) => return Err(error),
@@ -201,6 +227,11 @@ impl GemmaEmbedder {
);
self.info.device = device.into();
}
if response["fallback"].as_str() == Some("gpu_inference_failure") {
if let Some(profile) = &mut self.query_profile {
profile.insert("worker_cpu_retry_count".into(), 1);
}
}
if response["fatal"] == true {
self.healthy = false;
self.info.ready = false;
@@ -215,18 +246,21 @@ impl GemmaEmbedder {
}
fn exchange(&mut self, encoded: &[u8]) -> Result<Value> {
let write_started = self.query_profile.as_ref().map(|_| Instant::now());
let sent = (|| {
self.stdin.write_all(encoded)?;
self.stdin.write_all(b"\n")?;
self.stdin.flush()
})()
.context("Cannot write local Gemma worker request");
self.record_query_duration("request_write_us", write_started);
if let Err(error) = sent {
self.healthy = false;
self.info.ready = false;
let _ = self.child.kill();
return Err(error);
}
let wait_started = self.query_profile.as_ref().map(|_| Instant::now());
let response = match self.responses.recv_timeout(Duration::from_secs(180)) {
Ok(Ok(value)) => value,
other => {
@@ -236,6 +270,7 @@ impl GemmaEmbedder {
anyhow::bail!("Gemma worker request failed: {other:?}");
}
};
self.record_query_duration("worker_wait_and_response_decode_us", wait_started);
Ok(response)
}
@@ -281,15 +316,265 @@ impl GemmaEmbedder {
}
pub fn embed_query(&mut self, query: &str) -> Result<Vec<f32>> {
self.query_profile = None;
self.embed_query_inner(query, false)
}
/// Opt-in diagnostics contain only durations and counts, never query text.
pub fn embed_query_debug(&mut self, query: &str) -> Result<Vec<f32>> {
self.query_profile = Some(BTreeMap::new());
self.embed_query_inner(query, true)
}
pub fn query_diagnostics(&self) -> BTreeMap<String, u64> {
self.query_profile.clone().unwrap_or_default()
}
fn record_query_duration(&mut self, name: &str, started: Option<Instant>) {
if let (Some(profile), Some(started)) = (&mut self.query_profile, started) {
let micros = started.elapsed().as_micros().min(u64::MAX as u128) as u64;
*profile.entry(name.into()).or_default() += micros;
}
}
fn embed_query_inner(&mut self, query: &str, debug: bool) -> Result<Vec<f32>> {
ensure!(
!query.trim().is_empty() && query.len() <= 4096,
"Query must be nonempty and at most 4096 UTF-8 bytes"
);
let response = self.request(json!({"op":"query","text":query}))?;
let response =
self.request(json!({"op":"query","text":query,"debug_performance":debug}))?;
if debug {
if let Some(profile) = &mut self.query_profile {
for name in [
"tokenization_us",
"batch_prepare_us",
"inference_us",
"pool_normalize_us",
"token_count",
] {
if let Some(value) = response["performance"][name].as_u64() {
profile.insert(format!("worker_{name}"), value);
}
}
}
}
let validation_started = self.query_profile.as_ref().map(|_| Instant::now());
let result = extract_vectors(response, 1)?.remove(0);
self.record_query_duration("response_validate_us", validation_started);
Ok(result)
}
fn projector(&mut self) -> Result<PathBuf> {
ensure!(
self.info.media_supported,
"Installed embedding worker does not support media; repair/rebuild the native runtime"
);
let path = self.model_dir.join("mmproj-embeddinggemma-2-Q8_0.gguf");
if !self.projector_verified {
verify_artifact(
&path,
554821024,
"c4a8a52691ecef40618438928bdf9e68379b854e24166f292592353db0aab64f",
)?;
self.projector_verified = true;
}
Ok(path)
}
/// Genuine Google pixel embeddings, serial bounded image operations.
pub fn embed_images(&mut self, paths: &[&Path]) -> Result<Vec<Vec<f32>>> {
self.embed_images_with_budget(paths, 280)
}
/// 70 tokens for video frames, 140 for photos, 280 or 1120 for screenshots.
pub fn embed_images_with_budget(
&mut self,
paths: &[&Path],
budget: u32,
) -> Result<Vec<Vec<f32>>> {
Ok(self
.embed_images_with_metadata(paths, budget)?
.into_iter()
.map(|image| image.vector)
.collect())
}
pub fn embed_images_with_metadata(
&mut self,
paths: &[&Path],
budget: u32,
) -> Result<Vec<ImageEmbedding>> {
ensure!(
!paths.is_empty() && paths.len() <= 8,
"Image batch must contain 1–8 paths"
);
let projector = self.projector()?;
let mut vectors = Vec::with_capacity(paths.len());
for path in paths {
let pixels = decode_image(path, budget)?;
let response=self.request(json!({"op":"image","projector":projector,"width":pixels.width(),"height":pixels.height(),"data":base64::engine::general_purpose::STANDARD.encode(pixels.as_raw())}))?;
vectors.push(ImageEmbedding {
vector: extract_vectors(response, 1)?.remove(0),
encoded_width: pixels.width(),
encoded_height: pixels.height(),
token_budget: budget,
});
}
Ok(vectors)
}
/// One mono 16 kHz normalized PCM window, at most 30 seconds. Callers
/// segment entire recordings and preserve real timestamps; no truncation.
pub fn embed_audio_pcm(&mut self, samples: &[f32]) -> Result<Vec<f32>> {
ensure!(
!samples.is_empty() && samples.len() <= 480000,
"Audio window must contain 1–480000 mono 16 kHz samples"
);
ensure!(
samples.iter().all(|v| v.is_finite() && v.abs() <= 1.0),
"Invalid normalized audio PCM"
);
ensure!(
cfg!(target_endian = "little"),
"Native PCM worker requires little-endian host"
);
let projector = self.projector()?;
let bytes: Vec<u8> = samples.iter().flat_map(|v| v.to_le_bytes()).collect();
let response=self.request(json!({"op":"audio","projector":projector,"data":base64::engine::general_purpose::STANDARD.encode(bytes)}))?;
Ok(extract_vectors(response, 1)?.remove(0))
}
}
fn decode_image(path: &Path, budget: u32) -> Result<image::RgbImage> {
ensure!(
(70..=1120).contains(&budget),
"Image token budget must be 70–1120"
);
ensure!(
std::fs::metadata(path)?.len() <= 64 * 1024 * 1024,
"Image file exceeds 64 MiB"
);
let mut reader = image::ImageReader::open(path)?.with_guessed_format()?;
let is_jpeg = reader.format() == Some(image::ImageFormat::Jpeg);
let mut limits = image::Limits::default();
limits.max_alloc = Some(256 * 1024 * 1024);
limits.max_image_width = Some(16384);
limits.max_image_height = Some(16384);
reader.limits(limits);
use image::ImageDecoder;
let mut decoder = reader
.into_decoder()
.context("Cannot initialize bounded image decoder")?;
let (source_width, source_height) = decoder.dimensions();
ensure!(
u64::from(source_width) * u64::from(source_height) <= 64 * 1024 * 1024
&& decoder.total_bytes() <= 256 * 1024 * 1024,
"Decoded image exceeds memory/pixel budget"
);
let orientation = decoder
.orientation()
.context("Cannot read image EXIF orientation")?;
if is_jpeg && u64::from(source_width) * u64::from(source_height) > 16 * 1024 * 1024 {
drop(decoder);
return decode_large_jpeg(path, budget, source_width, source_height, orientation);
}
ensure!(u64::from(source_width)*u64::from(source_height)<=16*1024*1024,"Non-JPEG image exceeds16megapixels; bounded scaled decoding is currently available for JPEG");
let mut decoded = image::DynamicImage::from_decoder(decoder)
.context("Cannot decode supported image pixels")?;
decoded.apply_orientation(orientation);
ensure!(
u64::from(decoded.width()) * u64::from(decoded.height()) <= 16 * 1024 * 1024,
"Image exceeds 16 megapixels"
);
let (w, h) = image_target(decoded.width(), decoded.height(), budget)?;
Ok(image::imageops::resize(
&decoded.to_rgb8(),
w,
h,
image::imageops::FilterType::CatmullRom,
))
}
fn decode_large_jpeg(
path: &Path,
budget: u32,
width: u32,
height: u32,
orientation: image::metadata::Orientation,
) -> Result<image::RgbImage> {
let (target_width, target_height) = image_target(width, height, budget)?;
let mut decoder = jpeg_decoder::Decoder::new(BufReader::new(File::open(path)?));
decoder.set_max_decoding_buffer_size(256 * 1024 * 1024);
decoder
.read_info()
.context("Cannot read scaled JPEG header")?;
let source = decoder.info().context("Missing scaled JPEG header")?;
ensure!(
u32::from(source.width) == width && u32::from(source.height) == height,
"JPEG changed during decoding"
);
ensure!(
matches!(
source.pixel_format,
jpeg_decoder::PixelFormat::RGB24 | jpeg_decoder::PixelFormat::L8
),
"Large JPEG requires supported RGB24 or grayscale scaled decoding"
);
let (w, h) = decoder.scale(u16::try_from(target_width)?, u16::try_from(target_height)?)?;
ensure!(
u64::from(w) * u64::from(h) * 4 <= 256 * 1024 * 1024,
"Scaled JPEG output exceeds memory budget"
);
let pixels = decoder
.decode()
.context("Cannot decode large JPEG with scaled IDCT")?;
let mut decoded = match source.pixel_format {
jpeg_decoder::PixelFormat::RGB24 => image::DynamicImage::ImageRgb8(
image::RgbImage::from_raw(u32::from(w), u32::from(h), pixels)
.context("Invalid scaled JPEG RGB length")?,
),
jpeg_decoder::PixelFormat::L8 => image::DynamicImage::ImageLuma8(
image::GrayImage::from_raw(u32::from(w), u32::from(h), pixels)
.context("Invalid scaled JPEG grayscale length")?,
),
other => anyhow::bail!(
"Large JPEG pixel format {other:?} is not supported by bounded scaled decoder"
),
};
decoded.apply_orientation(orientation);
let (w, h) = image_target(decoded.width(), decoded.height(), budget)?;
Ok(image::imageops::resize(
&decoded.into_rgb8(),
w,
h,
image::imageops::FilterType::CatmullRom,
))
}
fn image_target(width: u32, height: u32, budget: u32) -> Result<(u32, u32)> {
ensure!(
width > 0 && height > 0 && (70..=1120).contains(&budget),
"Invalid image dimensions/budget"
);
let scale = (f64::from(budget) * 2304.0 / (f64::from(width) * f64::from(height))).sqrt();
let mut w = ((f64::from(width) * scale / 48.0).floor() as u32) * 48;
let mut h = ((f64::from(height) * scale / 48.0).floor() as u32) * 48;
if w == 0 {
w = 48;
h = (height / width * 48).min(budget * 48);
}
if h == 0 {
h = 48;
w = (width / height * 48).min(budget * 48);
}
ensure!(
w >= 48 && h >= 48 && u64::from(w) * u64::from(h) <= u64::from(budget) * 2304,
"Image target outside token budget"
);
Ok((w, h))
}
fn can_retry_cpu(actual_device: &str, force_cpu: bool) -> bool {
actual_device == "gpu" && !force_cpu
}
@@ -331,6 +616,10 @@ fn extract_vectors(response: Value, count: usize) -> Result<Vec<Vec<f32>>> {
}
fn verify_model(path: &Path) -> Result<()> {
verify_artifact(path, MODEL_BYTES, MODEL_SHA256)
}
fn verify_artifact(path: &Path, bytes: u64, sha: &str) -> Result<()> {
let file = File::open(path).with_context(|| {
format!(
"Missing Google EmbeddingGemma 2 model {}; run explicit model setup first",
@@ -338,7 +627,7 @@ fn verify_model(path: &Path) -> Result<()> {
)
})?;
ensure!(
file.metadata()?.len() == MODEL_BYTES,
file.metadata()?.len() == bytes,
"EmbeddingGemma 2 model size mismatch"
);
let mut reader = BufReader::new(file);
@@ -352,7 +641,7 @@ fn verify_model(path: &Path) -> Result<()> {
hash.update(&buffer[..count]);
}
ensure!(
format!("{:x}", hash.finalize()) == MODEL_SHA256,
format!("{:x}", hash.finalize()) == sha,
"EmbeddingGemma 2 model SHA256 mismatch"
);
Ok(())
@@ -362,6 +651,101 @@ fn verify_model(path: &Path) -> Result<()> {
mod tests {
use super::*;
#[test]
fn old_text_worker_handshake_does_not_claim_media_support() {
let mut handshake = json!({"ready":true,"model":"google/embeddinggemma-2","dimensions":768,"device":"cpu","threads":1,"runtime":"b11461-no-logits-v1"});
let old: WorkerInfo = serde_json::from_value(handshake.clone()).unwrap();
assert!(!old.media_supported);
handshake["media_supported"] = json!(true);
assert!(
serde_json::from_value::<WorkerInfo>(handshake)
.unwrap()
.media_supported
);
}
#[test]
fn media_image_grid_is_bounded_for_extreme_aspects_and_full_budget() {
for budget in [70, 140, 280, 1120] {
for (width, height) in [(1, 16384), (16384, 1), (1920, 1080), (500, 500)] {
let (w, h) = image_target(width, height, budget).unwrap();
assert_eq!(w % 48, 0);
assert_eq!(h % 48, 0);
assert!(u64::from(w) * u64::from(h) <= u64::from(budget) * 2304);
}
}
assert!(image_target(0, 100, 280).is_err());
assert!(image_target(100, 100, 1121).is_err());
}
#[test]
fn artifact_verification_rejects_projector_size_and_checksum_mismatch() {
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/LICENSE-DigUp.txt");
let bytes = std::fs::read(&path).unwrap();
let sha = format!("{:x}", Sha256::digest(&bytes));
assert!(verify_artifact(&path, bytes.len() as u64, &sha).is_ok());
assert!(verify_artifact(&path, bytes.len() as u64 + 1, &sha).is_err());
assert!(verify_artifact(&path, bytes.len() as u64, &"0".repeat(64)).is_err());
}
#[test]
#[ignore = "Explicit48MP JPEG resource test; no model needed, run release CPU"]
fn large_phone_jpeg_scaled_decode_preserves_exif_and_bounds() {
use image::ImageEncoder;
let image = image::RgbImage::from_fn(8000, 6000, |x, _| {
if x < 4000 {
image::Rgb([240, 10, 10])
} else {
image::Rgb([10, 10, 240])
}
});
let mut encoded = Vec::new();
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut encoded, 80);
// Little-endian TIFF orientation tag6: rotate90clockwise.
encoder
.set_exif_metadata(vec![
b'I', b'I', 42, 0, 8, 0, 0, 0, 1, 0, 0x12, 1, 3, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0,
0, 0,
])
.unwrap();
encoder.encode_image(&image).unwrap();
drop(image);
let path =
std::env::temp_dir().join(format!("searchi-phone48mp-{}.jpg", std::process::id()));
std::fs::write(&path, &encoded).unwrap();
drop(encoded);
let started = std::time::Instant::now();
let result = decode_image(&path, 280);
let _ = std::fs::remove_file(&path);
let pixels = result.unwrap();
println!(
"48MPactualJPEG scaleddecode+EXIF+resize {:?}; output{}x{} RGB{}bytes",
started.elapsed(),
pixels.width(),
pixels.height(),
pixels.as_raw().len()
);
assert!(pixels.height() > pixels.width());
assert!(pixels.get_pixel(pixels.width() / 2, pixels.height() / 4)[0] > 200);
assert!(pixels.get_pixel(pixels.width() / 2, pixels.height() * 3 / 4)[2] > 200);
assert!(u64::from(pixels.width()) * u64::from(pixels.height()) <= 280 * 2304);
}
#[test]
fn oversized_jpeg_header_rejected_before_pixel_decode() {
let mut encoded = Vec::new();
image::codecs::jpeg::JpegEncoder::new(&mut encoded)
.encode_image(&image::RgbImage::new(16, 16))
.unwrap();
let sof = encoded.windows(2).position(|p| p == [0xff, 0xc0]).unwrap();
encoded[sof + 5..sof + 7].copy_from_slice(&9000u16.to_be_bytes());
encoded[sof + 7..sof + 9].copy_from_slice(&9000u16.to_be_bytes());
let path =
std::env::temp_dir().join(format!("searchi-too-large-{}.jpg", std::process::id()));
std::fs::write(&path, encoded).unwrap();
let result = decode_image(&path, 280);
let _ = std::fs::remove_file(&path);
assert!(result
.unwrap_err()
.to_string()
.contains("memory/pixel budget"));
}
#[test]
fn missing_google_model_fails_without_starting_worker() {
assert!(GemmaEmbedder::load("missing-google-model", "missing-worker", 1).is_err());
}
crates/searchi-semantic/src/lib.rs
+4 -1
View File
@@ -1,7 +1,10 @@
//! Local Google EmbeddingGemma 2 inference through an isolated native worker.
//! The rejected MiniLM experiment is available only under `minilm-proof`.
mod gemma;
pub use gemma::{Device, GemmaEmbedder, WorkerInfo, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT};
pub use gemma::{
Device, GemmaEmbedder, ImageEmbedding, WorkerInfo, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT,
GEMMA_MEDIA_FINGERPRINT,
};
/// A verified source byte range covered by a model token window.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
crates/searchi-semantic/tests/fixtures/README.md
Vendored
+14
View File
@@ -0,0 +1,14 @@
# Pinned Google media reference fixtures
The `digup-photo.jpg`, `digup-screen.png`, `digup-speech.wav`, and
`digup-media-reference.json` files are copied unchanged from
`ARahim3/DigUp`, commit `6faa29e4c315a3a591de1b41bd78751bdf39588d`,
`Tests/LlamaRuntimeTests/Reference/`. They and the existing text reference
fixture are retained under the adjacent `LICENSE-DigUp.txt` MIT notice.
The source test uses image budgets 280 (screen), 140 (photo), 70 (frame),
and a mono 16 kHz speech recording. The native CPU test compares actual
768-dimensional Google vectors with the pinned llama.cpp b11461 reference
at cosine >=0.999. This is runtime parity evidence, not a production media
relevance corpus. Independent images, audio, and video require separate
quality evaluation.
crates/searchi-semantic/tests/fixtures/digup-media-reference.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"made_with":"llama.cpp b11461","vector_version":1,"vectors":{"document":[0.0023304052,-0.021797102,-0.025986522,-0.025894934,0.0046086097,-0.029744277,-0.04186634,0.03234369,0.015701862,-0.13444579,-0.049679857,-0.067124754,-0.008888296,0.049783457,0.04075608,-0.0027419934,0.0494676,0.0092225745,-0.003042357,0.021539804,-0.066572726,-0.04184304,-0.008146685,-0.015800655,0.0028940293,0.026850378,0.023585942,0.012522315,-0.018817414,0.19809185,-0.0541878,-0.033899978,0.06363426,-0.012184333,0.026218394,-0.030783277,-0.016514625,0.0010143976,0.016249185,0.00673781,0.07569905,-0.032087695,0.01618515,0.008316216,0.009405525,0.035788074,0.011760172,-0.0018986556,0.03811712,0.04686964,-0.0019338331,-0.06858261,-0.020950682,0.0061196685,-0.012488069,0.055386145,0.00828158,-0.028973866,0.029866058,0.019298207,0.0064102146,0.03210796,0.031232368,-0.024290783,-0.009193791,-0.00788917,0.07007842,-0.036232397,0.048318822,0.03329114,-0.021271994,-0.01810213,0.056723867,0.026635557,-0.0066668643,-0.03339305,-0.09275154,-0.005457068,-0.037602138,0.06852603,-0.086277,0.0108406255,-0.022301262,0.054799795,0.048695795,-0.0094250385,0.04326083,-0.05584021,0.009427591,0.018164966,0.0077456376,0.012890579,0.011140378,0.009403773,-0.013322473,0.014014791,0.0045265807,0.0026267597,-0.02777846,0.03337617,-0.014311906,-0.0027436074,0.0016559366,0.04267254,-0.0074306116,0.010555651,0.00801582,0.019190526,0.026427642,-0.02305468,0.017346293,-0.254231,-0.037654083,0.004254816,-0.029559055,0.30139065,-0.0017701226,0.012945749,0.045082387,-0.005745192,0.023981802,0.11611545,0.02849787,0.0050043385,0.019032571,0.01543062,0.013486391,-0.0073420904,-0.0120351305,0.027507395,0.017718071,0.0048503615,0.0070377868,-0.00946547,0.015142535,0.013139711,0.019906517,-0.038481727,0.060138095,0.0027306078,-0.010124241,0.020947324,-0.005082965,0.009129657,-0.06960446,0.04814667,0.012469531,-0.017168682,-0.007835558,-0.018339474,0.04499317,-0.048769984,-0.021295255,0.0008460758,0.042394932,0.02629723,0.015862137,-0.027827857,0.02877754,0.044078708,0.02864664,0.015502866,-0.011931367,0.006051371,-0.0676744,-0.032051444,-0.0051915697,0.022663914,0.010766299,-0.045957707,-0.071434505,-0.0071970616,-0.00080499344,-0.019176258,0.039329063,0.07615527,-0.02934876,0.03178097,0.030727895,-0.0014563488,0.033275653,-0.016663045,0.016650142,0.03149862,-0.042355653,-0.01447139,-0.031222051,-0.0371221,-0.071084455,0.02895739,0.012906462,-0.034588896,0.005292062,0.04567936,0.057948757,0.016161263,0.026154758,0.037441123,0.020264363,0.0021852585,0.010503882,0.02564951,-0.0034963347,-0.002975977,-0.05896342,0.090469755,-0.02141068,-0.018266847,0.03979448,0.006404804,0.048994806,-0.0013431136,0.003767527,0.009192836,0.03172842,0.0018301627,-0.032883424,0.070755646,-0.008929331,0.007731602,0.039488316,-0.0026821592,0.047416102,0.017939921,-0.046493225,-0.003571527,0.017453436,0.027679116,-0.045083456,-0.0018627502,-0.0058596916,-0.016741525,-0.041559335,-0.07398639,0.035852198,-0.01250213,-0.027620463,-0.022970477,-0.007722595,0.006369175,-0.027909646,0.05087572,0.04319605,0.0249135,-0.045503743,-0.050388955,-0.07399011,-0.031614304,-0.009977491,-0.011919013,0.017587284,0.046735547,0.051120915,-0.01913088,-0.0040965537,-0.030126736,0.022246089,-0.024694057,-0.0011967963,-0.0076620826,-0.0001045683,0.025630128,0.0023922755,-0.016251149,0.029261485,0.026775412,-0.0011348778,-0.023079926,0.0046075485,0.013615946,-0.022271913,0.0063756416,-0.011097959,-0.057059642,0.017879428,0.039326653,-0.010385263,-0.024296982,-0.0056849266,-0.021441974,0.023893535,-0.004312103,-0.0029023795,0.011346968,-0.05022057,-0.00962676,-0.030292084,0.0022458949,-0.0012382208,0.014016518,0.0130033605,-0.0075721424,0.022480633,0.0378293,-0.0060748337,0.020296352,-0.038013346,0.02737668,0.051704004,0.029559156,-0.008555367,-0.040540576,0.0028047708,-0.01888827,-0.017475538,0.022049233,-0.018655647,0.00035712068,0.0007242559,-0.031585597,0.002809768,-0.010847336,0.019464653,-0.038227003,0.004094894,-0.0045381063,-0.032818098,-0.025449943,-0.0048597767,-0.052715924,0.024247607,-0.025617195,-0.028776655,-0.01491966,0.027141426,0.022653097,0.01347185,0.0010195217,0.0077305734,0.006165836,-0.009121481,-0.011312435,0.0010023145,0.033321623,0.0020216978,-0.04601626,-0.020557044,-0.023531808,0.013099387,0.049464565,-0.0083905,-0.12218178,-0.0090951715,-0.037435923,-0.043180246,0.008593723,0.003133877,0.03239143,-0.02740052,0.028175933,0.00713765,0.010822759,-0.011402951,-0.042760924,-0.03611006,0.011457286,0.060568936,0.0027335465,-0.024305783,-0.004420147,0.0068441983,0.002837347,-0.02218109,-0.022486687,-0.01941801,0.01667096,-0.03958557,-0.0077348636,-0.0012779543,-0.009598079,-0.008962751,0.033793483,0.018625712,-0.006633453,0.026242023,0.021685809,0.013799533,-0.0060389466,0.0020937775,0.02297482,-0.02592566,0.0011351658,0.00793861,-0.01809629,0.050504446,-0.026299631,-0.028017156,0.006029957,-0.02648004,0.03030571,0.016809817,0.29351303,-0.028621448,0.01694766,0.021401705,0.012320341,0.0015229117,-0.Line truncated
crates/searchi-semantic/tests/fixtures/digup-photo.jpg
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 165 KiB

crates/searchi-semantic/tests/fixtures/digup-screen.png
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.3 KiB

crates/searchi-semantic/tests/fixtures/digup-speech.wav
Vendored
BIN
View File
Binary file not shown.
crates/searchi-semantic/tests/gemma_cpu.rs
+43
View File
@@ -1,6 +1,49 @@
use anyhow::Result;
use searchi_semantic::{Device, GemmaEmbedder, GEMMA_DIMENSIONS};
#[test]
#[ignore = "Requires explicitly provisioned Google model and CPU worker"]
fn google_cpu_debug_query_reports_actual_worker_stages_without_changing_vectors() -> Result<()> {
let model = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set SEARCHI_GEMMA_MODEL_DIR");
let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set SEARCHI_GEMMA_WORKER");
let mut embedder = GemmaEmbedder::load_with_device(model, worker, 1, Device::Cpu)?;
let query = "supplier contract cancellation policy";
let debug = embedder.embed_query_debug(query)?;
let diagnostics = embedder.query_diagnostics();
for name in [
"request_encode_us",
"request_write_us",
"worker_wait_and_response_decode_us",
"response_validate_us",
"worker_tokenization_us",
"worker_batch_prepare_us",
"worker_inference_us",
"worker_pool_normalize_us",
"worker_token_count",
] {
assert!(
diagnostics.contains_key(name),
"Missing actual stage {name}: {diagnostics:?}"
);
}
assert!(diagnostics["worker_inference_us"] > 0);
assert!(diagnostics["worker_token_count"] > 0);
assert!(!serde_json::to_string(&diagnostics)?.contains(query));
let normal = embedder.embed_query(query)?;
assert!(embedder.query_diagnostics().is_empty());
let cosine: f64 = debug
.iter()
.zip(&normal)
.map(|(a, b)| f64::from(*a) * f64::from(*b))
.sum();
assert!(
cosine >= 0.99999,
"Debug tracing changed embeddings: {cosine}"
);
println!("CPU query diagnostics: {diagnostics:?}; debug/normal cosine {cosine:.8}");
Ok(())
}
#[test]
#[ignore = "Requires explicitly provisioned Google model and CPU worker"]
fn google_cpu_real_embeddings_and_verified_tail_windows() -> Result<()> {
crates/searchi-semantic/tests/gemma_media_cpu.rs
+57
View File
@@ -0,0 +1,57 @@
use anyhow::Result;
use searchi_semantic::{Device, GemmaEmbedder};
use std::path::Path;
#[test]
#[ignore = "Requires explicitly provisioned Google media model and CPU worker"]
fn google_cpu_pixels_match_digup_reference() -> Result<()> {
let model = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set model path");
let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set worker path");
let mut embedder = GemmaEmbedder::load_with_device(model, worker, 1, Device::Cpu)?;
let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
let reference: serde_json::Value =
serde_json::from_str(include_str!("fixtures/digup-media-reference.json"))?;
for (name, file, budget) in [
("screen-280", "digup-screen.png", 280),
("photo-140", "digup-photo.jpg", 140),
("frame-70", "digup-photo.jpg", 70),
] {
let path = fixtures.join(file);
let vector = embedder
.embed_images_with_budget(&[path.as_path()], budget)?
.remove(0);
let cosine: f64 = vector
.iter()
.zip(reference["vectors"][name].as_array().unwrap())
.map(|(a, b)| f64::from(*a) * b.as_f64().unwrap())
.sum();
println!("Google image {name} reference cosine={cosine:.8}");
assert!(cosine >= 0.999, "Image parity below0.999: {name} {cosine}");
}
assert_eq!(embedder.info.device, "cpu");
let wav = include_bytes!("fixtures/digup-speech.wav");
let mut offset = 12;
let mut samples = Vec::new();
while offset + 8 <= wav.len() {
let length = u32::from_le_bytes(wav[offset + 4..offset + 8].try_into()?) as usize;
if &wav[offset..offset + 4] == b"data" {
for pair in wav[offset + 8..offset + 8 + length].as_chunks::<2>().0 {
samples.push(f32::from(i16::from_le_bytes(*pair)) / 32768.0);
}
break;
}
offset += 8 + length + (length % 2);
}
let vector = embedder.embed_audio_pcm(&samples)?;
let cosine: f64 = vector
.iter()
.zip(reference["vectors"]["speech"].as_array().unwrap())
.map(|(a, b)| f64::from(*a) * b.as_f64().unwrap())
.sum();
println!("Google audio reference cosine={cosine:.8}");
assert!(
cosine >= 0.999,
"Audio reference parity below0.999: {cosine}"
);
Ok(())
}
crates/searchi-service/Cargo.toml
+1
View File
@@ -10,6 +10,7 @@ anyhow.workspace = true
serde.workspace = true
searchi-core = { path = "../searchi-core" }
searchi-semantic = { path = "../searchi-semantic" }
searchi-media = { path = "../searchi-media" }
rusqlite = { version = "0.32", features = ["bundled"] }
sha2 = "0.10"
fs2 = "0.4"
crates/searchi-service/README.md
+43
View File
@@ -0,0 +1,43 @@
# Shared hybrid retrieval
`HybridSearch` combines genuine lexical and Google multimodal retrieval. The
CLI and desktop use the same service; it has no desktop-shell dependency.
`search`, `search_semantic` and `search_semantic_with_origin` preserve grouped
parent files. ANN candidates expand until enough unique eligible parents exist
or the complete bounded eligible chunk set is requested. Filename/type/folder,
size/date/duration and explicit source filters apply before ANN top-k. Media
candidate filesystem metadata is checked once per parent per query. Ninety-six
near-identical video vectors cannot hide a second eligible parent (regression
tested). HNSW remains approximate retrieval, not an exact recall guarantee.
Explicit spoken/audio wording prefers actual audio-source vectors, preserving
exact lexical filename priority. Conservative English/German token cues detect
requests such as “someone mentions”, “spoken”, “audio”, “jemand erwähnt” and
“Gespräch”. Visual/document cues veto automatic routing. This finite parser is
not language understanding; paraphrases, negation and mixed intents can remain
ambiguous. If the preferred lane has no eligible results, ordinary retrieval
is used. `SemanticOrigin::All` bypasses the hint; `Audio`, `Image`, `Video` and
`Text` select an explicit channel without fallback or reembedding.
Persisted custom sampling limits can be reopened without generating embeddings
again only when the complete model and preprocessing fingerprint matches.
The custom suffix must contain the eight valid numeric sampling limits. Model
or preprocessing changes never silently reuse incompatible vectors.
`search_measured(query,limit,options,debug)` returns `SearchResponse` with results
and monotonic total milliseconds/result count. Basic mode stores no trace.
`search_traced(...,&mut TraceCollector,parent)` shares the native request clock
so queue/lock time and actual engine stages remain comparable. Debug mode adds
bounded nested spans for lexical SQL, query embedding, worker diagnostics,
filtered eligibility and its cache, ANN read/checksum/deserialization/search
passes, parent materialization and validation, and rank fusion. Static stage
names and numerical counters exclude query/source text and paths. Worker-local
microsecond counters have a separate clock and are not aligned request spans.
Measured/source APIs retain existing result behavior except the explicit
spoken/audio preference. Debug instrumentation does not change vector policy.
Audio timestamps represent segment starts, not exact word locations. Smaller
windows alone have not demonstrated accurate spoken navigation. No-answer
calibration, music discrimination, exact spoken positions, process isolation,
unchanged-vector reuse and production platform checks remain release gates.
crates/searchi-service/src/lib.rs
+207 -18
View File
@@ -1,10 +1,15 @@
//! Shared offline hybrid search using real Google embeddings and persisted,
//! filtered HNSW retrieval. Production-scale validation remains a release gate.
mod media;
mod routing;
pub use routing::SemanticOrigin;
mod store;
pub use searchi_media::{FfmpegTools, MediaLimits};
use anyhow::{ensure, Result};
use searchi_core::metrics::{MetricsMode, SpanToken, TraceCollector};
use searchi_core::SearchIndex;
pub use searchi_core::{SearchOptions, SearchResult};
pub use searchi_core::{SearchMetrics, SearchOptions, SearchResponse, SearchResult};
pub use searchi_semantic::WorkerInfo;
use searchi_semantic::{GemmaEmbedder, TextWindow, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT};
use serde::Serialize;
@@ -17,7 +22,24 @@ const BYTE_OVERLAP: usize = 512;
const MAX_PASSAGE_WINDOWS: usize = 2048;
const CHUNK_POLICY: &str = "byte64k-overlap512-title512-v1";
fn generation_fingerprint(model: &str) -> String {
format!("{model}:{CHUNK_POLICY}")
format!(
"{model}:{CHUNK_POLICY}:{}:image280-frame70-audio30s-overlap2-distributed240-scene120-v1",
searchi_semantic::GEMMA_MEDIA_FINGERPRINT
)
}
fn reusable_policy(expected: &str, active: &str) -> bool {
if active == expected || active == format!("{expected}:text-only") {
return true;
}
let Some(policy) = active.strip_prefix(&format!("{expected}:custom-")) else {
return false;
};
let values = policy
.split('-')
.map(str::parse::<u64>)
.collect::<std::result::Result<Vec<_>, _>>();
matches!(values,Ok(v) if v.len()==8 && v[0]>=2 && (5000..=30000).contains(&v[3]) && v[4]<v[3] && v[5]>0 && v[6]<=u32::MAX as u64)
}
/// Model-specific query/document prompts and actual tokenizer windows belong
@@ -28,6 +50,18 @@ pub trait EmbeddingProvider: Send {
fn split_windows(&mut self, text: &str, title: &str) -> Result<Vec<TextWindow>>;
fn embed_documents(&mut self, texts: &[&str], title: &str) -> Result<Vec<Vec<f32>>>;
fn embed_query(&mut self, query: &str) -> Result<Vec<f32>>;
fn embed_query_debug(&mut self, query: &str) -> Result<Vec<f32>> {
self.embed_query(query)
}
fn query_diagnostics(&self) -> std::collections::BTreeMap<String, u64> {
Default::default()
}
fn embed_images(&mut self, _paths: &[&Path], _budget: u32) -> Result<Vec<Vec<f32>>> {
anyhow::bail!("Provider does not support real images")
}
fn embed_audio(&mut self, _pcm: &[f32]) -> Result<Vec<f32>> {
anyhow::bail!("Provider does not support real audio")
}
fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
None
}
@@ -49,13 +83,34 @@ impl EmbeddingProvider for GemmaEmbedder {
fn embed_query(&mut self, query: &str) -> Result<Vec<f32>> {
self.embed_query(query)
}
fn embed_query_debug(&mut self, query: &str) -> Result<Vec<f32>> {
GemmaEmbedder::embed_query_debug(self, query)
}
fn query_diagnostics(&self) -> std::collections::BTreeMap<String, u64> {
GemmaEmbedder::query_diagnostics(self)
}
fn embed_images(&mut self, paths: &[&Path], budget: u32) -> Result<Vec<Vec<f32>>> {
self.embed_images_with_budget(paths, budget)
}
fn embed_audio(&mut self, pcm: &[f32]) -> Result<Vec<f32>> {
self.embed_audio_pcm(pcm)
}
fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
Some(self.info.clone())
}
}
#[derive(Debug, Serialize)]
pub struct MediaFailure {
pub path: String,
pub message: String,
}
#[derive(Debug, Serialize)]
pub struct EmbeddingReport {
pub media_files: usize,
pub media_failures: usize,
pub failures: Vec<MediaFailure>,
pub passages: usize,
pub chunks: usize,
pub generation: i64,
@@ -64,6 +119,7 @@ pub struct EmbeddingReport {
#[derive(Debug, Serialize)]
pub struct EmbeddingStatus {
pub failures: Vec<MediaFailure>,
pub generation: Option<i64>,
pub chunks: usize,
pub model_fingerprint: Option<String>,
@@ -98,9 +154,10 @@ impl HybridSearch {
|row| row.get(0),
)?;
let pending: usize =
connection.query_row("SELECT count(*) FROM passages", [], |row| row.get(0))?;
connection.query_row("SELECT (SELECT count(*) FROM passages)+(SELECT count(*) FROM files WHERE extension IN ('png','jpg','jpeg','webp','bmp','mp4','mkv','mov','avi','webm','m4v','mpeg','mpg','mp3','flac','wav','m4a','ogg','opus','aac','aiff','aif'))", [], |row| row.get(0))?;
if !exists {
return Ok(EmbeddingStatus {
failures: Vec::new(),
generation: None,
chunks: 0,
model_fingerprint: None,
@@ -113,8 +170,9 @@ impl HybridSearch {
[],
|row| row.get(0),
)?;
if version < 5 {
if version < 6 {
return Ok(EmbeddingStatus {
failures: Vec::new(),
generation: None,
chunks: 0,
model_fingerprint: None,
@@ -123,7 +181,7 @@ impl HybridSearch {
});
}
ensure!(
version == 5,
version == 6,
"Unsupported semantic schema version {version}"
);
let (generation,fingerprint,revision,current):(Option<i64>,Option<String>,Option<i64>,i64)=connection.query_row(
@@ -134,9 +192,19 @@ impl HybridSearch {
[generation],
|row| row.get(0),
)?;
let mut failures_query=connection.prepare("SELECT path,message FROM semantic_media_errors WHERE generation=?1 ORDER BY path LIMIT 1000")?;
let failures = failures_query
.query_map([generation], |r| {
Ok(MediaFailure {
path: r.get(0)?,
message: r.get(1)?,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
let stale = revision != Some(current)
|| fingerprint.as_deref() != Some(generation_fingerprint(GEMMA_FINGERPRINT).as_str());
Ok(EmbeddingStatus {
failures,
generation,
chunks,
model_fingerprint: fingerprint,
@@ -162,7 +230,12 @@ impl HybridSearch {
) -> Result<Self> {
let lexical = SearchIndex::open(&db)?;
let store = Store::open(db.as_ref(), embedder.dimensions())?;
let generation_fingerprint = generation_fingerprint(embedder.fingerprint());
let expected = generation_fingerprint(embedder.fingerprint());
let active = store.active_fingerprint()?;
let generation_fingerprint = match active {
Some(active) if reusable_policy(&expected, &active) => active,
_ => expected,
};
Ok(Self {
lexical,
embedder,
@@ -189,6 +262,11 @@ impl HybridSearch {
/// storage errors leave the previous generation active. A crashed build is
/// reclaimed on the next explicit rebuild under an OS advisory lock.
pub fn index_embeddings(&mut self) -> Result<EmbeddingReport> {
let previous_fingerprint = self.generation_fingerprint.clone();
self.generation_fingerprint = format!(
"{}:text-only",
generation_fingerprint(self.embedder.fingerprint())
);
let _lock = self.store.build_lock()?;
let generation = self.store.begin(&self.generation_fingerprint)?;
let built = self.build(generation);
@@ -199,6 +277,7 @@ impl HybridSearch {
}
Err(error) => {
self.store.abandon(generation)?;
self.generation_fingerprint = previous_fingerprint;
Err(error)
}
}
@@ -206,6 +285,9 @@ impl HybridSearch {
fn build(&mut self, generation: i64) -> Result<EmbeddingReport> {
let mut report = EmbeddingReport {
media_files: 0,
media_failures: 0,
failures: Vec::new(),
passages: 0,
chunks: 0,
generation,
@@ -253,21 +335,81 @@ impl HybridSearch {
query: &str,
limit: usize,
options: &SearchOptions,
) -> Result<Vec<SearchResult>> {
let mut trace = TraceCollector::new(MetricsMode::Basic);
self.search_traced(query, limit, options, &mut trace, None)
}
pub fn search_measured(
&mut self,
query: &str,
limit: usize,
options: &SearchOptions,
debug: bool,
) -> Result<SearchResponse<SearchResult>> {
let mut trace = TraceCollector::new(if debug {
MetricsMode::Debug
} else {
MetricsMode::Basic
});
let results = self.search_traced(query, limit, options, &mut trace, None)?;
let metrics = trace.finish(results.len());
Ok(SearchResponse { results, metrics })
}
pub fn search_traced(
&mut self,
query: &str,
limit: usize,
options: &SearchOptions,
trace: &mut TraceCollector,
parent: Option<SpanToken>,
) -> Result<Vec<SearchResult>> {
trace.timed("hybrid.search", parent, |trace, parent| {
self.search_inner(query, limit, options, trace, parent)
})
}
fn search_inner(
&mut self,
query: &str,
limit: usize,
options: &SearchOptions,
trace: &mut TraceCollector,
parent: Option<SpanToken>,
) -> Result<Vec<SearchResult>> {
let candidates = limit.saturating_mul(4).clamp(20, 1000);
// Core validates filters and query budgets even for empty requests.
let mut lexical = self
.lexical
.search_with_options(query, candidates, options)?;
.search_with_options_traced(query, candidates, options, trace, parent)?;
if query.trim().is_empty() || limit == 0 {
return Ok(Vec::new());
}
promote_identifiers(query.trim(), &mut lexical);
let vector = self.embedder.embed_query(query.trim())?;
trace.timed("hybrid.identifier_priority", parent, |_, _| {
promote_identifiers(query.trim(), &mut lexical)
});
let embedding_span = trace.begin("semantic.query_embedding", parent);
let vector = if trace.is_debug() {
self.embedder.embed_query_debug(query.trim())
} else {
self.embedder.embed_query(query.trim())
};
trace.counter(embedding_span, "failed", u64::from(vector.is_err()));
trace.end(
embedding_span,
if trace.is_debug() {
self.embedder.query_diagnostics()
} else {
Default::default()
},
);
let vector = vector?;
let semantic =
self.store
.search(&vector, &self.generation_fingerprint, candidates, options)?;
Ok(fuse(lexical, semantic, limit.min(1000)))
self.routed_semantic_traced(query, &vector, candidates, options, trace, parent)?;
let results = trace.timed("hybrid.rank_fusion", parent, |trace, span| {
trace.counter(span, "lexical_files", lexical.len() as u64);
trace.counter(span, "semantic_files", semantic.len() as u64);
fuse(lexical, semantic, limit.min(1000))
});
Ok(results)
}
/// Semantic lane for reproducible relevance evaluation and diagnostics.
@@ -282,12 +424,7 @@ impl HybridSearch {
return Ok(Vec::new());
}
let vector = self.embedder.embed_query(query.trim())?;
self.store.search(
&vector,
&self.generation_fingerprint,
limit.min(1000),
options,
)
self.routed_semantic(query, &vector, limit.min(1000), options)
}
}
@@ -417,6 +554,58 @@ fn fuse(
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn failed_query_inference_retains_diagnostics_in_shared_trace() -> Result<()> {
// Failure-only provider: no fabricated embeddings are generated.
struct FailingProvider;
impl EmbeddingProvider for FailingProvider {
fn fingerprint(&self) -> &str {
"failure-injection"
}
fn dimensions(&self) -> usize {
768
}
fn split_windows(&mut self, _: &str, _: &str) -> Result<Vec<TextWindow>> {
anyhow::bail!("unused")
}
fn embed_documents(&mut self, _: &[&str], _: &str) -> Result<Vec<Vec<f32>>> {
anyhow::bail!("unused")
}
fn embed_query(&mut self, _: &str) -> Result<Vec<f32>> {
anyhow::bail!("injected worker failure")
}
fn query_diagnostics(&self) -> std::collections::BTreeMap<String, u64> {
std::collections::BTreeMap::from([("request_write_us".into(), 7)])
}
}
let directory = tempfile::tempdir()?;
let db = directory.path().join("index.db");
let mut service = HybridSearch::with_provider(&db, Box::new(FailingProvider))?;
let mut trace = TraceCollector::new(MetricsMode::Debug);
assert!(service
.search_traced(
"ordinary query",
5,
&SearchOptions::default(),
&mut trace,
None
)
.is_err());
let metrics = trace.finish(0);
let spans = metrics.trace.unwrap();
let embedding = spans
.iter()
.find(|s| s.name == "semantic.query_embedding")
.unwrap();
assert_eq!(embedding.counters["failed"], 1);
assert_eq!(embedding.counters["request_write_us"], 7);
let parent = &spans[embedding.parent.unwrap()];
assert!(
embedding.start_ms + embedding.duration_ms
<= parent.start_ms + parent.duration_ms + 1e-6
);
Ok(())
}
fn result(path: &str, kind: &str) -> SearchResult {
SearchResult {
path: path.into(),
crates/searchi-service/src/media.rs
+232
View File
@@ -0,0 +1,232 @@
use super::*;
use searchi_media::*;
#[derive(Debug)]
struct InferenceFailure(anyhow::Error);
impl std::fmt::Display for InferenceFailure {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Media inference failed: {}", self.0)
}
}
impl std::error::Error for InferenceFailure {}
impl HybridSearch {
/// Publish text and actual image/frame/audio vectors in one atomic generation.
pub fn index_embeddings_with_media(
&mut self,
tools: &FfmpegTools,
limits: &MediaLimits,
) -> Result<EmbeddingReport> {
let previous_fingerprint = self.generation_fingerprint.clone();
self.generation_fingerprint = generation_fingerprint(self.embedder.fingerprint());
let defaults = MediaLimits::default();
if limits.max_frames != defaults.max_frames
|| limits.frame_interval_ms != defaults.frame_interval_ms
|| limits.max_scene_frames != defaults.max_scene_frames
|| limits.audio_window_ms != defaults.audio_window_ms
|| limits.audio_overlap_ms != defaults.audio_overlap_ms
|| limits.max_audio_windows != defaults.max_audio_windows
|| limits.max_duration_ms != defaults.max_duration_ms
|| limits.max_file_bytes != defaults.max_file_bytes
{
self.generation_fingerprint.push_str(&format!(
":custom-{}-{}-{}-{}-{}-{}-{}-{}",
limits.max_frames,
limits.frame_interval_ms,
limits.max_scene_frames,
limits.audio_window_ms,
limits.audio_overlap_ms,
limits.max_audio_windows,
limits.max_duration_ms,
limits.max_file_bytes
));
}
let _lock = self.store.build_lock()?;
let generation = self.store.begin(&self.generation_fingerprint)?;
let built = (|| {
let mut report = self.build(generation)?;
let mut after = 0;
let mut media_count = 0usize;
loop {
let files = self.store.media_batch(after)?;
if files.is_empty() {
break;
}
for (id, path, _, _) in files {
after = id;
let path = Path::new(&path);
let Some(kind) = classify(path) else {
continue;
};
media_count += 1;
ensure!(
media_count <= 10000,
"Media generation exceeds 10000-file resource budget"
);
let before_chunks = report.chunks;
match self.build_media_file(
generation,
id,
path,
kind,
tools,
limits,
&mut report,
) {
Ok(()) => report.media_files += 1,
Err(error) => {
if error.downcast_ref::<InferenceFailure>().is_some() {
return Err(error);
}
self.store.media_failure(
generation,
&path.to_string_lossy(),
&format!("{error:#}"),
)?;
report.chunks = before_chunks;
if report.failures.len() < 1000 {
report.failures.push(MediaFailure {
path: path.to_string_lossy().into_owned(),
message: format!("{error:#}").chars().take(2048).collect(),
});
}
report.media_failures += 1;
}
}
}
}
Ok(report)
})();
match built {
Ok(report) => {
self.store.activate(generation)?;
Ok(report)
}
Err(error) => {
self.store.abandon(generation)?;
self.generation_fingerprint = previous_fingerprint;
Err(error)
}
}
}
#[allow(clippy::too_many_arguments)]
fn build_media_file(
&mut self,
generation: i64,
id: i64,
path: &Path,
kind: MediaKind,
tools: &FfmpegTools,
limits: &MediaLimits,
report: &mut EmbeddingReport,
) -> Result<()> {
let identity = MediaIdentity::read(
path,
if kind == MediaKind::Image {
64 * 1024 * 1024
} else {
limits.max_file_bytes
},
)?;
let mut ordinal = 0;
if kind == MediaKind::Image {
validate_image(path)?;
ensure!(
report.chunks < MAX_SCAN_CHUNKS,
"Media exceeds generation chunk budget"
);
let vectors = self
.embedder
.embed_images(&[path], 280)
.map_err(|e| anyhow::Error::new(InferenceFailure(e)))?;
ensure!(vectors.len() == 1, "Image embedding count mismatch");
self.store
.insert_media(generation, id, ordinal, &identity, "image", 0, &vectors[0])?;
report.chunks += 1;
} else {
let metadata = probe(tools, path, limits)?;
let tags = metadata
.tags
.iter()
.map(|(k, v)| format!("{k}: {v}"))
.collect::<Vec<_>>()
.join("\n");
self.lexical.set_media_metadata(
&path.to_string_lossy(),
&tags,
metadata.duration_ms,
identity.bytes,
identity.modified_ns,
)?;
if metadata.has_video && kind == MediaKind::Video {
let mut seen = std::collections::HashSet::new();
let mut scenes = 0;
for time in frame_times(&metadata, limits)? {
let frames = decode_frames(
tools,
path,
&metadata,
time,
scenes < limits.max_scene_frames,
limits,
)?;
for (i, frame) in frames.frames.iter().enumerate() {
if !seen.insert(frame.timestamp_ms) {
continue;
}
if i > 0 {
scenes += 1;
}
ensure!(
report.chunks < MAX_SCAN_CHUNKS,
"Media exceeds generation chunk budget"
);
let vectors = self
.embedder
.embed_images(&[frame.path.as_path()], 70)
.map_err(|e| anyhow::Error::new(InferenceFailure(e)))?;
ensure!(vectors.len() == 1, "Frame embedding count mismatch");
self.store.insert_media(
generation,
id,
ordinal,
&identity,
"video",
frame.timestamp_ms,
&vectors[0],
)?;
ordinal += 1;
report.chunks += 1;
}
}
}
if metadata.has_audio {
for window in audio_windows(&metadata, limits)? {
ensure!(
report.chunks < MAX_SCAN_CHUNKS,
"Audio exceeds generation chunk budget"
);
let pcm = decode_audio(tools, path, window, limits)?;
let vector = self
.embedder
.embed_audio(&pcm)
.map_err(|e| anyhow::Error::new(InferenceFailure(e)))?;
self.store.insert_media(
generation,
id,
ordinal,
&identity,
"audio",
window.start_ms,
&vector,
)?;
ordinal += 1;
report.chunks += 1;
}
}
}
identity.ensure_unchanged()?;
Ok(())
}
}
crates/searchi-service/src/routing.rs
+213
View File
@@ -0,0 +1,213 @@
use super::*;
/// Explicit semantic source channel, independent of filename/type filters.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SemanticOrigin {
All,
Text,
Image,
Video,
Audio,
}
impl SemanticOrigin {
fn source_type(self) -> Option<&'static str> {
match self {
Self::All => None,
Self::Text => Some("text"),
Self::Image => Some("image"),
Self::Video => Some("video"),
Self::Audio => Some("audio"),
}
}
}
/// A conservative intent hint, not a language understanding model. Explicit
/// document/visual cues veto it; ordinary topic and instrument queries do not
/// silently restrict modality. Word boundaries avoid filename substrings.
fn preferred_origin(query: &str) -> Option<SemanticOrigin> {
if query.split_whitespace().count() == 1 && Path::new(query).extension().is_some() {
return None;
}
let lower = query.to_lowercase();
let words: Vec<_> = lower
.split(|c: char| !c.is_alphanumeric())
.filter(|s| !s.is_empty())
.collect();
let has = |values: &[&str]| words.iter().any(|word| values.contains(word));
if has(&[
"image",
"photo",
"picture",
"screenshot",
"diagram",
"document",
"pdf",
"text",
"paper",
"bild",
"foto",
"screenshot",
"dokument",
"textdatei",
]) {
return None;
}
if has(&[
"audio",
"spoken",
"speech",
"recording",
"conversation",
"interview",
"podcast",
"sound",
"music",
"gesprochen",
"gespräch",
"gespraech",
"geräusch",
"geraeusch",
"musik",
"hörbar",
"hörbarer",
"hörbare",
"hoerbar",
"hoerbarer",
"hoerbare",
"tonaufnahme",
]) {
return Some(SemanticOrigin::Audio);
}
if (has(&["someone", "speaker"])
&& has(&[
"mentions",
"mentioned",
"says",
"said",
"speaks",
"speaking",
]))
|| (has(&["jemand", "sprecher", "sprecherin"])
&& has(&["erwähnt", "erwaehnt", "sagt", "gesagt", "spricht"]))
{
return Some(SemanticOrigin::Audio);
}
None
}
impl HybridSearch {
pub(super) fn routed_semantic(
&mut self,
query: &str,
vector: &[f32],
limit: usize,
options: &SearchOptions,
) -> Result<Vec<SearchResult>> {
let mut trace = TraceCollector::new(MetricsMode::Basic);
self.routed_semantic_traced(query, vector, limit, options, &mut trace, None)
}
pub(super) fn routed_semantic_traced(
&mut self,
query: &str,
vector: &[f32],
limit: usize,
options: &SearchOptions,
trace: &mut TraceCollector,
parent: Option<SpanToken>,
) -> Result<Vec<SearchResult>> {
trace.timed("semantic.routing", parent, |trace, parent| {
if let Some(origin) = preferred_origin(query) {
trace.counter(parent, "audio_preference", 1);
let preferred = self.store.search_origin_traced(
vector,
&self.generation_fingerprint,
limit,
options,
origin.source_type(),
trace,
parent,
)?;
if !preferred.is_empty() {
return Ok(preferred);
}
}
trace.counter(parent, "general_lane", 1);
self.store.search_origin_traced(
vector,
&self.generation_fingerprint,
limit,
options,
None,
trace,
parent,
)
})
}
/// Query one actual source channel, or All to bypass automatic spoken/audio
/// preference. This never embeds files again or changes an active policy.
pub fn search_semantic_with_origin(
&mut self,
query: &str,
limit: usize,
options: &SearchOptions,
origin: SemanticOrigin,
) -> Result<Vec<SearchResult>> {
let _ = self.lexical.search_with_options(query, 0, options)?;
if query.trim().is_empty() || limit == 0 {
return Ok(Vec::new());
}
let vector = self.embedder.embed_query(query.trim())?;
self.store.search_origin(
&vector,
&self.generation_fingerprint,
limit.min(1000),
options,
origin.source_type(),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn persisted_custom_limits_require_identical_model_and_preprocessing() {
let expected = "pinned-model:pinned-media:preprocessing-v1";
assert!(reusable_policy(
expected,
&format!("{expected}:custom-240-2000-120-10000-2000-1800-14400000-17179869184")
));
for active in ["other-model:pinned-media:preprocessing-v1:custom-240-2000-120-10000-2000-1800-14400000-17179869184","pinned-model:pinned-media:preprocessing-v2:custom-240-2000-120-10000-2000-1800-14400000-17179869184","pinned-model:pinned-media:preprocessing-v1:custom-unknown"] {assert!(!reusable_policy(expected,active));}
}
#[test]
fn explicit_spoken_intent_is_conservative_and_bilingual() {
for query in [
"where someone mentions a contract",
"a speaker says the date",
"spoken discussion of ecology",
"audio of a waterfall",
"jemand erwähnt das Budget",
"Wo spricht jemand über Kündigung?",
"Gespräch über die neue Wohnung",
"hörbarer Regen",
] {
assert_eq!(
preferred_origin(query),
Some(SemanticOrigin::Audio),
"{query}"
);
}
for query in [
"contract deadline",
"violin solo",
"photo of someone speaking",
"Bild von einem Sprecher",
"a document that someone mentioned",
"speech.png",
"a screenshot of an audio waveform",
] {
assert_eq!(preferred_origin(query), None, "{query}");
}
}
}
crates/searchi-service/src/store.rs
+651 -24
View File
@@ -8,6 +8,7 @@ use std::{
use anyhow::{ensure, Context, Result};
use fs2::FileExt;
use rusqlite::{params, Connection, OptionalExtension};
use searchi_core::metrics::{MetricsMode, SpanToken, TraceCollector};
use searchi_core::{IndexedPassage, SearchOptions, SearchResult, SourceLocation};
use searchi_semantic::TextWindow;
use sha2::{Digest, Sha256};
@@ -24,6 +25,9 @@ struct FilterKey {
max_size: Option<i64>,
modified_after: Option<i64>,
modified_before: Option<i64>,
min_duration: Option<u32>,
max_duration: Option<u32>,
source_type: Option<String>,
}
struct EligibilityCache {
@@ -57,8 +61,13 @@ pub(crate) fn content_hash(text: &str) -> String {
#[cfg(test)]
thread_local! { static HASH_CALLS:std::cell::Cell<usize>=const {std::cell::Cell::new(0)}; }
#[cfg(test)]
thread_local! { static MEDIA_STAT_CALLS:std::cell::Cell<usize>=const {std::cell::Cell::new(0)}; }
impl Store {
pub(crate) fn active_fingerprint(&self) -> Result<Option<String>> {
Ok(self.connection.query_row("SELECT g.fingerprint FROM semantic_meta m JOIN semantic_generations g ON g.id=m.active_generation WHERE m.singleton=1",[],|r|r.get(0)).optional()?)
}
pub(crate) fn open(db: &Path, dimensions: usize) -> Result<Self> {
ensure!(
(1..=4096).contains(&dimensions),
@@ -85,7 +94,7 @@ impl Store {
|row| row.get(0),
)?;
ensure!(
(1..=5).contains(&version),
(1..=6).contains(&version),
"Unsupported semantic schema version {version}"
);
initialization.execute_batch("
@@ -130,8 +139,25 @@ impl Store {
UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
INSERT INTO semantic_dirty_paths(path,revision) VALUES(old.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;
UPDATE semantic_meta SET schema_version=5 WHERE singleton=1;",
UPDATE semantic_meta SET schema_version=6 WHERE singleton=1;",
)?;
for (column, declaration) in [
("source_type", "TEXT NOT NULL DEFAULT 'text'"),
("file_size", "INTEGER"),
("file_mtime", "INTEGER"),
] {
let exists: bool = initialization.query_row(
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('semantic_chunks') WHERE name=?1)",
[column],
|r| r.get(0),
)?;
if !exists {
initialization.execute_batch(&format!(
"ALTER TABLE semantic_chunks ADD COLUMN {column} {declaration}"
))?;
}
}
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?;
initialization.commit()?;
let canonical = db.canonicalize()?;
let lock_path = PathBuf::from(format!("{}.semantic.lock", canonical.to_string_lossy()));
@@ -170,6 +196,67 @@ impl Store {
Ok(id)
}
pub(crate) fn media_batch(&self, after: i64) -> Result<Vec<(i64, String, i64, i64)>> {
let mut q = self.connection.prepare(
"SELECT rowid,path,size,mtime FROM files WHERE rowid>?1 ORDER BY rowid LIMIT 64",
)?;
let rows = q
.query_map([after], |r| {
Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows)
}
pub(crate) fn media_failure(&self, generation: i64, path: &str, message: &str) -> Result<()> {
self.connection.execute(
"DELETE FROM semantic_chunks WHERE generation=?1 AND path=?2 AND source_type!='text'",
params![generation, path],
)?;
self.connection.execute("INSERT OR REPLACE INTO semantic_media_errors(generation,path,message) VALUES(?1,?2,?3)",params![generation,path,message.chars().take(2048).collect::<String>()])?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn insert_media(
&self,
generation: i64,
file_id: i64,
ordinal: usize,
identity: &searchi_media::MediaIdentity,
source_type: &str,
timestamp: u32,
vector: &[f32],
) -> Result<()> {
ensure!(
vector.len() == self.dimensions && vector.iter().all(|v| v.is_finite()),
"Invalid media embedding"
);
let norm = vector.iter().map(|v| v * v).sum::<f32>().sqrt();
ensure!(
(norm - 1.).abs() < 1e-3,
"Media embeddings must be normalized"
);
let path = identity.path.to_string_lossy();
let valid:bool=self.connection.query_row("SELECT EXISTS(SELECT 1 FROM files WHERE rowid=?1 AND path=?2 AND size=?3 AND mtime=?4)",params![file_id,path,identity.bytes as i64,identity.modified_ns],|r|r.get(0))?;
ensure!(valid, "Media file changed since filesystem indexing");
let kind = if source_type == "image" {
"image"
} else {
"timestamp_ms"
};
let snippet = if source_type == "image" {
"Image content".to_string()
} else {
format!(
"{} content at {:.3} seconds",
source_type,
f64::from(timestamp) / 1000.
)
};
let bytes: Vec<u8> = vector.iter().flat_map(|v| v.to_le_bytes()).collect();
self.connection.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime) VALUES(?1,?2,?3,?4,?5,?6,?7,0,0,?8,?9,?10,?11,?12)",params![generation,-file_id,ordinal as i64,path,kind,timestamp,identity.sha256,snippet,bytes,source_type,identity.bytes as i64,identity.modified_ns])?;
Ok(())
}
pub(crate) fn insert(
&mut self,
generation: i64,
@@ -312,6 +399,61 @@ impl Store {
fingerprint: &str,
limit: usize,
options: &SearchOptions,
) -> Result<Vec<SearchResult>> {
self.search_origin(query, fingerprint, limit, options, None)
}
pub(crate) fn search_origin(
&mut self,
query: &[f32],
fingerprint: &str,
limit: usize,
options: &SearchOptions,
source_type: Option<&str>,
) -> Result<Vec<SearchResult>> {
let mut trace = TraceCollector::new(MetricsMode::Basic);
self.search_origin_traced(
query,
fingerprint,
limit,
options,
source_type,
&mut trace,
None,
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn search_origin_traced(
&mut self,
query: &[f32],
fingerprint: &str,
limit: usize,
options: &SearchOptions,
source_type: Option<&str>,
trace: &mut TraceCollector,
parent: Option<SpanToken>,
) -> Result<Vec<SearchResult>> {
trace.timed("semantic.retrieval", parent, |trace, parent| {
self.search_inner(
query,
fingerprint,
limit,
options,
source_type,
trace,
parent,
)
})
}
#[allow(clippy::too_many_arguments)]
fn search_inner(
&mut self,
query: &[f32],
fingerprint: &str,
limit: usize,
options: &SearchOptions,
source_type: Option<&str>,
trace: &mut TraceCollector,
parent: Option<SpanToken>,
) -> Result<Vec<SearchResult>> {
ensure!(
query.len() == self.dimensions && query.iter().all(|v| v.is_finite()),
@@ -321,17 +463,30 @@ impl Store {
return Ok(Vec::new());
}
// A consistent read snapshot covers eligibility and materialization.
let generation_span = trace.begin("semantic.generation_snapshot", parent);
let snapshot = self.connection.unchecked_transaction()?;
let active:Option<i64>=snapshot.query_row("SELECT g.id FROM semantic_generations g JOIN semantic_meta m ON m.active_generation=g.id WHERE g.fingerprint=?1 AND g.dimensions=?2",
params![fingerprint,self.dimensions as i64],|row|row.get(0)).optional()?;
let Some(active) = active else {
return Ok(Vec::new());
};
trace.end(generation_span, Default::default());
let ann_load = trace.begin("semantic.ann_load", parent);
trace.counter(
ann_load,
"cache_hit",
u64::from(
self.ann_cache
.as_ref()
.is_some_and(|(generation, _)| *generation == active),
),
);
if !self
.ann_cache
.as_ref()
.is_some_and(|(generation, _)| *generation == active)
{
let blob_span = trace.begin("semantic.ann_blob_read", ann_load);
let length: Option<usize> = snapshot.query_row(
"SELECT length(ann_blob) FROM semantic_generations WHERE id=?1",
[active],
@@ -346,10 +501,15 @@ impl Store {
[active],
|row| Ok((row.get(0)?, row.get(1)?)),
)?;
trace.counter(blob_span, "bytes", serialized.len() as u64);
trace.end(blob_span, Default::default());
let checksum_span = trace.begin("semantic.ann_checksum", ann_load);
ensure!(
format!("{:x}", Sha256::digest(&serialized)) == checksum,
"ANN generation SHA256 mismatch; rebuild embeddings explicitly"
);
trace.end(checksum_span, Default::default());
let deserialize_span = trace.begin("semantic.ann_deserialize", ann_load);
let ann = self.new_ann()?;
ann.load_from_buffer(&serialized)?;
ensure!(
@@ -357,8 +517,12 @@ impl Store {
"ANN dimensions differ from embedding provider"
);
ann.reserve_capacity_and_threads(ann.size().max(1), 1)?;
trace.counter(deserialize_span, "chunks", ann.size() as u64);
trace.end(deserialize_span, Default::default());
self.ann_cache = Some((active, ann));
}
trace.end(ann_load, Default::default());
let filter_span = trace.begin("semantic.filter_normalization", parent);
let folder = options
.folder
.as_ref()
@@ -384,13 +548,25 @@ impl Store {
max_size,
modified_after: options.modified_after_unix_ms,
modified_before: options.modified_before_unix_ms,
min_duration: options.min_duration_ms,
max_duration: options.max_duration_ms,
source_type: source_type.map(str::to_owned),
};
trace.end(filter_span, Default::default());
let eligibility_span = trace.begin("semantic.eligibility", parent);
let (revision, floor): (i64, i64) = snapshot.query_row(
"SELECT corpus_revision,dirty_floor FROM semantic_meta WHERE singleton=1",
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)?;
let existing = self.eligibility_cache.take();
trace.counter(
eligibility_span,
"cache_hit",
u64::from(existing.as_ref().is_some_and(|c| {
c.generation == active && c.filter == filter && c.revision == revision
})),
);
let mut cache = match existing {
Some(mut cache)
if cache.generation == active
@@ -404,8 +580,17 @@ impl Store {
row.get::<_, String>(0)
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
trace.counter(eligibility_span, "dirty_paths", paths.len() as u64);
if paths.len() > 1000 {
cache = collect_eligibility(&snapshot, active, revision, &filter, None)?;
cache = collect_eligibility_traced(
&snapshot,
active,
revision,
&filter,
None,
trace,
eligibility_span,
)?;
} else {
for path in paths {
if let Some(old) = cache.by_path.remove(&path) {
@@ -413,12 +598,14 @@ impl Store {
Arc::make_mut(&mut cache.keys).remove(&key);
}
}
let changed = collect_eligibility(
let changed = collect_eligibility_traced(
&snapshot,
active,
revision,
&filter,
Some(&path),
trace,
eligibility_span,
)?;
Arc::make_mut(&mut cache.keys).extend(changed.keys.iter().copied());
cache.by_path.extend(changed.by_path);
@@ -428,12 +615,22 @@ impl Store {
}
cache
}
_ => collect_eligibility(&snapshot, active, revision, &filter, None)?,
_ => collect_eligibility_traced(
&snapshot,
active,
revision,
&filter,
None,
trace,
eligibility_span,
)?,
};
ensure!(
cache.keys.len() <= MAX_SCAN_CHUNKS,
"Eligibility cache exceeds chunk resource budget"
);
trace.counter(eligibility_span, "eligible_chunks", cache.keys.len() as u64);
trace.end(eligibility_span, Default::default());
cache.revision = revision;
let eligible = Arc::clone(&cache.keys);
self.eligibility_cache = Some(cache);
@@ -443,40 +640,97 @@ impl Store {
let ann = &self.ann_cache.as_ref().expect("loaded ANN cache").1;
let mut candidate_count = limit.saturating_mul(4).min(eligible.len()).max(1);
let mut results;
let mut filesystem_validity = HashMap::<String, bool>::new();
let mut passes = 0u64;
loop {
passes += 1;
let pass_span = trace.begin("semantic.ann_pass", parent);
trace.counter(pass_span, "requested_chunks", candidate_count as u64);
let ann_span = trace.begin("semantic.ann_search", pass_span);
let matches =
ann.filtered_search(query, candidate_count, |key| eligible.contains(&key))?;
trace.counter(ann_span, "returned_chunks", matches.keys.len() as u64);
trace.end(ann_span, Default::default());
let materialization_span = trace.begin("semantic.parent_materialization", pass_span);
let mut ranked: Vec<_> = matches.keys.into_iter().zip(matches.distances).collect();
ranked.sort_by(|a, b| a.1.total_cmp(&b.1).then_with(|| a.0.cmp(&b.0)));
let mut best = HashMap::<String, SearchResult>::new();
let mut materialize=snapshot.prepare("SELECT c.path,f.filename,c.snippet,c.kind,c.number FROM semantic_chunks c JOIN files f ON f.path=c.path WHERE c.rowid=?1 AND c.generation=?2")?;
let mut materialize=snapshot.prepare("SELECT c.path,f.filename,c.snippet,c.kind,c.number,c.source_type,c.file_size,c.file_mtime FROM semantic_chunks c JOIN files f ON f.path=c.path WHERE c.rowid=?1 AND c.generation=?2")?;
for (key, distance) in ranked {
let cosine = 1.0 - distance;
ensure!(cosine.is_finite(), "ANN produced nonfinite distance");
if cosine < MIN_COSINE {
continue;
}
let result =
let (result, size, mtime) =
materialize.query_row(params![i64::try_from(key)?, active], |row| {
Ok(SearchResult {
path: row.get(0)?,
filename: row.get(1)?,
snippet: row.get(2)?,
score: f64::from(cosine),
match_kind: "semantic".into(),
source: Some(SourceLocation {
kind: row.get(3)?,
number: row.get(4)?,
}),
extraction_error: None,
})
Ok((
SearchResult {
path: row.get(0)?,
filename: row.get(1)?,
snippet: row.get(2)?,
score: f64::from(cosine),
match_kind: match row.get::<_, String>(5)?.as_str() {
"text" => "semantic".into(),
other => format!("semantic_{other}"),
},
source: Some(SourceLocation {
kind: row.get(3)?,
number: row.get(4)?,
}),
extraction_error: None,
},
row.get::<_, Option<i64>>(6)?,
row.get::<_, Option<i64>>(7)?,
))
})?;
if let (Some(size), Some(mtime)) = (size, mtime) {
let valid = *filesystem_validity
.entry(result.path.clone())
.or_insert_with(|| {
#[cfg(test)]
MEDIA_STAT_CALLS.with(|count| count.set(count.get() + 1));
let validation_span = trace.begin(
"semantic.parent_filesystem_validation",
materialization_span,
);
let valid =
std::fs::metadata(&result.path)
.ok()
.is_some_and(|metadata| {
metadata.is_file()
&& i64::try_from(metadata.len()).ok() == Some(size)
&& metadata
.modified()
.ok()
.and_then(|t| {
t.duration_since(std::time::UNIX_EPOCH).ok()
})
.and_then(|d| i64::try_from(d.as_nanos()).ok())
== Some(mtime)
});
trace.counter(validation_span, "valid", u64::from(valid));
trace.end(validation_span, Default::default());
valid
});
if !valid {
continue;
}
}
if best.contains_key(&result.path) {
continue;
}
best.insert(result.path.clone(), result);
}
results = best.into_values().collect::<Vec<_>>();
trace.counter(materialization_span, "unique_parents", results.len() as u64);
trace.counter(
materialization_span,
"filesystem_checks",
filesystem_validity.len() as u64,
);
trace.end(materialization_span, Default::default());
trace.end(pass_span, Default::default());
if results.len() >= limit || candidate_count == eligible.len() {
break;
}
@@ -488,6 +742,13 @@ impl Store {
.then_with(|| a.path.cmp(&b.path))
});
results.truncate(limit);
trace.counter(parent, "ann_passes", passes);
trace.counter(parent, "result_files", results.len() as u64);
trace.counter(
parent,
"filesystem_checks",
filesystem_validity.len() as u64,
);
snapshot.commit()?;
Ok(results)
}
@@ -614,6 +875,30 @@ impl Store {
}
}
#[allow(clippy::too_many_arguments)]
fn collect_eligibility_traced(
connection: &Connection,
generation: i64,
revision: i64,
filter: &FilterKey,
path: Option<&str>,
trace: &mut TraceCollector,
parent: Option<SpanToken>,
) -> Result<EligibilityCache> {
trace.timed(
"semantic.eligibility_filtered_scan_and_hash",
parent,
|trace, span| {
let result = collect_eligibility(connection, generation, revision, filter, path);
if let Ok(cache) = &result {
trace.counter(span, "eligible_chunks", cache.keys.len() as u64);
}
trace.counter(span, "single_dirty_file", u64::from(path.is_some()));
result
},
)
}
fn collect_eligibility(
connection: &Connection,
generation: i64,
@@ -621,12 +906,15 @@ fn collect_eligibility(
filter: &FilterKey,
path: Option<&str>,
) -> Result<EligibilityCache> {
let mut statement=connection.prepare("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path FROM semantic_chunks c
JOIN passages p ON p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
JOIN files f ON f.path=c.path WHERE c.generation=?1
let mut statement=connection.prepare("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c
LEFT JOIN passages p ON c.source_type='text' AND p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
JOIN files f ON f.path=c.path WHERE c.generation=?1 AND ((c.source_type='text' AND p.rowid IS NOT NULL) OR (c.source_type!='text' AND c.file_size=f.size AND c.file_mtime=f.mtime))
AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2) AND (?3 IS NULL OR f.extension=?3)
AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10))
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11))
AND (?12 IS NULL OR c.source_type=?12)
AND (?9 IS NULL OR c.path=?9) ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
let mut rows = statement.query(params![
generation,
@@ -637,7 +925,10 @@ fn collect_eligibility(
filter.modified_after,
filter.modified_before,
MAX_SCAN_CHUNKS as i64 + 1,
path
path,
filter.min_duration,
filter.max_duration,
filter.source_type
])?;
let mut hashes = HashMap::<i64, String>::new();
let mut keys = HashSet::new();
@@ -650,7 +941,9 @@ fn collect_eligibility(
"Corpus exceeds {MAX_SCAN_CHUNKS} chunk resource budget"
);
let passage_id: i64 = row.get(1)?;
let hash = if let Some(hash) = hashes.get(&passage_id) {
let hash = if row.get::<_, String>(5)? != "text" {
row.get::<_, String>(2)?
} else if let Some(hash) = hashes.get(&passage_id) {
hash.clone()
} else {
let hash = content_hash(&row.get::<_, String>(3)?);
@@ -678,6 +971,340 @@ mod tests {
use searchi_core::SearchIndex;
const DIMENSIONS: usize = 768;
#[test]
fn many_video_windows_do_not_crowd_out_other_parent_files() -> Result<()> {
// Geometry verifies grouped retrieval, not model inference or codecs.
let directory = tempfile::tempdir()?;
let root = directory.path().join("media");
std::fs::create_dir(&root)?;
for name in ["long.mp4", "other.mp4", "excluded.png"] {
std::fs::write(root.join(name), b"parent-grouping identity fixture")?;
}
let db = directory.path().join("index.db");
let mut lexical = SearchIndex::open(&db)?;
lexical.index_folder(&root)?;
let mut store = Store::open(&db, DIMENSIONS)?;
let generation = store.begin("grouped-retrieval-proof")?;
let mut query = vec![0.; DIMENSIONS];
query[0] = 1.;
let mut second = query.clone();
second[0] = 0.95;
second[1] = (1.0_f32 - 0.95_f32.powi(2)).sqrt();
for (id, path, _, _) in store.media_batch(0)? {
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
if path.ends_with("long.mp4") {
for ordinal in 0..96 {
store.insert_media(
generation,
id,
ordinal,
&identity,
"video",
ordinal as u32 * 2000,
&query,
)?;
}
} else if path.ends_with("other.mp4") {
store.insert_media(generation, id, 0, &identity, "video", 4000, &second)?;
} else {
store.insert_media(generation, id, 0, &identity, "image", 0, &query)?;
}
}
store.activate(generation)?;
let options = SearchOptions {
extension: Some("mp4".into()),
..Default::default()
};
MEDIA_STAT_CALLS.with(|count| count.set(0));
let mut trace = TraceCollector::new(MetricsMode::Debug);
let results = store.search_origin_traced(
&query,
"grouped-retrieval-proof",
2,
&options,
None,
&mut trace,
None,
)?;
assert_eq!(results.len(), 2);
assert!(results[0].path.ends_with("long.mp4"));
assert!(results[1].path.ends_with("other.mp4"));
assert_eq!(results[1].source.as_ref().unwrap().number, 4000);
assert_eq!(
MEDIA_STAT_CALLS.with(std::cell::Cell::get),
2,
"Each parent is stat-checked once despite repeated ANN expansions"
);
let metrics = trace.finish(results.len());
let spans = metrics.trace.unwrap();
assert!(
spans
.iter()
.filter(|s| s.name == "semantic.ann_pass")
.count()
> 1
);
assert_eq!(
spans
.iter()
.filter(|s| s.name == "semantic.parent_filesystem_validation")
.count(),
2
);
assert!(
spans
.iter()
.find(|s| s.name == "semantic.retrieval")
.unwrap()
.counters["ann_passes"]
> 1
);
for span in &spans {
if let Some(parent) = span.parent {
let parent = &spans[parent];
assert!(span.start_ms >= parent.start_ms);
assert!(
span.start_ms + span.duration_ms <= parent.start_ms + parent.duration_ms + 1e-6
);
}
}
Ok(())
}
#[test]
fn media_origins_persist_filters_and_changes_remove_stale_candidates() -> Result<()> {
let directory = tempfile::tempdir()?;
let root = directory.path().join("media");
std::fs::create_dir(&root)?;
let file = root.join("clip.mp4");
std::fs::write(&file, b"storage identity fixture; not an inference test")?;
let db = directory.path().join("index.db");
let mut lexical = SearchIndex::open(&db)?;
lexical.index_folder(&root)?;
let mut store = Store::open(&db, DIMENSIONS)?;
let (id, path, _, _) = store.media_batch(0)?.remove(0);
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
lexical.set_media_metadata(
&path,
"artist: real extracted metadata fixture",
65000,
identity.bytes,
identity.modified_ns,
)?;
let mut vector = vec![0.; DIMENSIONS];
vector[0] = 1.;
let generation = store.begin("media-storage-proof")?;
store.insert_media(generation, id, 0, &identity, "video", 2345, &vector)?;
store.insert_media(generation, id, 1, &identity, "audio", 28000, &vector)?;
store.activate(generation)?;
drop(store);
let mut store = Store::open(&db, DIMENSIONS)?;
let mut cold = TraceCollector::new(MetricsMode::Debug);
let results = store.search_origin_traced(
&vector,
"media-storage-proof",
5,
&SearchOptions::default(),
None,
&mut cold,
None,
)?;
let cold = cold.finish(results.len()).trace.unwrap();
assert_eq!(
cold.iter()
.find(|s| s.name == "semantic.ann_load")
.unwrap()
.counters["cache_hit"],
0
);
assert!(cold.iter().any(|s| s.name == "semantic.ann_checksum"));
let mut warm = TraceCollector::new(MetricsMode::Debug);
let warm_results = store.search_origin_traced(
&vector,
"media-storage-proof",
5,
&SearchOptions::default(),
None,
&mut warm,
None,
)?;
assert_eq!(
serde_json::to_value(&results)?,
serde_json::to_value(&warm_results)?
);
let warm = warm.finish(warm_results.len()).trace.unwrap();
assert_eq!(
warm.iter()
.find(|s| s.name == "semantic.ann_load")
.unwrap()
.counters["cache_hit"],
1
);
assert_eq!(
warm.iter()
.find(|s| s.name == "semantic.eligibility")
.unwrap()
.counters["cache_hit"],
1
);
assert!(!warm.iter().any(|s| s.name == "semantic.ann_blob_read"));
assert_eq!(results.len(), 1);
assert_eq!(results[0].source.as_ref().unwrap().kind, "timestamp_ms");
assert!([2345, 28000].contains(&results[0].source.as_ref().unwrap().number));
let audio = store.search_origin(
&vector,
"media-storage-proof",
5,
&SearchOptions::default(),
Some("audio"),
)?;
assert_eq!(audio.len(), 1);
assert_eq!(audio[0].match_kind, "semantic_audio");
assert_eq!(audio[0].source.as_ref().unwrap().number, 28000);
let video = store.search_origin(
&vector,
"media-storage-proof",
5,
&SearchOptions::default(),
Some("video"),
)?;
assert_eq!(video.len(), 1);
assert_eq!(video[0].match_kind, "semantic_video");
assert_eq!(video[0].source.as_ref().unwrap().number, 2345);
assert!(store
.search_origin(
&vector,
"media-storage-proof",
5,
&SearchOptions::default(),
Some("image")
)?
.is_empty());
assert!(
results[0].match_kind == "semantic_video" || results[0].match_kind == "semantic_audio"
);
let filter = SearchOptions {
extension: Some("png".into()),
..SearchOptions::default()
};
assert!(store
.search(&vector, "media-storage-proof", 5, &filter)?
.is_empty());
let duration_filter = SearchOptions {
min_duration_ms: Some(66000),
..Default::default()
};
assert!(store
.search(&vector, "media-storage-proof", 5, &duration_filter)?
.is_empty());
let duration_filter = SearchOptions {
min_duration_ms: Some(64000),
max_duration_ms: Some(66000),
..Default::default()
};
assert_eq!(
store
.search(&vector, "media-storage-proof", 5, &duration_filter)?
.len(),
1
);
std::fs::write(&file, b"changed")?;
// Foreground hits check only actual candidate metadata, before watcher
// reconciliation; they never rehash the complete filesystem corpus.
assert!(store
.search(&vector, "media-storage-proof", 5, &SearchOptions::default())?
.is_empty());
lexical.index_folder(&root)?;
assert!(store
.search(&vector, "media-storage-proof", 5, &SearchOptions::default())?
.is_empty());
assert!(store
.insert_media(generation, id, 2, &identity, "video", 0, &vector)
.is_err());
std::fs::remove_file(&file)?;
lexical.index_folder(&root)?;
assert!(store
.search(&vector, "media-storage-proof", 5, &SearchOptions::default())?
.is_empty());
Ok(())
}
#[test]
fn media_failure_removes_partial_vectors_and_persists_reason() -> Result<()> {
let directory = tempfile::tempdir()?;
let file = directory.path().join("picture.png");
std::fs::write(&file, b"identity fixture")?;
let db = directory.path().join("index.db");
let mut lexical = SearchIndex::open(&db)?;
lexical.index_folder(directory.path())?;
let mut store = Store::open(&db, DIMENSIONS)?;
let (id, path, _, _) = store
.media_batch(0)?
.into_iter()
.find(|(_, p, _, _)| p.ends_with("picture.png"))
.unwrap();
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
let mut vector = vec![0.; DIMENSIONS];
vector[0] = 1.;
let generation = store.begin("failure-proof")?;
store.insert_media(generation, id, 0, &identity, "image", 0, &vector)?;
store.media_failure(generation, &path, "Decoder rejected corrupt image")?;
let chunks: usize = store.connection.query_row(
"SELECT count(*) FROM semantic_chunks WHERE generation=?1",
[generation],
|r| r.get(0),
)?;
assert_eq!(chunks, 0);
let reason: String = store.connection.query_row(
"SELECT message FROM semantic_media_errors WHERE generation=?1",
[generation],
|r| r.get(0),
)?;
assert!(reason.contains("corrupt"));
Ok(())
}
#[test]
fn deleted_media_without_reconciliation_does_not_hide_later_valid_top_k() -> Result<()> {
let directory = tempfile::tempdir()?;
let root = directory.path().join("images");
std::fs::create_dir(&root)?;
for name in ["a.png", "b.png"] {
std::fs::write(root.join(name), b"storage fixture")?;
}
let db = directory.path().join("index.db");
let mut lexical = SearchIndex::open(&db)?;
lexical.index_folder(&root)?;
let mut store = Store::open(&db, DIMENSIONS)?;
let generation = store.begin("candidate-stat-proof")?;
let mut vector = vec![0.; DIMENSIONS];
vector[0] = 1.;
for (id, path, _, _) in store.media_batch(0)? {
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
store.insert_media(generation, id, 0, &identity, "image", 0, &vector)?;
}
store.activate(generation)?;
std::fs::remove_file(root.join("a.png"))?;
let results = store.search(
&vector,
"candidate-stat-proof",
1,
&SearchOptions::default(),
)?;
assert_eq!(results.len(), 1);
assert!(results[0].path.ends_with("b.png"));
std::fs::remove_file(root.join("b.png"))?;
assert!(store
.search(
&vector,
"candidate-stat-proof",
1,
&SearchOptions::default()
)?
.is_empty());
Ok(())
}
#[test]
fn generation_persists_failed_staging_preserves_active_and_changes_hide_stale_vectors(
) -> Result<()> {
docs/BUILDING.md
+15 -6
View File
@@ -22,6 +22,7 @@ The Python CMake package is needed only when CMake is absent from PATH.
python -m pip install cmake
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
python tools/embeddinggemma2/stage_runtime.py --backend cpu
python tools/media-runtime/provision.py --stage
python tools/embeddinggemma2/download.py
$env:SEARCHI_FORCE_CPU = '1'
$env:SEARCHI_GEMMA_MODEL_DIR = (Resolve-Path .searchi/models/embeddinggemma2).Path
@@ -38,8 +39,7 @@ The worker is statically linked with the MSVC CRT on Windows. Staging copies
it and the llama.cpp license into the desktop resources directory. Add
`--bundle-model` to staging for an offline model bundle after downloading;
otherwise desktop setup provisions verified models on first launch. Search
itself performs no network inference or downloads. Media weights are staged
for future integration; the current worker supports text only.
itself performs no network inference or downloads. The worker now uses the verified media projector for genuine image/audio embeddings. The FFmpeg script stages verified standalone decoders and their notices.
The desktop is a standalone Cargo workspace and needs separate validation:
@@ -50,7 +50,7 @@ npm ci
npm run format:check
npm test
npm run build
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu'
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
npm run tauri -- dev
```
@@ -67,13 +67,18 @@ Linux release. On a native Linux build host, run from the repository root:
```sh
python3 tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
python3 tools/embeddinggemma2/download.py
python3 tools/media-runtime/provision.py
export SEARCHI_FORCE_CPU=1
cargo build --release -p searchi-cli --locked
./target/release/searchi index --db searchi.sqlite --root /path/to/documents
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --cpu-threads 1
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --ffmpeg .searchi/media-runtime/bin/ffmpeg --ffprobe .searchi/media-runtime/bin/ffprobe --cpu-threads 1
./target/release/searchi search --db searchi.sqlite 'supplier cancellation terms' --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --cpu-threads 1
```
`embed` builds document, image, video and audio/music vectors by default. Use
`--text-only` explicitly to omit media. Default media indexing needs paired
FFmpeg/FFprobe paths or packaged helpers beside the installed CLI.
`searchi setup --model-dir PATH` is an alternative verified model-provisioning
command. It provisions model assets, not the native worker. Package the worker
separately and pass its path to headless embedding/search. Each short-lived CLI
@@ -114,5 +119,9 @@ installers, stage the target worker, retain dependency/model notices, verify
clean-machine launch and setup, run native Google smoke tests, and complete
upgrade/migration, signatures, and rollback checks. Optional offline bundles
need both verified model files. See [semantic bounds and evidence](SEMANTIC.md)
for current text-only, corpus, and performance limitations. Keep signing keys,
repository tokens, and model credentials outside source control.
for current corpus, localization, and performance limitations. Keep signing keys,
repository tokens, and model credentials outside source control.
FFmpeg provisioning supports manifest-pinned Windows/Linux x86-64 and ARM64 artifacts;
only Windows x86-64 has executed here. macOS decoder packaging remains a gate.
See [media runtime provenance](../tools/media-runtime/README.md) and
[measured media limitations](MEDIA-QUALITY.md).
docs/MEDIA-QUALITY.md
+99
View File
@@ -0,0 +1,99 @@
# Media search quality
## Evaluation contract
The independent v1 corpus contains **20 real assets and 65 frozen queries**: 12 photos, four videos, two music performances, and two sound effects. Queries include English/German visual memories, visually similar hard negatives, timestamp-sensitive actions, spoken topics, exact artist/title metadata, strict filter constraints, and no-answer cases.
Labels were frozen before media inference with fixture SHA256 `b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393`. Original asset hashes and licenses are recorded in [fixture.json](../fixtures/media/fixture.json) and [provenance](../fixtures/media/README.md). All Windows VM evaluation uses the real Google worker on **CPU, one thread**, with CPU FFmpeg decoding. No simulated embeddings or inferred model captions enter the corpus labels.
The reference environment is Windows 11 Pro in a shared VM exposing four logical CPUs on an AMD Ryzen 7 7800X3D host, approximately 9.93 GiB visible memory. Inference deliberately uses one thread and `SEARCHI_FORCE_CPU=1`. [Recorded environment and worker hash](../fixtures/media/environment.json). Observed worker RAM varied around 0.9–1.2 GB during indexing; these snapshots are not a measured peak-memory guarantee.
The corpus has one assistant annotator. Actual image/video frames were visually inspected; speech timestamps use independently published NASA captions. Human audio listening and blinded second-annotator review remain unexecuted. Existing artist/title tags are retained and checked at the stream level, where Vorbis stores them. English/German pairs refer to the same assets and are correlated, not twice as many independent examples. There is only one spoken-topic video and two music files; category-perfect recall can therefore be easy and must not be extrapolated.
## Results
The first **direct real-model dense reference** completed on CPU with 94/95 candidate encodings. This is an embedding/retrieval diagnostic, not yet the production pipeline. Candidate filtering in that reference is a benchmark constraint; exact metadata was deliberately marked unsupported.
| Category | Queries | File recall@1 | File recall@5 | MRR | Precise timestamp hit@1 |
|---|---:|---:|---:|---:|---:|
| Photo memory | 24 | 100% | 100% | 1.000 | — |
| Video actions | 4 | 100% | 100% | 1.000 | 100% |
| Video scenes | 6 | 100% | 100% | 1.000 | 100% |
| Flower temporal states | 4 | 100% | 100% | 1.000 | 75% |
| Spoken topics in video | 8 | 100% | 100% | 1.000 | **25%** |
| Music memories | 4 | **50%** | **50%** | 0.500 | — |
| Sound effects | 4 | 100% | 100% | 1.000 | — |
Across 55 supported positive queries (including one reference extension constraint), recall@1/@5 and MRR were **96.36% / 96.36% / 0.964**. Across 22 temporal queries, segment overlap@1 was **95.45%**, but precise point hits were **68.18%**. [Full recorded summary and failures](../fixtures/media/reference-summary.json), [rankings](../fixtures/media/reference-results.json).
Important failures:
- **Music encoding harness:** the original jazz violin Vorbis decode briefly overshot normalized PCM range. The direct tool sent unclamped floats, and the worker correctly rejected that one candidate. The two associated music misses are an explicit preprocessing failure, not evidence that the model confuses piano with violin. The production decoder clamps its PCM and must be evaluated separately; preserve this failed reference rather than hiding it.
- **Speech localization:** all eight spoken-topic queries found the right file and an overlapping thirty-second audio segment, but six returned a segment start outside the labelled utterance. Correct-file retrieval therefore does **not** fulfill precise “jump to the bit where somebody mentions a topic.” Shorter local windows or another alignment mechanism need independent evaluation.
- **Temporal visual distinction:** one red-flower query selected the right clip but the wrong stage of opening.
- **No-answer behavior:** all six supported absent/negative queries returned nearest neighbors (**100% unsuppressed return rate**). No arbitrary cosine threshold was invented. This exposes missing abstention evidence, not a calibrated confidence score.
- **Exact metadata:** four identity/no-answer cases remain unsupported in the direct embedding lane. A separate actual FFprobe → guarded Searchi core FTS run passed all three frozen artist/title queries at rank 1 and returned no result for the absent artist: **recall@1/@5 = 100%, MRR = 1.0; absent-artist return rate = 0%**. [Metadata results](../fixtures/media/metadata-summary.json). These are three small lexical identity cases, not proof that hybrid fallback abstains or that all music collections work.
Encoding took **498.66 seconds** for the photo/frame/audio candidates in this CPU-one-thread run. Query p50/p95 were **92.78 / 122.93 ms** after encoding. Runtime reported `google/embeddinggemma-2`, 768 dimensions, CPU, one thread, `b11461-no-logits-v1`. Other concurrent VM work can affect timings; this is not a clean-room hardware benchmark.
### Actual production pipeline: 30-second audio windows
The real shared Searchi service, FFmpeg 9.0.2 decoder, metadata extraction, guarded core index, ANN retrieval and hybrid ranking completed **20/20 files and 118 chunks with zero encoding failures**. This rerun used the fixed decoder's exact sample trimming; the earlier direct harness failure remains recorded separately. Indexing took **668.04 seconds**, and warm query p50/p95 were **92.23 / 113.38 ms**, on one CPU thread. These serial evaluation runs can still share the VM with unrelated work.
Across 58 supported positive queries, actual file recall@1/@5 was **96.55% / 100%**, with **MRR 0.983**. EN recall@1 was 96.77% (31 positives), DE 96.30% (27 positives). Photo memories 24/24, video file retrieval 22/22, sound effects 4/4, artist/title metadata 3/3 and the positive extension filter 1/1 passed at rank 1. Music memories passed **2/4 at rank 1 and 4/4 in the first five**: both violin queries ranked piano first despite successful decoding, an actual music hard-negative confusion.
Temporal point accuracy was **12/22 (54.55%)**; segment overlap was 18/22 (81.82%). Video action/scenery point hits passed 10/10, flower state hits 2/4, and spoken-topic hits **0/8**. Two spoken orange-soil queries surfaced image/video-frame hits instead of audio. Remaining speech starts ranged from 0.539 to 15.777 seconds outside the labelled utterance. The 0/8 strict score does not mean every start is equally unusable; raw distance and wrong-channel failures are retained rather than tuning a tolerance to this test corpus.
All six supported no-answer cases returned results, including the absent artist through semantic fallback. The separate lexical-only metadata absence test passed, but that does **not** establish abstention for the hybrid product. A requested temporal-range negative was explicitly unsupported, rather than silently counted as a pass. [Production summary](../fixtures/media/pipeline-30s-summary.json), [complete actual rankings](../fixtures/media/pipeline-30s-results.json).
### Same-label shorter audio experiment
The same fixed decoder, files, queries and two-second overlap were rerun with ten-second audio windows. All 20 files indexed successfully into **154 chunks**, with zero failures. This is a predeclared policy comparison, not a new production default.
| Metric | 30-second windows | 10-second windows |
|---|---:|---:|
| CPU indexing time | 668.04 s | 671.51 s |
| Actual chunks | 118 | 154 |
| File recall@1 / @5 | 96.55% / 100% | 96.55% / 100% |
| MRR | 0.983 | 0.983 |
| Speech precise point hits | 0/8 | 1/8 |
| Speech segment overlap | 6/8 | 6/8 |
| Overall temporal point hits | 12/22 | 13/22 |
| Worst correct-channel timestamp distance | 15.777 s | 3.777 s |
| Warm query p50 / p95 | 92.23 / 113.38 ms | 90.65 / 110.42 ms |
| Music top-1 | 2/4 | 2/4 |
| No-answer returned results | 6/6 | 6/6 |
Shorter windows improve the worst raw localization distance, but only one strict utterance hit and unchanged wrong-channel failures are insufficient for the original spoken-topic navigation goal. Similar measured cost here does not establish cost equivalence for long recordings; window count, media mix and VM contention differ. The actual hybrid artist/title regression still passes all three identity cases. [Ten-second summary](../fixtures/media/pipeline-10s-summary.json), [complete rankings and policy fingerprint](../fixtures/media/pipeline-10s-results.json).
### Predeclared photo-budget comparison
The same 12 photos and 24 frozen EN/DE memories were encoded at 70, 280 and 1120 tokens. Query embeddings were computed once and shared; no labels or thresholds changed.
| Image tokens | Actual CPU batch encoding time | Recall@1 / @5 | MRR |
|---|---:|---:|---:|
| 70 | 47.61 s | 100% / 100% | 1.000 |
| 280 | 199.65 s | 100% / 100% | 1.000 |
| 1120 | 1493.32 s | 100% / 100% | 1.000 |
There was no retrieval improvement on this small corpus despite a substantial high-budget cost. The high-budget run overlapped independent native compilation/profiling in the shared CPU VM, so the table is an observed experiment, not a controlled hardware scaling law. A worker RAM snapshot reached approximately 1.5 GB; peak RAM was not instrumented. Equal recall here does not establish equivalent retrieval for small text, dense screenshots or subtle visual distinctions. Keep the existing production policy pending a broader quality/cost corpus; these observations do not justify making 1120 tokens the default. [Budget summary](../fixtures/media/image-budget-summary.json), [actual dimensions, rankings and worker report](../fixtures/media/image-budget-results.json).
Production defaults have not changed as part of these experiments. The current **self-rating is 4/10: experimental media retrieval**. Real visual retrieval is promising, but precise speech timestamps, music hard negatives and abstention remain essential gaps. This rating follows the rubric below rather than extrapolating the small corpus's high file-recall score. The production measurements above precede a subsequent spoken-cue channel-routing improvement; that change has not been rescored against this corpus and must not inherit a better measured score without a new run.
The existing **20-document/24-query synthetic regression** was rerun with the actual Google CPU worker and unchanged fixture SHA256 `3bab37fc0ec9c2e2e40684962b049172b5078358c6487c9033ed8697b8128dd7`:
| Lane | Top-1 | Recall@3 | Recall@10 |
|---|---:|---:|---:|
| Filename/keyword | 20.83% | 20.83% | 20.83% |
| Semantic | 100% | 100% | 100% |
| Hybrid | 100% | 100% | 100% |
Measured hybrid p95 was **146.83 ms** on this run. [Recorded output](../fixtures/media/document-baseline.json). This remains a synthetic seed, not independent production-quality evidence or a large-corpus performance claim.
## Scoring and release gates
Report recall@1/@5, MRR, language/type breakdowns, segment-overlap hits, precise timestamp point hits, and failure categories. No-answer return rate is distinct from positive recall. Do not invent an arbitrary cosine rejection threshold from these results; calibrate on a separate development set, then validate on held-out negatives.
Self-rating rubric is declared before scores: **1–2/10** broken core retrieval; **3–4/10** experimental retrieval with missing essential capabilities or unvalidated abstention/metadata; **5–6/10** useful limited beta after independent basic retrieval/filter/metadata checks; **7–8/10** strong pilot with high recall and timestamp quality on representative corpora; **9–10/10** requires production release evidence across all operating systems, large representative libraries, calibrated no-answer behavior, and complete lifecycle/security/resource verification. A small diagnostic alone cannot justify a production score.
Remaining gates: broader production metadata/filter enforcement beyond the four positive seed constraints, calibrated no-answer handling, reliable spoken-topic alignment and channel selection, realistic long videos, audio listening review, unseen music/accents, corrupt codec cases, missing files/renames, hardware resource bounds, and native Windows/Linux/macOS validation. The measured production pipeline and explicit failures above establish a reproducible baseline; they do not close these gates.
docs/PERFORMANCE.md
+158
View File
@@ -0,0 +1,158 @@
# Search performance diagnostics
Search performance must distinguish total request latency from the work inside
it. The shared Rust schema exposes basic metrics for routine searches and a
bounded nested trace for deliberate debugging. It uses monotonic `Instant`
timing, not wall-clock timestamps. No query text, source passages or file paths
belong in stage names/counters.
## Shared response
`SearchResponse<T>` contains `results: Vec<T>` and `metrics: SearchMetrics`.
| Field | Meaning |
| --- | --- |
| total_ms | Elapsed time inside the collector's measured request boundary |
| result_count | Number of final returned results |
| trace | null in basic mode; debug spans in begin order |
| dropped_spans | Spans omitted when the 512-span ceiling or invalid parent is encountered |
A `SearchSpan` has a static `name`, relative `start_ms`, `duration_ms`, optional
parent index and integer `counters`. Parent indices reference earlier spans in
the same trace; child timings lie within their parent. Nested durations overlap,
so summing every span would double-count work. Counters are capped at 32 keys per
span, with 96-character labels. Missing spans due to the cap are visible through
`dropped_spans`, never presented as complete instrumentation.
Basic mode retains only the request start Instant and result count; there is no
per-stage span Vec or diagnostic string allocation. Debug mode captures up to
512 spans. Scope timings preserve normal closure return values and errors.
Already-ended or foreign tokens are harmless, and unfinished scopes close at
collector finish. Closing a parent closes its open descendants for containment.
## Instrumenting the shared engine
```rust
use searchi_core::metrics::{MetricsMode, TraceCollector};
let mut trace = TraceCollector::new(MetricsMode::Debug);
let results = trace.timed("search", None, |trace, parent| {
trace.timed("candidate_selection", parent, |trace, token| {
trace.counter(token, "candidate_count", candidates.len() as u64);
retrieve_candidates(candidates)
})
});
let metrics = trace.finish(results.len());
```
This is an API illustration, not a measured benchmark. `begin(name,parent)` /
`end(token,counters)` supports explicit scopes spanning locks or separate calls;
`SpanToken` is Copy and exposes its index. `counter` records bounded integer
counts without source data. `elapsed_ms` reports elapsed request time. Capture
queue/lock boundaries in the caller, then pass that same collector into the
shared core/service so stages refer to one origin.
Useful actual boundaries include request admission/reader lock, filter snapshot,
filename/FTS retrieval, tokenizer/query inference, active ANN load, eligibility
refresh, filtered HNSW traversal, per-file deduplication, ranking/fusion and
result assembly. Name only work that is truly measured. Do not label the whole
worker round trip as pure neural execution or fabricate stage splits from a
single elapsed number. Model load/warmup and initial eligibility work should be
reported separately from a retained worker's first foreground and warm queries.
## CLI and desktop integration
The CLI exposes these actual modes:
```sh
searchi search --db searchi.sqlite contract --json
searchi search --db searchi.sqlite contract --performance --json
searchi search --db searchi.sqlite contract --debug-performance --json
```
Plain `--json` preserves the legacy result array. `--performance --json` emits
`SearchResponse` with basic metrics; `--debug-performance --json` emits the
same envelope with detailed spans. Normal terminal output prints results and
reports elapsed time/count on stderr, including CLI index/session initialization.
Debug terminal mode also prints the detailed metrics on stderr. Legacy Vec
interfaces remain wrappers for compatibility. The desktop should use a compact ordinary summary and an optional
diagnostic view, without confusing score/similarity with elapsed time or confidence.
Both measured GUI commands start the native clock at command entry, include instrumented queue/lock waits, and finish before response serialization. UI timing separately captures input/debounce, lane/retry waits, invoke/round trip and synchronous DOM updates. Display latency ends at the DOM update; actual compositor paint and deferred media preview loading are not measured. Their overlapping boundaries must not be summed. Native metrics exclude frontend delays and post-return paint. A fast Rust request does not
prove a responsive first foreground interaction. Conversely, pending setup or
model loading must not be silently included in a warm-query claim.
## Reproducible performance validation
Record build profile, CPU/device and thread budget, VM/host resources, corpus and
active generation, filters, cold/warm state, concurrent indexing and setup costs.
Use multiple queries and p50/p95 distributions; retain raw reports. Compare basic
and debug mode on the same corpus to quantify instrumentation overhead. Debug
traces guide investigation but are not a substitute for representative benchmarks.
Current recorded filename, actual Google seed and synthetic ANN evidence is in
[VALIDATION](VALIDATION.md) and [SEMANTIC](SEMANTIC.md). Media quality/timing is in
[MEDIA-QUALITY](MEDIA-QUALITY.md). Synthetic vectors measure index behavior only;
real Google inference remains required for product semantic search. Platform,
large-corpus, concurrent-indexing and GPU performance gates remain independent.
## Worker round-trip diagnostics
Measured desktop failures return `{message, metrics}` and retain the trace for
the failed attempt. Busy retries are separate attempts. If an unexpected worker
task failure loses its collector, elapsed time is retained, the trace is null,
and a dropped-span marker makes the missing detail explicit. Error messages are
excluded from the exported timing report.
Embedding IPC spans expose named `*_us` integer counters for request encoding,
writing, worker wait/response decode, validation, and native tokenization, batch
preparation, inference and pooling/normalization when genuinely reported. They
also expose `worker_token_count` and the GPU-to-CPU restart count. Microsecond
counters are durations, not occurrence counts; their suffix records units.
The native worker clock is not aligned with Rust's collector origin. Native
stage values remain counters on the IPC span rather than invented positioned
child spans. Worker wait includes response-reader JSON parsing and scheduling;
it is not pure inference time. The encompassing round trip and its component
counters overlap. A fallback/restart may include setup outside the last worker's
reported neural pass; retain that distinction when diagnosing slow requests.
## Small instrumentation-overhead diagnostic
The deterministic example creates 1,000 local text files, builds/prewarms SQLite,
then alternates basic/debug order over 24 exact filename queries and 20 repeats.
Both modes must retrieve the correct file. It records 480 samples per mode,
outer-call p50/p95/mean and actual debug span counts, excluding fixture setup
and JSON serialization. It runs no model, media decoder or network.
```powershell
cargo run --release -p searchi-core --example trace_overhead -- .searchi/trace-overhead.json
```
This tiny warm lexical corpus measures diagnostic overhead under the current
VM conditions. It does not certify production search latency or model-query
tracing overhead. Keep the raw report and profile/device/concurrency caveats;
repeat on representative corpora and compare instrumentation policies before
claiming low overhead across every retrieval mode.
Windows CPU release diagnostic, 2026-10-12 (480 samples per mode):
| Mode | Mean | p50 | p95 |
| --- | --- | --- | --- |
| Basic | 0.556 ms | 0.440 ms | 0.934 ms |
| Detailed | 0.968 ms | 0.819 ms | 1.615 ms |
The detailed searches retained 14 spans each. These measurements include the
search itself and metrics assembly, and exclude fixture setup and JSON encoding.
Other work in the VM could contend for CPU. [Raw samples and measurement scope](evidence/windows-cpu-trace-overhead.json).
The recorded Windows x86-64 release run retained 480 raw samples per mode and
14 actual debug spans per request:
| Mode | Mean | p50 | p95 |
| --- | ---: | ---: | ---: |
| Basic | 0.556 ms | 0.440 ms | 0.934 ms |
| Debug | 0.968 ms | 0.819 ms | 1.615 ms |
[Raw samples and measurement contract](evidence/windows-cpu-trace-overhead.json)
include the active VM-contention caveat: native photo inference and other work
may overlap. This is an absolute small-corpus comparison, not an old-baseline
regression claim or proof of model/media tracing overhead. The extra cost
supports keeping detailed mode optional rather than claiming it is free.
docs/ROADMAP.md
+11
View File
@@ -38,10 +38,21 @@ Status: implementation started; no production release yet. Windows VM tests use
- [ ] Watchers and reconciliation across Windows, macOS and Linux.
## 4. Media
- [x] Genuine Google image, audio and video embeddings integrated with the shared
search service and CPU-tested FFmpeg decoding; music tags and duration filters.
- [x] Frozen independent media corpus with retrieval, timestamp, negative-query
and indexing-cost measurements (see MEDIA-QUALITY.md).
- [ ] Improve precise spoken-passage localization, music hard negatives and
calibrated handling of queries without a relevant match.
- [ ] Package and verify decoders on every release OS; test production acceleration.
- [ ] CPU OCR, image retrieval, audio transcription, video segment locations.
- [ ] Isolated extraction workers with timeouts and resource ceilings.
## 5. Release gates
- [x] Shared monotonic Basic/Debug search metrics, granular bounded engine traces,
actual worker query diagnostics, and headless JSON performance output.
- [ ] Verify detailed performance UI interaction on native desktop platforms;
repeat overhead measurements on representative libraries and hardware.
- [ ] Formatting, lint, tests and security/dependency review.
- [ ] CPU benchmarks: filename p95 <50 ms/1M records; text <150 ms/100K documents; semantic <500 ms on declared hardware.
- [ ] Measure cold start, indexing throughput, RAM, disk and idle CPU.
docs/SEMANTIC.md
+93 -5
View File
@@ -38,9 +38,7 @@ on other operating systems remain independent release checks.
| embeddinggemma-2-Q8_0.gguf | 309,855,456 | 2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135 |
| mmproj-embeddinggemma-2-Q8_0.gguf | 554,821,024 | c4a8a52691ecef40618438928bdf9e68379b854e24166f292592353db0aab64f |
Setup downloads both pinned files. **The current worker implements text**;
image/audio projection loading and image/audio/video embeddings remain required
work. Downloaded media weights do not establish functioning multimodal search.
Setup downloads both pinned files. The native worker implements genuine text, image pixels, and mono 16 kHz audio embeddings through the verified Google projector in the same 768-dimensional query space. The shared service now persists unified document, image, video-frame and soundtrack/audio/music vectors with source locations and metadata. Scoped desktop media previews and an independent retrieval evaluation are implemented; precise localization, broader quality, native playback QA and releases remain gates.
Weights use Apache-2.0; llama.cpp and DigUp use MIT; USearch and NumKong use
Apache-2.0. Retain notices with distributions. Five DigUp text reference vectors
@@ -64,14 +62,15 @@ HybridSearch is the shared CLI/desktop service:
HybridSearch::open(db, model_dir, worker_executable, threads)
HybridSearch::embedding_status(db) // No model/runtime load or network
service.warmup() // Real query inference + ANN cache
service.index_embeddings()
service.index_embeddings() // Explicit document-only generation
service.index_embeddings_with_media(&tools, &MediaLimits::default()) // Unified generation
service.search(query, limit, &SearchOptions)
service.search_semantic(query, limit, &SearchOptions) // Diagnostics
service.worker_info() // Actual model/runtime/device/threads
~~~
Threads are bounded to 1–8; VM tests use one. Document batches contain at most
eight windows. Request/response budgets are 2/4 MiB, startup has a 120-second
eight windows. Request/response budgets are 16/4 MiB, startup has a 120-second
deadline, and each request has a 180-second deadline. Malformed or interrupted
IPC and invalid embeddings fail explicitly.
@@ -177,3 +176,92 @@ The ignored ANN test deliberately requires release mode. Default tests require
no model/network and cover protocol bounds, invalid vectors, UTF-8 coverage,
fusion, lifecycle, checksums, pre-top-k filtering, and dirty-path caching.
Linux/macOS and GPU checks are not inferred from this Windows CPU run.
## Genuine image and audio adapter
`GemmaEmbedder::embed_images(&[&Path])` uses a provisional 280-token image
budget. `embed_images_with_budget(paths, budget)` supports 70 through 1120
Google visual tokens, including full-resolution model input. Use
`embed_images_with_metadata` for encoded dimensions and the chosen budget.
The aspect-preserving dimensions are multiples of 48; Rust performs bounded
PNG/JPEG/WebP/BMP decoding, applies EXIF orientation, and bicubic Catmull-Rom
resizing before sending RGB pixels. No captions or filenames produce vectors.
Bounds are eight paths per call, one image per inference operation, 64 MiB
input files, 64 Mi-pixel JPEG sources (scaled decoding above 16 Mi pixels),
16 Mi-pixel non-JPEG sources, and a 256 MiB decoding-buffer/output ceiling. Decoder dimensions
and output size are checked before pixel allocation. Tiny/extreme images still
receive a valid bounded grid. The same API embeds decoded video frame files.
`GemmaEmbedder::embed_audio_pcm(samples)` accepts normalized finite mono 16 kHz
PCM with 1–480,000 samples, at most 30 seconds, on a little-endian host. It
encodes actual sound through Google MTMD and the shared mean-pooled text model.
Callers must segment the entire recording/movie soundtrack and retain actual
window timestamps; this API never silently truncates audio. It provides no
transcript. Speech, music, and environmental sounds require independent quality
evaluation even though all use this genuine encoder.
The projector is size/SHA256 verified on first media use, then loaded lazily in
the retained native worker. Model initialization and returned inference failures
share the CPU fallback policy. Native video/FFmpeg spawning is disabled; the
media adapter owns extraction. Media generations must include
`GEMMA_MEDIA_FINGERPRINT` and image budget/window policy in their identity.
Text fingerprints remain unchanged. GPU media inference is untested here.
Actual Windows CPU parity with copied MIT DigUp fixtures passed for screenshot
280, photo 140, frame 70, and speech: cosine respectively **0.99994194,
0.99974583, 0.99921928, and 0.99983318**, all exceeding the 0.999 gate. The test
completed in 58.79 seconds including fresh model/projector startup. This proves
runtime parity on those inputs, not broad image/audio retrieval quality.
Run with the model/worker environment above:
~~~powershell
cargo test -p searchi-semantic --test gemma_media_cpu -- --ignored --nocapture
~~~
The large camera JPEG policy, established decoder tradeoffs, EXIF/resource test,
and memory-bound caveats are documented in [the semantic crate](../crates/searchi-semantic/README.md).
## Unified media indexing and setup
The desktop and default CLI `embed` command use `index_embeddings_with_media`.
Document-only indexing is an explicit `--text-only` choice. `searchi-media`
prepares actual image files, movie frames and mono 16 kHz audio windows with
verified caller-supplied FFmpeg/FFprobe executables. There is no transcription,
fabricated caption or model-derived filename substitution. Existing stream/file
artist/title metadata enters guarded keyword indexing.
Default media policy uses image 280/frame 70 visual tokens, 30-second primary
audio windows with two-second overlap, up to 240 distributed frames plus 120
scene candidates, up to 600 audio windows, a four-hour/16 GiB file ceiling and
45-second subprocess deadlines. Coverage includes the audio tail or reports an
explicit resource error; video sampling spans the movie without guaranteeing
every transient scene. Media files have stable content identity, persisted
vectors and sources (`image`, `timestamp_ms`); source changes/deletions suppress
stale results. Generations include preprocessing/window/sampling fingerprints.
Per-file decoding failures are reported and persisted rather than silently
counted as successful media indexing. Other filename/text matches stay useful.
Developer preparation from the repository root:
~~~powershell
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
python tools/embeddinggemma2/stage_runtime.py --backend cpu
python tools/media-runtime/provision.py --stage
$env:SEARCHI_FORCE_CPU = '1'
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
~~~
The decoder script verifies a pinned FFmpeg 9.0.2 LGPL artifact and stages tools,
notices and the manifest without global installation. Only Windows x86-64 has
been executed; Linux/ARM64 entries require native checks, and macOS decoder
packaging is still missing. End-user packages must include the worker/decoders;
Google weights are provisioned automatically and verified locally. Search does
not perform setup downloads. Headless CLI helper discovery and explicit paths
are documented in [README](../README.md) and [BUILDING](BUILDING.md).
[Independent media evaluation](MEDIA-QUALITY.md) preserves the frozen corpus,
actual production pipeline, direct-harness failure, music confusions, speech
point-localization failures and unsuppressed no-answer results. Strong file
recall is insufficient for accurate spoken-topic navigation. A ten-second audio
experiment reduces some distances but does not close that gate; it is not a
new default. Quality policies must be validated on held-out data rather than
inventing thresholds from the evaluation corpus.
docs/VALIDATION.md
+84 -54
View File
@@ -1,64 +1,94 @@
# Validation evidence
Development host: Windows VM, x86_64 MSVC, Rust stable 1.99.0.
All local tests run on CPU; no GPU provider is configured.
Development host: Windows VM, x86_64 MSVC, stable Rust 1.99.0. All local
inference, FFmpeg decoding and WebView tests use CPU only. UI launches set
`WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS='--disable-gpu --disable-accelerated-video-decode'`
and `SEARCHI_FORCE_CPU=1`.
## Current integration checkpoint (2026-10-12)
- `cargo test --workspace --locked`: 41 default tests passed after final core integration. Four asset-dependent
or explicit benchmark tests remain opt-in; a default pass does not run them.
- Native desktop tests: 7 passed with SEARCHI_FORCE_CPU=1 and WebView2 GPU disabled.
- Frontend: 7 async-state tests passed, Prettier passed, TypeScript/Vite build passed.
- Genuine Google CPU reference/window test passed separately: five DigUp reference
vectors had cosine similarity above 0.999; actual multilingual retrieval and
tokenizer tail-window coverage passed. MiniLM evidence below is historical.
- Repository description/topics were read back after update; the branded avatar
upload returned HTTP 204 and the repository API returned its avatar URL.
- New desktop styling is in progress. `design-qa.md` records the visual gate as
blocked because browser bootstrap and native screen capture are unavailable.
No visual match or complete UI interaction verification is claimed.
- Native desktop: **9/9 tests passed**, including structured trace/basic responses.
Post-generation cached query warmup is implemented but its interactive first-query gate remains open.
Frontend: **18/18 tests passed**, TypeScript/Vite production build and
formatting passed. Preview policy tests cover fractional timestamp seeking,
image labels, one active/latest pending request, stale replies and Escape.
- Semantic adapter: **8 default tests passed**; the explicit 48 MP resource test
remains opt-in. Strict targeted all-target Clippy and formatting passed.
Protocol bounds, invalid vectors, artifact size/checksum mismatch, old-worker
capability defaults and oversized JPEG rejection are covered.
- The CPU native helper rebuilt and staged successfully with genuine Google
MTMD image/audio support. Its handshake reports `media_supported:true`;
older text workers default false rather than claiming media readiness.
- Actual Google CPU media parity passed separately in **52.85 seconds**:
screenshot 280 cosine 0.99994194, photo 140: 0.99974583, frame 70: 0.99921928 and
speech: 0.99983318 against pinned MIT DigUp vectors. All exceed the 0.999 gate.
This verifies runtime parity, not broad retrieval quality.
- The release resource test encodes/decodes an actual 8000x6000 JPEG with EXIF
rotation and verifies region colors after scaled decoding/resizing. Decode
plus resize took 176–220 ms. A sampled whole-test peak was 146.86 MiB, including
source fixture generation; it is not decoder-only RSS or neural quality.
- The independent production media pipeline has run on 20 real assets/65 frozen
queries using actual Google inference, FFmpeg, metadata, persistence and HNSW.
It encoded 20/20 files with no failures in the recorded 30-second-window run.
Broad file recall is promising; precise speech timestamps, music confusion
and no-answer behavior remain failed quality gates. Preserve the earlier
direct harness failure separately. [Raw results and limitations](MEDIA-QUALITY.md).
- Actual document seed regression passed 24/24 semantic/hybrid queries. This is
a small synthetic regression, not a production relevance corpus.
- The explicit 100,000-vector release ANN diagnostic passed 98.5% recall@10,
HNSW p95 1.37 ms and warm service p95 6.40 ms with zero warm passage hashes.
Vectors are synthetic index benchmark data, never production embeddings.
- The fixed one-million-file first-input report records 40 inputs below 50 ms in
that run: [recorded benchmark](../crates/searchi-core/benchmarks/results/windows-cpu-full-candidate-first-final.json).
These measurements do not establish every query or concurrent-indexing goal.
Final UI integration: the branded compact launcher, local Inter/Fluent assets,
themes and motion implementation passed frontend build, formatting, and seven
state tests. Native launch succeeded; accessibility text showed real controls
and a warmed Google CPU worker. UI input and screenshots remain blocked by
Windows access/capture errors, as recorded in design-qa.md. Workspace formatting
and strict all-target Clippy passed. The fixed full-corpus first-input report
is in crates/searchi-core/benchmarks/results/windows-cpu-full-candidate-first-final.json;
all 40 measured filename inputs were below 50 ms in that run.
The integrated CPU workspace checkpoint passed **62 default tests** (8 assets, 6 CLI, 23 core, 4 media, 8 semantic, 13 service), with **10 explicit opt-in tests ignored**. Workspace and native all-target Clippy passed with warnings denied. The later semantic CPU-restart counter change passed workspace Clippy and the actual CPU query diagnostics test. Earlier aggregate counts are historical.
The active parallel development branches of work may add tests or change counts
after this checkpoint. Run the documented checks again after integration.
The updated pinned C++ helper compiled in an independent CPU build directory and
was staged for desktop resources without interrupting media evaluation. Actual
Google query diagnostics passed separately in 18.09 seconds including model
initialization and two queries: 13 tokens, tokenization 149 us, batch preparation
22 us, inference 112914 us, pooling/normalization 17 us. Rust worker wait/response
decode measured 113960 us. Debug/normal embedding cosine was 0.99999999. These
are one diagnostic sample under VM contention, not production latency evidence.
See [performance boundaries and raw overhead samples](PERFORMANCE.md).
## Historical foundation evidence
## Native UI and platform limits
Initial foundation checks (2026-10-12):
- `cargo test --workspace`: CLI parser/executable checks and engine retrieval,
lifecycle, exclusions, filters, migration and concurrent-reader checks passed.
- `cargo clippy --workspace --all-targets -- -D warnings`: passed.
- Rust formatting applied; final formatting verification follows integration.
- Desktop TypeScript/Vite build passed; npm audit reported zero vulnerabilities.
- Initial Windows desktop native check, embedded app build, strict Clippy and
two desktop tests passed (index gate recovery and guarded source opening).
- Desktop launched with WebView2 `--disable-gpu`. UI automation could not capture
the window: `GetCursorPos` access denied, then monitor capture error
`0x80070057`. Visual layout and end-to-end UI interactions remain unverified.
- Real Python CPU ONNX embedding proof completed on 8 documents / 6 queries:
5/6 top-1 accuracy, 6/6 recall@3. The strict top-1 quality gate is not met.
See `tools/semantic-proof/windows-cpu-result.json` for raw timings and rankings.
This tiny fixture is not evidence of large-corpus performance.
- Native Rust CPU embedding adapter compiled, passed its three unit tests and
the explicit local-model smoke test. The smoke reproduced the same 5/6 top-1
limitation. Default CI omits this asset-dependent test; it must be executed
separately with provisioned local artifacts.
- Integrated workspace validation: 19 default tests passed; 1 local-model test
is opt-in. Rust formatting and strict Clippy passed after lockfile refresh.
- PDF/DOCX fixture tests include matching source locations, corrupt replacement,
encrypted PDF behavior and compressed-input limits.
The branded launcher launched natively. Accessibility inspection showed real
controls and a warmed Google CPU worker. Native screenshots, input automation
and complete image/audio/video playback interactions remain blocked by Windows
access/capture errors recorded in [design QA](../design-qa.md). Passing pure
preview policy tests does not establish real WebView codec playback or visual
fidelity. Default-application opening is not exact timestamp navigation.
Not yet established: production latency targets, semantic inference, desktop
runtime behavior, Linux/macOS execution, installer signing or upgrades.
CI configuration is source configuration, not evidence that remote jobs ran.
Gitea workflows require matching Windows, Linux and macOS runners; runner
availability has not been established. GitHub workflows are also supplied for
future mirroring, but do not run merely because code was pushed to Gitea.
Windows x86-64 verified decoder provisioning/staging has executed. Manifest
entries for Linux and ARM64 are configuration, not evidence of native execution.
A vetted macOS decoder package remains missing. Linux/macOS worker/desktop
execution, production GPU parity/fallback, installer signing, upgrade/rollback,
licenses/source distribution, large libraries and calibrated absence handling
remain unexecuted release gates. FFmpeg subprocess memory is not sandbox-capped;
bounded outputs/timeouts are not an absolute RSS guarantee.
Gitea needs matching registered runners. GitHub workflows are supplied for
mirroring; they do not run merely because code is pushed to Gitea. The current
workflow checks do not automatically certify downloaded Google assets or the
separately built worker. CI configuration is not remote job evidence.
## Historical evidence
Initial engine/CLI checks covered retrieval, persistence, filesystem changes,
exclusions/filters, migration, concurrent readers, PDF pages, DOCX paragraphs,
corrupt replacement, encrypted PDF errors and bounded compressed input.
The first actual Google text adapter matched five pinned reference vectors
above 0.999, including multilingual queries and long-document window coverage.
The rejected MiniLM/ONNX experiment achieved 5/6 top-1 on its six-query fixture;
it is retained under the opt-in historical proof only and is not a production
fallback. [Historical experiment](../tools/semantic-proof/HISTORICAL.md).
Repository metadata/avatar updates were read back successfully. Historical
foundation/default test totals changed as implementation expanded and must not
be presented as the current aggregate.
Reproduction commands and exact interfaces are in [BUILDING.md](BUILDING.md),
[SEMANTIC.md](SEMANTIC.md), and [MEDIA-QUALITY.md](MEDIA-QUALITY.md).
docs/evidence/windows-cpu-google-query-diagnostics.json
+22
View File
@@ -0,0 +1,22 @@
{
"date": "2026-10-12",
"model": "google/embeddinggemma-2",
"runtime": "llama.cpp b11461 with pinned no-logits patch",
"device": "cpu",
"threads": 1,
"test": "google_cpu_debug_query_reports_actual_worker_stages_without_changing_vectors",
"test_elapsed_seconds_including_initialization_and_two_queries": 18.09,
"debug_normal_cosine": 0.99999999,
"diagnostics": {
"request_encode_us": 24,
"request_write_us": 52,
"response_validate_us": 92,
"worker_batch_prepare_us": 22,
"worker_inference_us": 112914,
"worker_pool_normalize_us": 17,
"worker_token_count": 13,
"worker_tokenization_us": 149,
"worker_wait_and_response_decode_us": 113960
},
"scope": "One actual query diagnostic; native worker clocks are not aligned to Rust. VM had concurrent media inference. No production latency or broad overhead claim."
}
docs/evidence/windows-cpu-trace-overhead.json
+990
View File
@@ -0,0 +1,990 @@
{
"arch": "x86_64",
"basic": {
"mean_ms": 0.5563772916666668,
"p50_ms": 0.4403,
"p95_ms": 0.9344,
"samples": 480
},
"basic_samples_ms": [
0.434,
0.4441,
0.5638000000000001,
0.41350000000000003,
0.4078,
0.44470000000000004,
0.4078,
0.5291,
0.416,
0.4223,
0.43360000000000004,
0.4181,
0.5680999999999999,
0.44630000000000003,
0.4105,
0.3995,
0.43829999999999997,
0.4223,
0.4192,
0.4153,
0.5199999999999999,
0.4115,
0.482,
0.5616,
0.417,
0.4108,
0.412,
0.3987,
0.4224,
0.5749000000000001,
0.39780000000000004,
0.40240000000000004,
0.4188,
0.39809999999999995,
0.3923,
0.532,
0.44880000000000003,
0.5005,
0.4576,
0.4341,
0.5256,
0.4138,
0.5062000000000001,
0.4152,
0.3983,
0.5354,
0.4163,
0.5561999999999999,
0.7023,
2.0184,
0.483,
0.5897,
0.5286,
0.4496,
0.41750000000000004,
2.5051,
0.4249,
0.8365999999999999,
0.5505,
0.4127,
0.5488999999999999,
0.47009999999999996,
0.6473,
0.4203,
0.6688,
0.4081,
0.46749999999999997,
0.4396,
0.4638,
0.4878,
0.5753,
0.8668,
0.4817,
0.41829999999999995,
0.4106,
0.482,
0.4837,
0.3935,
0.5321,
0.42860000000000004,
0.41859999999999997,
0.3914,
0.3881,
0.522,
0.4021,
0.3938,
0.3918,
0.38739999999999997,
0.5252,
0.39740000000000003,
0.3877,
0.421,
0.40850000000000003,
0.5257000000000001,
0.40299999999999997,
0.4779,
1.9503000000000001,
0.416,
0.4799,
0.5547,
0.41209999999999997,
0.4281,
0.6578,
0.4112,
0.704,
0.5748,
0.4073,
3.9541999999999997,
0.5597,
0.6794,
0.46690000000000004,
0.4059,
0.5046999999999999,
0.4214,
0.5058,
0.4331,
0.4039,
0.41100000000000003,
2.5677,
0.4401,
0.4058,
0.4004,
0.4566,
0.4249,
0.5615,
0.3956,
0.6221,
0.53,
0.4347,
0.5176999999999999,
0.3959,
0.4193,
0.627,
0.42719999999999997,
1.1775,
0.6066999999999999,
0.5718,
0.45,
0.4439,
0.5364,
0.5308,
0.4416,
0.5453,
0.4424,
0.5058,
0.4075,
0.46099999999999997,
0.5885,
0.5032,
1.3806999999999998,
0.4668,
0.5145,
0.40959999999999996,
0.5816,
0.4249,
0.3986,
0.6983999999999999,
2.6059,
0.4276,
0.5638000000000001,
0.5913,
0.4168,
0.4255,
1.4086999999999998,
1.5349000000000002,
0.4954,
0.5483,
0.41350000000000003,
0.42450000000000004,
1.1743999999999999,
0.6155,
0.48319999999999996,
0.6431,
0.4142,
0.43370000000000003,
0.5195,
0.3939,
0.41609999999999997,
0.41619999999999996,
0.38949999999999996,
1.9449999999999998,
0.5754,
0.4729,
0.5639,
0.40719999999999995,
1.8393000000000002,
0.46490000000000004,
0.5271,
0.4041,
0.5678000000000001,
0.5701,
0.4301,
0.9139,
0.4016,
0.4486,
0.41659999999999997,
0.6462,
0.7081999999999999,
0.6969,
0.6972,
0.5947,
0.4736,
0.6512,
0.4312,
0.4189,
0.5238,
0.39130000000000004,
0.4522,
0.4057,
0.5138,
0.8949,
0.9114,
0.5758,
0.5318999999999999,
0.4661,
0.4411,
1.6324,
0.4342,
2.5791999999999997,
0.4112,
0.4033,
3.1165,
0.424,
0.5136000000000001,
0.40549999999999997,
0.631,
0.4153,
2.9606,
0.4458,
0.4709,
0.4316,
0.4279,
0.45790000000000003,
0.4303,
0.397,
1.674,
1.2691,
0.7015,
0.5775,
0.5308,
0.4457,
0.7706,
0.7252,
0.49720000000000003,
0.5576,
0.5545,
0.4265,
0.5495,
0.40690000000000004,
0.4043,
0.3976,
0.3864,
0.3887,
0.4118,
0.7454000000000001,
0.7821,
0.7821,
0.627,
0.623,
0.41590000000000005,
0.4231,
0.45130000000000003,
0.4584,
0.5218,
0.4885,
0.38880000000000003,
0.6336,
0.4138,
0.3964,
0.5286,
0.4095,
0.40440000000000004,
0.7086,
0.5809,
0.6849,
0.4264,
0.5157,
0.4281,
0.4668,
0.6745,
0.7035,
0.7287,
0.6853,
0.415,
0.5748,
0.42319999999999997,
0.3932,
0.4022,
0.5082,
0.39599999999999996,
0.42700000000000005,
0.4045,
0.3875,
0.5247999999999999,
0.4077,
0.4099,
0.4001,
0.3945,
1.2222,
0.423,
0.4063,
0.4059,
0.40890000000000004,
0.528,
0.4066,
0.402,
0.39530000000000004,
0.3986,
0.3931,
0.48069999999999996,
0.4489,
0.4098,
0.40059999999999996,
0.5538000000000001,
0.4022,
0.4113,
0.6244,
0.40440000000000004,
0.4828,
0.40349999999999997,
0.4216,
0.41800000000000004,
0.3955,
0.3921,
0.5814,
0.4569,
0.4053,
0.47869999999999996,
0.4563,
0.5081,
0.5031,
0.44270000000000004,
0.6819999999999999,
0.44670000000000004,
0.42719999999999997,
0.5216,
0.41619999999999996,
0.4459,
0.4079,
0.5176000000000001,
0.4091,
0.4403,
0.4271,
0.399,
0.5476,
0.4369,
1.5406,
0.4229,
0.5256,
0.4348,
0.40190000000000003,
0.491,
0.6232,
0.4165,
0.5216,
0.3937,
0.41619999999999996,
0.4152,
0.40640000000000004,
0.5112,
0.381,
0.38230000000000003,
0.5358,
0.4103,
0.3923,
0.3892,
0.4074,
0.5296,
0.40900000000000003,
0.419,
0.3923,
0.4018,
0.518,
0.3943,
0.443,
0.3878,
0.3998,
0.39120000000000005,
0.4237,
0.5232,
0.3859,
0.4138,
0.4401,
0.4011,
0.39530000000000004,
0.4001,
0.3907,
0.4242,
0.5098,
0.3998,
0.5058,
0.4314,
0.388,
0.3998,
0.3986,
0.5114,
0.3856,
0.39490000000000003,
0.4326,
0.4249,
0.5303,
0.3922,
0.3905,
0.4299,
0.3973,
0.47409999999999997,
0.38939999999999997,
0.5234000000000001,
0.4118,
0.4851,
0.4848,
0.4082,
0.4038,
0.48410000000000003,
0.4568,
0.4204,
0.7379,
0.7857,
0.7318,
0.9344,
0.9292,
0.883,
0.8180000000000001,
0.7191,
1.4266999999999999,
0.43629999999999997,
0.5381,
0.4556,
0.6560999999999999,
0.5197999999999999,
0.4394,
0.46309999999999996,
0.5307000000000001,
0.42319999999999997,
0.4075,
0.3964,
0.5482,
0.3926,
0.388,
0.4299,
0.44499999999999995,
0.7485999999999999,
0.44470000000000004,
0.40759999999999996,
0.5051,
0.3889,
0.5062000000000001,
0.40700000000000003,
0.4002,
0.3934,
0.3889,
0.5202,
0.40049999999999997,
0.40650000000000003,
0.3976,
0.4211,
0.4279,
0.5324,
0.40059999999999996,
1.4782,
0.6034999999999999,
0.43679999999999997,
0.4105,
0.408,
0.7263000000000001,
0.4593,
0.6996,
0.4271,
0.5413,
0.3998,
0.4329,
0.7967,
0.9076000000000001,
0.8878,
0.4333,
0.431,
0.8344999999999999,
1.1369,
0.42410000000000003,
0.4025
],
"debug": {
"mean_ms": 0.967733333333333,
"p50_ms": 0.8194,
"p95_ms": 1.6146,
"samples": 480
},
"debug_samples_ms": [
0.8167,
0.7293,
0.9592,
0.8642,
0.7673,
0.8222,
0.7207,
0.7231,
0.8286,
0.7298,
0.8266,
0.7201,
0.7448,
0.8205,
0.7162,
1.0768,
0.8357,
0.7537,
0.8582000000000001,
0.7343999999999999,
0.7179000000000001,
4.9764,
0.8008,
0.8557,
0.7527,
0.9688,
0.8746999999999999,
0.727,
0.8711,
0.7105,
0.7215,
0.8220000000000001,
0.7176,
0.8272,
0.7191,
0.9924999999999999,
0.9016000000000001,
0.7423,
1.2084000000000001,
0.8556,
0.7384,
0.7409,
0.7081999999999999,
0.8427,
0.8474,
0.7284,
0.7603,
3.1463,
1.6184999999999998,
1.1203999999999998,
0.8165,
1.4208,
0.924,
0.85,
0.7396,
0.8244,
0.9145,
0.7588,
0.812,
1.0149,
0.7919,
0.9867999999999999,
0.8725999999999999,
0.7468,
0.8335,
1.1976,
0.9091,
0.7374999999999999,
0.8851,
1.9234,
1.1454,
1.2746000000000002,
1.474,
0.7353000000000001,
0.8841,
0.9075,
0.7607999999999999,
0.8289000000000001,
0.8323999999999999,
0.8009000000000001,
0.8202,
0.7681,
0.8252999999999999,
0.8329,
0.7232999999999999,
0.8313,
0.7251,
0.8301,
0.7288,
0.7092,
0.8169,
0.7212,
0.882,
0.7234,
0.7201,
3.2547,
0.7303000000000001,
0.9255,
0.755,
0.7154,
0.9,
0.8394,
0.7272,
0.9393,
0.8936000000000001,
0.733,
6.3655,
2.8007,
0.8151999999999999,
0.7929999999999999,
0.8479,
1.1844,
0.8132,
0.9790000000000001,
0.8571,
0.7755000000000001,
0.8656,
0.73,
0.7969999999999999,
0.8276,
0.7308,
0.883,
0.7273,
0.8855,
0.7218,
0.7303999999999999,
0.8796999999999999,
0.9757,
2.321,
0.7583,
0.7388,
0.8352,
0.7089000000000001,
1.1594000000000002,
0.9188999999999999,
1.6146,
7.5508,
0.8053,
1.2301,
0.7657,
1.4819,
1.0386,
0.7665000000000001,
0.7372000000000001,
0.7472,
0.9138,
0.782,
0.8009999999999999,
0.9136,
0.9881,
0.846,
0.7302,
0.7045,
0.7411,
0.8925,
0.7107,
3.5707,
0.7821,
0.8599,
0.8071,
0.7342,
0.8309,
0.7102999999999999,
2.6215,
0.8529,
0.7293,
0.7728,
0.8239000000000001,
1.3806,
3.0549,
1.4774,
1.4057,
1.4697,
0.8741000000000001,
1.2217,
0.8339000000000001,
0.7242000000000001,
0.8094,
0.7016,
0.8065,
0.7062999999999999,
1.1670999999999998,
0.7271,
1.4295,
0.7381,
2.2805,
1.673,
0.7472,
0.7219,
3.197,
0.7538999999999999,
0.8595999999999999,
0.7359,
0.8168,
0.7067,
0.7971,
2.1982999999999997,
1.2154,
1.27,
3.3795,
1.1157000000000001,
0.7726999999999999,
0.8442000000000001,
0.8882,
0.7017,
0.7211,
1.2641,
0.8506,
0.723,
3.8099000000000003,
1.1316,
1.0410000000000001,
0.8524999999999999,
1.0685,
0.7464999999999999,
0.8846999999999999,
1.0812,
1.107,
1.093,
1.1158,
0.7374,
1.5673,
1.0413,
0.8394999999999999,
0.7274,
0.8664000000000001,
1.0905,
0.7726000000000001,
1.5097,
0.9828,
0.9055,
0.7475,
0.9872,
0.9333,
0.8344999999999999,
0.7199,
1.4668,
2.1616999999999997,
1.0192999999999999,
0.7895000000000001,
1.8851,
1.3231,
1.2965,
0.7644000000000001,
0.7393000000000001,
0.9441,
0.7245,
0.7081000000000001,
0.8285,
0.7283,
0.8108,
0.7171000000000001,
0.7234999999999999,
0.7841,
1.0449000000000002,
1.0277,
1.0941,
1.2095,
1.158,
0.8891,
0.7197,
0.882,
0.7267,
0.7133,
0.7043,
0.727,
0.7603,
0.8140000000000001,
0.8274,
0.7349,
0.723,
0.8446,
0.9939,
0.9430999999999999,
1.2679,
0.8774,
0.7139,
0.7448999999999999,
0.7282000000000001,
0.7496,
1.2924,
1.2619,
1.2734999999999999,
0.8732,
0.9319000000000001,
0.7432,
0.842,
0.7283,
0.7115,
0.8179,
0.7108,
0.8030999999999999,
0.7085,
0.7425,
0.9125,
0.7362,
0.8239000000000001,
0.7027,
0.7646000000000001,
0.841,
0.727,
0.8307,
0.7421,
0.7456,
0.8228000000000001,
0.7144,
0.8956,
0.7155,
0.8194,
1.1168,
0.8448,
0.7318,
0.8819,
0.7425,
0.7328,
0.8884,
0.7295999999999999,
0.7747,
0.8804000000000001,
0.7343999999999999,
1.1522,
0.9186,
0.7333,
0.8239000000000001,
0.8241999999999999,
0.759,
0.861,
0.7278,
0.8876,
0.7179000000000001,
0.7827,
0.8582000000000001,
0.7512000000000001,
1.0376,
0.8305,
0.7426,
0.7012999999999999,
0.8982,
0.7402,
0.7324,
0.8277,
0.7297,
0.8775000000000001,
0.7464999999999999,
0.7191,
1.0924,
0.8524999999999999,
0.7319,
0.7194,
0.8802,
0.7388,
1.2646,
0.8413,
0.7404999999999999,
0.7242000000000001,
0.8245,
0.725,
0.8404,
0.7255,
0.7195,
0.7114,
0.8140000000000001,
0.7406,
0.7184,
0.8338,
0.7412,
0.8614,
0.7208,
0.7693,
0.8305,
0.7565000000000001,
0.8252,
0.7134,
0.7383,
0.8237,
0.7262,
0.8196,
0.7092,
0.8099,
0.7102999999999999,
0.7117,
0.837,
0.7314,
0.8402000000000001,
0.8168,
0.7099,
0.8225,
0.7115,
0.713,
0.7081999999999999,
0.7036,
0.8197,
0.7163,
0.8418,
0.7012,
0.7218,
0.8236,
0.7665000000000001,
0.8529,
0.759,
0.7396,
0.8301,
0.7272,
0.828,
0.7037,
0.7199,
0.7303999999999999,
1.0776,
0.8279,
0.8061,
0.7394,
0.7934,
0.805,
0.8004,
0.9452,
0.791,
1.0659,
1.2731000000000001,
1.2185,
1.1789,
1.4793,
1.3232000000000002,
1.3123,
1.1511,
4.9007,
1.9954999999999998,
0.7574,
0.7156,
1.063,
0.7659,
0.7311000000000001,
0.8957,
0.9449000000000001,
0.7726999999999999,
0.7187,
0.7123,
0.7200000000000001,
0.8279,
0.7073,
0.7967,
0.785,
0.7429,
0.8313,
0.7189,
0.7351000000000001,
0.7239,
0.7041000000000001,
0.8423,
0.7414999999999999,
0.8124,
0.7135,
0.7106,
0.8559,
0.7453,
0.9529,
0.7197,
0.8180000000000001,
0.7674,
0.7205,
0.8537,
1.0651,
0.862,
0.7998000000000001,
0.8413,
0.7829,
0.7494999999999999,
0.9843000000000001,
1.0847,
0.7273,
0.7253999999999999,
0.8928,
0.722,
1.1499,
1.1878,
0.8394,
0.8515,
1.642,
1.8285,
0.7554000000000001,
0.8442999999999999
],
"debug_span_count_max": 14,
"debug_span_count_min": 14,
"files": 1000,
"kind": "prepared_sqlite_trace_overhead",
"limits": "tiny warm lexical corpus; harness serial/no model/media/GPU; other VM work including native photo inference may contend; not isolated production performance",
"measurement": "outer Rust call including metrics assembly; excludes fixture setup/JSON serialization",
"os": "windows",
"production_claim": false,
"profile": "release",
"queries": 24,
"repetitions": 20
}
fixtures/media/.gitignore
+3
View File
@@ -0,0 +1,3 @@
derived/
inspection/
*.sqlite*
fixtures/media/README.md
+22
View File
@@ -0,0 +1,22 @@
# Independently labelled media diagnostic v1
`fixture.json` and `labels.lock.json` were frozen before media inference. The fixture contains 20 original public/licensed files and 65 English/German queries. Neutral filenames prevent filename semantics from giving away visual answers. Do not edit v1 labels after seeing rankings; corrections require a new version and another pre-scoring lock.
The assistant inspected every photo and sampled real video frames through `view_image`. Cat-action/flower-state intervals came from these frames. Spoken-topic intervals came from NASA's publisher-provided SRT, not a model transcript. Music/sound labels came from independent publisher descriptions and original embedded tags; human listening verification is **not** claimed. One annotator authored these labels, so a blinded second human review remains necessary.
## Provenance and redistribution
Every asset has its exact source URL, description page, license, byte count, SHA256, and modification record in the fixture. The assets are a small educational search benchmark, not a stock-media product.
- Twelve 640px photos are publisher-generated derivatives under the [Pexels License](https://www.pexels.com/license/); no local pixel modifications were made. Pexels IDs identify the sources; attribution is not required by that license. Creator names are not guessed.
- `clip01.webm`: **Cat opening door**, Astropulse, [source](https://commons.wikimedia.org/wiki/File:Cat_opening_door.webm), [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/). The original is unmodified. Reproduced frames are derivatives and retain that license and attribution.
- `clip02.webm`: **Water flowing strem**, BhotaRohit, [source](https://commons.wikimedia.org/wiki/File:Water_flowing_strem.webm), [CC0](https://creativecommons.org/publicdomain/zero/1.0/). Visual inspection reveals a misty waterfall; labels deliberately follow the pixels rather than the filename.
- `clip03.webm` and `sound02.mp3`: MDN contributors, original [interactive-example assets](https://github.com/mdn/interactive-examples), repository [CC0 notice](https://raw.githubusercontent.com/mdn/interactive-examples/main/LICENSE). The files are the real flower timelapse and roaring-creature effect.
- `clip04.mp4`: NASA **Artemis Origins**, [source](https://images.nasa.gov/details/Artemis%20Origins), used for educational/informational evaluation under [NASA media usage permission](https://www.nasa.gov/nasa-brand-center/images-and-media/). No endorsement is implied; NASA insignia/trademarks are not used as Searchi branding. The footage is unmodified.
- `music01.ogg`: **Variatio 13 a 2 Clav.**, Kimiko Ishizaka, [source](https://commons.wikimedia.org/wiki/File:Kimiko_Ishizaka_-_14_-_Variatio_13_a_2_Clav.ogg), [CC0](https://creativecommons.org/publicdomain/zero/1.0/). The project's primary [release description](https://github.com/robertDouglass/opengoldbergvariations.org/blob/master/config.toml) confirms the public-domain release, independently of the Commons pending-review banner. Original audio and Vorbis metadata are retained.
- `music02.ogg`: **Jazz Violin Solo**, VWD, [source](https://commons.wikimedia.org/wiki/File:00_Jazz_Violin_Solo.ogg), author's worldwide public-domain dedication on that page. Original tags identify the artist as **Violinist WTD**, which is intentionally the strict metadata query rather than a guessed identity.
- `sound01.wav`: **Dog barking mono**, Brandon Morris, uploaded by HaelDB, [source](https://opengameart.org/content/dog-barking-mono), [CC0](https://creativecommons.org/publicdomain/zero/1.0/). Original waveform retained.
All source licenses remain attached to their files. Root repository MIT licensing does not replace these third-party media terms. `derived/` and `inspection/` are ignored; generated derivatives carry their source licenses when redistributed.
This collection is deliberately tiny. Twenty public assets do not establish quality on a user's complete photo library, music collection, recorded meetings, or long movies.
fixtures/media/assets/clip01.webm
BIN
View File
Binary file not shown.
fixtures/media/assets/clip02.webm
BIN
View File
Binary file not shown.
fixtures/media/assets/clip03.webm
BIN
View File
Binary file not shown.
fixtures/media/assets/clip04.mp4
BIN
View File
Binary file not shown.
fixtures/media/assets/image01.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 27 KiB

fixtures/media/assets/image02.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 67 KiB

fixtures/media/assets/image03.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 34 KiB

fixtures/media/assets/image04.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 37 KiB

fixtures/media/assets/image05.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 50 KiB

fixtures/media/assets/image06.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 27 KiB

fixtures/media/assets/image07.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 49 KiB

fixtures/media/assets/image08.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 64 KiB

fixtures/media/assets/image09.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 67 KiB

fixtures/media/assets/image10.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 23 KiB

fixtures/media/assets/image11.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 27 KiB

fixtures/media/assets/image12.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 14 KiB

fixtures/media/assets/music01.ogg
BIN
View File
Binary file not shown.
fixtures/media/assets/music02.ogg
BIN
View File
Binary file not shown.
fixtures/media/assets/sound01.wav
BIN
View File
Binary file not shown.
fixtures/media/assets/sound02.mp3
BIN
View File
Binary file not shown.
fixtures/media/document-baseline.json
+852
View File
@@ -0,0 +1,852 @@
{
"background_prewarm_ms": 133.0573,
"device": "cpu",
"documents": 20,
"embedding_report": {
"chunks": 20,
"generation": 1,
"model_fingerprint": "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1:byte64k-overlap512-title512-v1",
"passages": 20
},
"first_foreground_ms": 108.6225,
"first_foreground_top1": "supplier-agreement.txt",
"hybrid": {
"recall_at10": 1.0,
"recall_at3": 1.0,
"top1_accuracy": 1.0
},
"hybrid_p50_ms": 107.3451,
"hybrid_p95_ms": 146.82819999999998,
"indexing_ms": 4840.859100000001,
"lexical": {
"recall_at10": 0.20833333333333334,
"recall_at3": 0.20833333333333334,
"top1_accuracy": 0.20833333333333334
},
"limitation": "Synthetic seed corpus only; no production quality or large-corpus latency certification",
"load_ms": 8644.3299,
"model": "google/embeddinggemma-2",
"queries": 24,
"rankings": [
{
"expected": "supplier-contract",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "supplier-agreement.txt",
"match_kind": "exact_filename",
"source": null
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "INV-2026-1042.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "exact",
"lexical_rank": 1,
"query": "supplier-agreement.txt",
"semantic_rank": 1
},
{
"expected": "invoice",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "INV-2026-1042.txt",
"match_kind": "filename",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "network-incident.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "identifier",
"lexical_rank": 1,
"query": "INV-2026-1042",
"semantic_rank": 1
},
{
"expected": "error-code",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "support-error.txt",
"match_kind": "exact_identifier",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "search-engine-design.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "INV-2026-1042.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "identifier",
"lexical_rank": 1,
"query": "E_SEARCH_071",
"semantic_rank": 1
},
{
"expected": "supplier-contract",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "supplier-agreement.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "retry-design.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "How can we end our relationship with the supplier?",
"semantic_rank": 1
},
{
"expected": "supplier-contract",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "supplier-agreement.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "mietvertrag.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "de-crosslingual",
"lexical_rank": null,
"query": "Wie kündigen wir die Zusammenarbeit mit unserem Lieferanten?",
"semantic_rank": 1
},
{
"expected": "apartment-lease",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "mietvertrag.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "support-error.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "urlaubsrichtlinie.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en-crosslingual",
"lexical_rank": null,
"query": "Can I keep a pet in the rented apartment?",
"semantic_rank": 1
},
{
"expected": "apartment-lease",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "mietvertrag.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "reisekosten.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "monitor-warranty.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "de",
"lexical_rank": null,
"query": "Wo steht etwas über die Kaution für die Wohnung?",
"semantic_rank": 1
},
{
"expected": "travel-expenses",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "reisekosten.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "INV-2026-1042.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "conference-agenda.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en-crosslingual",
"lexical_rank": null,
"query": "How do I claim my train and hotel costs?",
"semantic_rank": 1
},
{
"expected": "conference-agenda",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "conference-agenda.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "support-error.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "search-engine-design.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "de-crosslingual",
"lexical_rank": null,
"query": "Wann ist der Workshop zur Datenbankleistung?",
"semantic_rank": 1
},
{
"expected": "deployment",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "deployment-runbook.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "product-launch.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "support-error.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "What should we do if the new release fails its health check?",
"semantic_rank": 1
},
{
"expected": "backup-restore",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "backup-recovery.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "network-incident.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "de-crosslingual",
"lexical_rank": null,
"query": "Ich habe Dateien gelöscht und brauche sie aus der Sicherung zurück",
"semantic_rank": 1
},
{
"expected": "api-retry",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "retry-design.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "deployment-runbook.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "network-incident.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "How should the app retry a temporary server failure?",
"semantic_rank": 1
},
{
"expected": "oauth",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "authentication-notes.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "retry-design.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "backup-recovery.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "How do we obtain another access token after it expires?",
"semantic_rank": 1
},
{
"expected": "network-outage",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "network-incident.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "backup-recovery.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "de-crosslingual",
"lexical_rank": null,
"query": "Warum war der Netzwerkswitch nicht erreichbar?",
"semantic_rank": 1
},
{
"expected": "launch-plan",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "product-launch.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "research-budget.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "deployment-runbook.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "The plan for announcing the new product after beta testing",
"semantic_rank": 1
},
{
"expected": "research-budget",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "research-budget.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "INV-2026-1042.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "retry-design.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "Money allocated for interviews and prototype usability studies",
"semantic_rank": 1
},
{
"expected": "vacation-policy",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "urlaubsrichtlinie.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "INV-2026-1042.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en-crosslingual",
"lexical_rank": null,
"query": "Who approves my time off and who covers urgent work?",
"semantic_rank": 1
},
{
"expected": "recipe",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "tomato-soup.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "gartenplanung.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "INV-2026-1042.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "A soup made with roasted tomatoes and basil",
"semantic_rank": 1
},
{
"expected": "renovation",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "renovation-estimate.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "INV-2026-1042.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "Does the kitchen renovation quote include new flooring?",
"semantic_rank": 1
},
{
"expected": "warranty",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "monitor-warranty.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "network-incident.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "backup-recovery.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "The document about replacing a defective display under warranty",
"semantic_rank": 1
},
{
"expected": "search-architecture",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "search-engine-design.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "support-error.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "backup-recovery.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": null,
"query": "How can keyword results and embeddings be combined for file search?",
"semantic_rank": 1
},
{
"expected": "garden",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "gartenplanung.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "renovation-estimate.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en-crosslingual",
"lexical_rank": null,
"query": "Where is the plan for the raised vegetable bed and rainwater barrel?",
"semantic_rank": 1
},
{
"expected": "cancellation-email",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "meeting-cancellation.txt",
"match_kind": "hybrid",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "urlaubsrichtlinie.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "conference-agenda.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "en",
"lexical_rank": 1,
"query": "Cancel tomorrow's planning meeting",
"semantic_rank": 1
},
{
"expected": "invoice",
"hybrid_rank": 1,
"hybrid_top3": [
{
"filename": "INV-2026-1042.txt",
"match_kind": "exact_identifier",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "meeting-cancellation.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
},
{
"filename": "search-engine-design.txt",
"match_kind": "semantic",
"source": {
"kind": "line",
"number": 1
}
}
],
"language": "identifier",
"lexical_rank": 1,
"query": "OFFICE-CHAIRS-42",
"semantic_rank": 1
}
],
"semantic": {
"recall_at10": 1.0,
"recall_at3": 1.0,
"top1_accuracy": 1.0
},
"startup_init_ms": 8286.3466,
"threads": 1
}
fixtures/media/environment.json
+14
View File
@@ -0,0 +1,14 @@
{
"date": "2026-10-12",
"os": "Microsoft Windows 11 Pro",
"os_version": "10.0.26300",
"visible_memory_kib": 10416556,
"cpu_name": "AMD Ryzen 7 7800X3D 8-Core Processor ",
"vm_vcpus": 4,
"model_threads": 1,
"device": "cpu",
"force_cpu": "1",
"ffmpeg_version": "9.0.2",
"worker_sha256": "f2ad020307469371519b848c9fa92ae4c51ee8ae8a0ce917f8f408f5756e9086",
"scope": "Shared Windows VM; timing affected by other work; RAM observations are not measured peak"
}
fixtures/media/fixture.json
+1362
View File
File diff suppressed because it is too large. Load diff
fixtures/media/image-budget-results.json
+7525
View File
File diff suppressed because it is too large. Load diff
fixtures/media/image-budget-summary.json
+75
View File
@@ -0,0 +1,75 @@
{
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
"worker": {
"device": "cpu",
"dimensions": 768,
"media_supported": true,
"model": "google/embeddinggemma-2",
"ready": true,
"runtime": "b11461-no-logits-v1",
"threads": 1
},
"photos": 12,
"queries": 24,
"query_embedding_ms": 2262.7884,
"budgets": [
{
"token_budget": 70,
"encoding_ms": 47612.655699999996,
"metrics": {
"positive_queries": 24,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
}
},
{
"token_budget": 280,
"encoding_ms": 199652.08899999998,
"metrics": {
"positive_queries": 24,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
}
},
{
"token_budget": 1120,
"encoding_ms": 1493321.6608,
"metrics": {
"positive_queries": 24,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
}
}
],
"interpretation": "Same frozen 12 photos and 24 correlated EN/DE queries. Query embeddings shared across budgets. Encoding time is CPU batch cost; ranking-only time excludes inference. Equal small-corpus recall does not prove equal quality on larger or more difficult images."
}
fixtures/media/labels.lock.json
+7
View File
@@ -0,0 +1,7 @@
{
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
"frozen_before_media_scoring": true,
"document_seed_sha256": "3bab37fc0ec9c2e2e40684962b049172b5078358c6487c9033ed8697b8128dd7",
"query_count": 65,
"asset_count": 20
}
fixtures/media/metadata-results.json
+102
View File
@@ -0,0 +1,102 @@
{
"encoding_failures": [],
"encoding_ms": 1863.0235000000002,
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
"rows": [
{
"file_rankings": [
{
"candidate": {
"asset": "music01",
"match_kind": "media_metadata",
"modality": "metadata"
},
"score": 2.847686714371868
}
],
"query": {
"category": "exact-metadata",
"expected": [
"music01"
],
"id": "q055",
"language": "en",
"metadata": {
"artist": "Kimiko Ishizaka"
},
"text": "Kimiko Ishizaka"
},
"query_ms": 1.0931,
"segment_top5": []
},
{
"file_rankings": [
{
"candidate": {
"asset": "music02",
"match_kind": "media_metadata",
"modality": "metadata"
},
"score": 4.671020217612865
}
],
"query": {
"category": "exact-metadata",
"expected": [
"music02"
],
"id": "q056",
"language": "en",
"metadata": {
"title": "Jazz Violin Solo"
},
"text": "Jazz Violin Solo"
},
"query_ms": 0.7408,
"segment_top5": []
},
{
"file_rankings": [
{
"candidate": {
"asset": "music02",
"match_kind": "media_metadata",
"modality": "metadata"
},
"score": 3.6759070488571206
}
],
"query": {
"category": "exact-metadata",
"expected": [
"music02"
],
"id": "q057",
"language": "en",
"metadata": {
"artist": "Violinist WTD"
},
"text": "Violinist WTD"
},
"query_ms": 0.6757,
"segment_top5": []
},
{
"file_rankings": [],
"query": {
"category": "metadata-no-answer",
"expected": [],
"id": "q065",
"language": "en",
"metadata": {
"artist": "Adele"
},
"text": "Adele"
},
"query_ms": 0.4148,
"segment_top5": []
}
],
"scope": "Actual FFprobe stream/format metadata + guarded Searchi core FTS; lexical identity queries, no embedding fallback; encoding_ms measures metadata extraction, not model inference",
"worker": null
}
fixtures/media/metadata-summary.json
+75
View File
@@ -0,0 +1,75 @@
{
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
"worker": null,
"overall": {
"positive_queries": 3,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 0.0
},
"by_category": {
"exact-metadata": {
"positive_queries": 3,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"metadata-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 0.0
}
},
"by_language": {
"en": {
"positive_queries": 3,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 0.0
}
},
"encoding_failures": [],
"encoding_ms": 1863.0235000000002,
"query_p50_ms": 0.7408,
"query_p95_ms": 1.0931,
"failures": [],
"source_scope": "Actual FFprobe stream/format metadata + guarded Searchi core FTS; lexical identity queries, no embedding fallback; encoding_ms measures metadata extraction, not model inference",
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
}
fixtures/media/pipeline-10s-results.json
+22540
View File
File diff suppressed because it is too large. Load diff
fixtures/media/pipeline-10s-summary.json
+561
View File
@@ -0,0 +1,561 @@
{
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
"worker": {
"device": "cpu",
"dimensions": 768,
"media_supported": true,
"model": "google/embeddinggemma-2",
"ready": true,
"runtime": "b11461-no-logits-v1",
"threads": 1
},
"overall": {
"positive_queries": 58,
"unsupported_queries": 1,
"recall_at1": 0.9655172413793104,
"recall_at5": 1.0,
"mrr": 0.9827586206896551,
"temporal_queries": 22,
"segment_overlap_hit_at1": 0.8181818181818182,
"timestamp_point_hit_at1": 0.5909090909090909,
"segment_overlap_hit_at5": 0.8181818181818182,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 3.777000000000001,
"no_answer_queries": 6,
"no_answer_return_rate": 1.0
},
"by_category": {
"content-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"exact-filter": {
"positive_queries": 1,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"exact-metadata": {
"positive_queries": 3,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"filter-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"image-memory": {
"positive_queries": 24,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"metadata-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"music-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 0.5,
"recall_at5": 1.0,
"mrr": 0.75,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"sound-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"speech-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"temporal-negative": {
"positive_queries": 0,
"unsupported_queries": 1,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-action": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-scene": {
"positive_queries": 6,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 6,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-speech": {
"positive_queries": 8,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 8,
"segment_overlap_hit_at1": 0.75,
"timestamp_point_hit_at1": 0.125,
"segment_overlap_hit_at5": 0.75,
"timestamp_distance_median_s": 1.551,
"timestamp_distance_max_s": 3.777000000000001,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-temporal": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 0.5,
"timestamp_point_hit_at1": 0.5,
"segment_overlap_hit_at5": 0.5,
"timestamp_distance_median_s": 1.005,
"timestamp_distance_max_s": 1.005,
"no_answer_queries": 0,
"no_answer_return_rate": null
}
},
"by_language": {
"de": {
"positive_queries": 27,
"unsupported_queries": 0,
"recall_at1": 0.9629629629629629,
"recall_at5": 1.0,
"mrr": 0.9814814814814815,
"temporal_queries": 11,
"segment_overlap_hit_at1": 0.8181818181818182,
"timestamp_point_hit_at1": 0.6363636363636364,
"segment_overlap_hit_at5": 0.8181818181818182,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 1.551,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"en": {
"positive_queries": 31,
"unsupported_queries": 1,
"recall_at1": 0.967741935483871,
"recall_at5": 1.0,
"mrr": 0.9838709677419355,
"temporal_queries": 11,
"segment_overlap_hit_at1": 0.8181818181818182,
"timestamp_point_hit_at1": 0.5454545454545454,
"segment_overlap_hit_at5": 0.8181818181818182,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 3.777000000000001,
"no_answer_queries": 4,
"no_answer_return_rate": 1.0
}
},
"encoding_failures": [],
"encoding_ms": 671510.0948000001,
"query_p50_ms": 90.6467,
"query_p95_ms": 110.4152,
"failures": [
{
"query": "q029",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip03",
"end_s": 4.005,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 2005
},
"start_s": 2.005
},
"expected_intervals": {
"clip03": [
[
0,
1.0
]
]
},
"timestamp_distance_s": 1.005
},
{
"query": "q030",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip03",
"end_s": 4.005,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 2005
},
"start_s": 2.005
},
"expected_intervals": {
"clip03": [
[
0,
1.0
]
]
},
"timestamp_distance_s": 1.005
},
{
"query": "q039",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 10.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 0
},
"start_s": 0.0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q040",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 10.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 0
},
"start_s": 0.0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q041",
"category": "wrong-media-channel",
"candidate": {
"asset": "clip04",
"end_s": 26.167,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 24167
},
"start_s": 24.167
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": null
},
{
"query": "q042",
"category": "wrong-media-channel",
"candidate": {
"asset": "clip04",
"end_s": 14.467,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 12467
},
"start_s": 12.467
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": null
},
{
"query": "q043",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 50.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 40000
},
"start_s": 40.0
},
"expected_intervals": {
"clip04": [
[
43.777,
52.419
]
]
},
"timestamp_distance_s": 3.777000000000001
},
{
"query": "q045",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 66.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 56000
},
"start_s": 56.0
},
"expected_intervals": {
"clip04": [
[
56.539,
63.496
]
]
},
"timestamp_distance_s": 0.5390000000000015
},
{
"query": "q046",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 66.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 56000
},
"start_s": 56.0
},
"expected_intervals": {
"clip04": [
[
56.539,
63.496
]
]
},
"timestamp_distance_s": 0.5390000000000015
},
{
"query": "q049",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q050",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q059",
"category": "unsupported-query-capability",
"detail": "Production query API has no temporal-range filter yet"
},
{
"query": "q060",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.01639344262295082
},
{
"query": "q061",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.01639344262295082
},
{
"query": "q062",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.01639344262295082
},
{
"query": "q063",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.01639344262295082
},
{
"query": "q064",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.01639344262295082
},
{
"query": "q065",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.01639344262295082
}
],
"source_scope": "Actual Searchi hybrid pipeline and native extension filters; temporal-range filter unsupported; segment ends inferred from configured audio/frame policy, exact point timestamps come from actual result API",
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
}
fixtures/media/pipeline-30s-results.json
+22531
View File
File diff suppressed because it is too large. Load diff
fixtures/media/pipeline-30s-summary.json
+586
View File
@@ -0,0 +1,586 @@
{
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
"worker": {
"device": "cpu",
"dimensions": 768,
"media_supported": true,
"model": "google/embeddinggemma-2",
"ready": true,
"runtime": "b11461-no-logits-v1",
"threads": 1
},
"overall": {
"positive_queries": 58,
"unsupported_queries": 1,
"recall_at1": 0.9655172413793104,
"recall_at5": 1.0,
"mrr": 0.9827586206896551,
"temporal_queries": 22,
"segment_overlap_hit_at1": 0.8181818181818182,
"timestamp_point_hit_at1": 0.5454545454545454,
"segment_overlap_hit_at5": 0.8181818181818182,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 15.777000000000001,
"no_answer_queries": 6,
"no_answer_return_rate": 1.0
},
"by_category": {
"content-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"exact-filter": {
"positive_queries": 1,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"exact-metadata": {
"positive_queries": 3,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"filter-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"image-memory": {
"positive_queries": 24,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"metadata-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"music-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 0.5,
"recall_at5": 1.0,
"mrr": 0.75,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"sound-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"speech-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"temporal-negative": {
"positive_queries": 0,
"unsupported_queries": 1,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-action": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-scene": {
"positive_queries": 6,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 6,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-speech": {
"positive_queries": 8,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 8,
"segment_overlap_hit_at1": 0.75,
"timestamp_point_hit_at1": 0.0,
"segment_overlap_hit_at5": 0.75,
"timestamp_distance_median_s": 1.551,
"timestamp_distance_max_s": 15.777000000000001,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-temporal": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 0.5,
"timestamp_point_hit_at1": 0.5,
"segment_overlap_hit_at5": 0.5,
"timestamp_distance_median_s": 1.005,
"timestamp_distance_max_s": 1.005,
"no_answer_queries": 0,
"no_answer_return_rate": null
}
},
"by_language": {
"de": {
"positive_queries": 27,
"unsupported_queries": 0,
"recall_at1": 0.9629629629629629,
"recall_at5": 1.0,
"mrr": 0.9814814814814815,
"temporal_queries": 11,
"segment_overlap_hit_at1": 0.8181818181818182,
"timestamp_point_hit_at1": 0.5454545454545454,
"segment_overlap_hit_at5": 0.8181818181818182,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 15.777000000000001,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"en": {
"positive_queries": 31,
"unsupported_queries": 1,
"recall_at1": 0.967741935483871,
"recall_at5": 1.0,
"mrr": 0.9838709677419355,
"temporal_queries": 11,
"segment_overlap_hit_at1": 0.8181818181818182,
"timestamp_point_hit_at1": 0.5454545454545454,
"segment_overlap_hit_at5": 0.8181818181818182,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 15.777000000000001,
"no_answer_queries": 4,
"no_answer_return_rate": 1.0
}
},
"encoding_failures": [],
"encoding_ms": 668040.7295,
"query_p50_ms": 92.2329,
"query_p95_ms": 113.3751,
"failures": [
{
"query": "q029",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip03",
"end_s": 4.005,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 2005
},
"start_s": 2.005
},
"expected_intervals": {
"clip03": [
[
0,
1.0
]
]
},
"timestamp_distance_s": 1.005
},
{
"query": "q030",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip03",
"end_s": 4.005,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 2005
},
"start_s": 2.005
},
"expected_intervals": {
"clip03": [
[
0,
1.0
]
]
},
"timestamp_distance_s": 1.005
},
{
"query": "q039",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 0
},
"start_s": 0.0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q040",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 0
},
"start_s": 0.0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q041",
"category": "wrong-media-channel",
"candidate": {
"asset": "clip04",
"end_s": 26.167,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 24167
},
"start_s": 24.167
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": null
},
{
"query": "q042",
"category": "wrong-media-channel",
"candidate": {
"asset": "clip04",
"end_s": 14.467,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_video",
"modality": "image",
"source": {
"kind": "timestamp_ms",
"number": 12467
},
"start_s": 12.467
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": null
},
{
"query": "q043",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 58.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 28000
},
"start_s": 28.0
},
"expected_intervals": {
"clip04": [
[
43.777,
52.419
]
]
},
"timestamp_distance_s": 15.777000000000001
},
{
"query": "q044",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 58.0,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 28000
},
"start_s": 28.0
},
"expected_intervals": {
"clip04": [
[
43.777,
52.419
]
]
},
"timestamp_distance_s": 15.777000000000001
},
{
"query": "q045",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 80.63,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 56000
},
"start_s": 56.0
},
"expected_intervals": {
"clip04": [
[
56.539,
63.496
]
]
},
"timestamp_distance_s": 0.5390000000000015
},
{
"query": "q046",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 80.63,
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
"match_kind": "semantic_audio",
"modality": "audio",
"source": {
"kind": "timestamp_ms",
"number": 56000
},
"start_s": 56.0
},
"expected_intervals": {
"clip04": [
[
56.539,
63.496
]
]
},
"timestamp_distance_s": 0.5390000000000015
},
{
"query": "q049",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q050",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q059",
"category": "unsupported-query-capability",
"detail": "Production query API has no temporal-range filter yet"
},
{
"query": "q060",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.01639344262295082
},
{
"query": "q061",
"category": "no-answer-nearest-neighbor-return",
"winner": "music02",
"score": 0.01639344262295082
},
{
"query": "q062",
"category": "no-answer-nearest-neighbor-return",
"winner": "music02",
"score": 0.01639344262295082
},
{
"query": "q063",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.01639344262295082
},
{
"query": "q064",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.01639344262295082
},
{
"query": "q065",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.01639344262295082
}
],
"source_scope": "Actual Searchi hybrid pipeline and native extension filters; temporal-range filter unsupported; segment ends inferred from configured audio/frame policy, exact point timestamps come from actual result API",
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
}
fixtures/media/reference-results.json
+13214
View File
File diff suppressed because it is too large. Load diff
fixtures/media/reference-summary.json
+502
View File
@@ -0,0 +1,502 @@
{
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
"worker": {
"device": "cpu",
"dimensions": 768,
"model": "google/embeddinggemma-2",
"ready": true,
"runtime": "b11461-no-logits-v1",
"threads": 1
},
"overall": {
"positive_queries": 55,
"unsupported_queries": 4,
"recall_at1": 0.9636363636363636,
"recall_at5": 0.9636363636363636,
"mrr": 0.9636363636363636,
"temporal_queries": 22,
"segment_overlap_hit_at1": 0.9545454545454546,
"timestamp_point_hit_at1": 0.6818181818181818,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 13.777000000000001,
"no_answer_queries": 6,
"no_answer_return_rate": 1.0
},
"by_category": {
"content-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"exact-filter": {
"positive_queries": 1,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"exact-metadata": {
"positive_queries": 0,
"unsupported_queries": 3,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"filter-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"image-memory": {
"positive_queries": 24,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"metadata-no-answer": {
"positive_queries": 0,
"unsupported_queries": 1,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"music-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 0.5,
"recall_at5": 0.5,
"mrr": 0.5,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"sound-memory": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"speech-no-answer": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"temporal-negative": {
"positive_queries": 0,
"unsupported_queries": 0,
"recall_at1": null,
"recall_at5": null,
"mrr": null,
"temporal_queries": 0,
"segment_overlap_hit_at1": null,
"timestamp_point_hit_at1": null,
"segment_overlap_hit_at5": null,
"timestamp_distance_median_s": null,
"timestamp_distance_max_s": null,
"no_answer_queries": 1,
"no_answer_return_rate": 1.0
},
"video-action": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-scene": {
"positive_queries": 6,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 6,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 1.0,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 0,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-speech": {
"positive_queries": 8,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 8,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 0.25,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 13.129,
"timestamp_distance_max_s": 13.777000000000001,
"no_answer_queries": 0,
"no_answer_return_rate": null
},
"video-temporal": {
"positive_queries": 4,
"unsupported_queries": 0,
"recall_at1": 1.0,
"recall_at5": 1.0,
"mrr": 1.0,
"temporal_queries": 4,
"segment_overlap_hit_at1": 0.75,
"timestamp_point_hit_at1": 0.75,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 1.0,
"no_answer_queries": 0,
"no_answer_return_rate": null
}
},
"by_language": {
"de": {
"positive_queries": 27,
"unsupported_queries": 0,
"recall_at1": 0.9629629629629629,
"recall_at5": 0.9629629629629629,
"mrr": 0.9629629629629629,
"temporal_queries": 11,
"segment_overlap_hit_at1": 0.9090909090909091,
"timestamp_point_hit_at1": 0.6363636363636364,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 13.777000000000001,
"no_answer_queries": 2,
"no_answer_return_rate": 1.0
},
"en": {
"positive_queries": 28,
"unsupported_queries": 4,
"recall_at1": 0.9642857142857143,
"recall_at5": 0.9642857142857143,
"mrr": 0.9642857142857143,
"temporal_queries": 11,
"segment_overlap_hit_at1": 1.0,
"timestamp_point_hit_at1": 0.7272727272727273,
"segment_overlap_hit_at5": 1.0,
"timestamp_distance_median_s": 0,
"timestamp_distance_max_s": 13.777000000000001,
"no_answer_queries": 4,
"no_answer_return_rate": 1.0
}
},
"encoding_failures": [
{
"candidate": {
"asset": "music02",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\music02-audio-0000.f32",
"start_s": 0
},
"error": "Invalid normalized audio PCM"
}
],
"encoding_ms": 498657.0532,
"query_p50_ms": 92.78150000000001,
"query_p95_ms": 122.9277,
"failures": [
{
"query": "q030",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip03",
"end_s": 4,
"modality": "image",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip03-frame-0002.png",
"start_s": 2,
"token_budget": 70
},
"expected_intervals": {
"clip03": [
[
0,
1.0
]
]
},
"timestamp_distance_s": 1.0
},
{
"query": "q039",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
"start_s": 0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q040",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
"start_s": 0
},
"expected_intervals": {
"clip04": [
[
1.551,
3.603
]
]
},
"timestamp_distance_s": 1.551
},
{
"query": "q041",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
"start_s": 0
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": 13.129
},
{
"query": "q042",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 30,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
"start_s": 0
},
"expected_intervals": {
"clip04": [
[
13.129,
16.316
]
]
},
"timestamp_distance_s": 13.129
},
{
"query": "q043",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 60,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0030.f32",
"start_s": 30
},
"expected_intervals": {
"clip04": [
[
43.777,
52.419
]
]
},
"timestamp_distance_s": 13.777000000000001
},
{
"query": "q044",
"category": "timestamp-localization-miss",
"candidate": {
"asset": "clip04",
"end_s": 60,
"modality": "audio",
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0030.f32",
"start_s": 30
},
"expected_intervals": {
"clip04": [
[
43.777,
52.419
]
]
},
"timestamp_distance_s": 13.777000000000001
},
{
"query": "q049",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q050",
"category": "hard-negative-confusion",
"winner": "music01",
"expected": [
"music02"
]
},
{
"query": "q055",
"category": "unsupported-query-capability",
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
},
{
"query": "q056",
"category": "unsupported-query-capability",
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
},
{
"query": "q057",
"category": "unsupported-query-capability",
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
},
{
"query": "q059",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip01",
"score": 0.750250822387441
},
{
"query": "q060",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.5716409940327322
},
{
"query": "q061",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.6223961635527826
},
{
"query": "q062",
"category": "no-answer-nearest-neighbor-return",
"winner": "music01",
"score": 0.6181057666050869
},
{
"query": "q063",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip04",
"score": 0.6229413326862353
},
{
"query": "q064",
"category": "no-answer-nearest-neighbor-return",
"winner": "clip03",
"score": 0.5715326739972073
},
{
"query": "q065",
"category": "unsupported-query-capability",
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
}
],
"source_scope": "Direct real-model dense reference baseline; candidate filters here are benchmark reference constraints, not production-filter certification; no rejection threshold is tuned",
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
}
tools/media-eval/.gitignore
+4
View File
@@ -0,0 +1,4 @@
.venv/
target/
_*.py
__pycache__/
tools/media-eval/Cargo.lock
Generated
+1311
View File
File diff suppressed because it is too large. Load diff
tools/media-eval/Cargo.toml
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "searchi-media-eval"
version = "0.1.0"
edition = "2021"
publish = false
[workspace]
[dependencies]
anyhow = "1"
serde_json = "1"
sha2 = "0.10"
searchi-semantic = { path = "../../crates/searchi-semantic" }
searchi-core = { path = "../../crates/searchi-core" }
searchi-service = { path = "../../crates/searchi-service" }
searchi-media = { path = "../../crates/searchi-media" }
tempfile = "3"
tools/media-eval/README.md
+35
View File
@@ -0,0 +1,35 @@
# Real-media quality evaluation
This independent tool owns no production/UI code. It feeds real pixels and decoded PCM to the actual Google CPU worker, keeps frozen source labels outside inference input, and reports ranking/localization failures.
```powershell
$env:SEARCHI_FORCE_CPU = '1'
python tools/media-eval/download.py
python tools/media-eval/prepare.py --ffmpeg .searchi/media-runtime/bin/ffmpeg.exe --ffprobe .searchi/media-runtime/bin/ffprobe.exe
cargo run --manifest-path tools/media-eval/Cargo.toml --bin searchi-media-eval -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe fixtures/media/reference-results.json
python tools/media-eval/score.py fixtures/media/reference-results.json fixtures/media/reference-summary.json
cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata -- fixtures/media .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe fixtures/media/metadata-results.json
python tools/media-eval/score.py fixtures/media/metadata-results.json fixtures/media/metadata-summary.json
cargo run --manifest-path tools/media-eval/Cargo.toml --bin pipeline -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe fixtures/media/pipeline-30s-results.json 30
cargo run --manifest-path tools/media-eval/Cargo.toml --bin pipeline -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe fixtures/media/pipeline-10s-results.json 10
python tools/media-eval/score.py fixtures/media/pipeline-30s-results.json fixtures/media/pipeline-30s-summary.json
python tools/media-eval/score.py fixtures/media/pipeline-10s-results.json fixtures/media/pipeline-10s-summary.json
cargo run --manifest-path tools/media-eval/Cargo.toml --bin image_budgets -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe fixtures/media/image-budget-results.json
python tools/media-eval/score_budgets.py fixtures/media/image-budget-results.json fixtures/media/image-budget-summary.json
python -m unittest discover -s tools/media-eval -p test_*.py
```
The standalone Cargo workspace keeps evaluation dependencies separate. Native inference and FFmpeg decoding use CPU only and one thread. No test enables a GPU provider. The model/worker/decoder paths above refer to explicitly provisioned local developer artifacts; they are not end-user setup requirements.
The dense reference samples video every two seconds and PCM in thirty-second windows. Photos use 140 image tokens and frames 70. The separate comparison fixes the same 12 photos/24 queries and varies only photo token budgets **70, 280, 1120**. It does not change labels or choose a threshold after observing scores.
## Interpretation
- Recall@1/@5 and MRR rank distinct source files, choosing each file's best segment. Per-category/per-language metrics remain separate.
- Segment overlap credits a returned segment that actually intersects the labelled event in the correct file. Point hits require the returned start timestamp itself to fall inside the event. A thirty-second speech window may pass overlap while opening far too early; both metrics are required.
- No-answer queries remain separate from positive recall. The tool exposes returned neighbors and scores without inventing an abstention threshold. A reject threshold needs independent calibration and held-out validation.
- Direct model candidate filters are **reference pool constraints**, not evidence that production filters work. Artist/title identity cannot be recovered reliably from a performance embedding; exact metadata rows are explicitly unsupported until evaluated through Searchi's real lexical metadata lane.
- `metadata` tests the actual FFprobe/guarded core FTS lane; `pipeline` tests the actual unified Searchi media generation and hybrid ranking. The optional final argument varies audio window seconds (30 versus 10) while retaining two-second overlap, the same real files, decoder and frozen labels. It does not change production defaults.
- Model agreement with another implementation, synthetic document-seed scores, and this small-media benchmark each answer different questions. None is production quality certification.
Detailed provenance is in [the fixture README](../../fixtures/media/README.md). The final assessment belongs in [MEDIA-QUALITY.md](../../docs/MEDIA-QUALITY.md).
tools/media-eval/download.py
+32
View File
@@ -0,0 +1,32 @@
"""Reacquire the locked licensed corpus. Updated bytes are never accepted."""
import argparse
import hashlib
import json
from pathlib import Path
import urllib.request
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--root", type=Path, default=Path("fixtures/media"))
args = parser.parse_args()
root = args.root.resolve()
fixture_path = root / "fixture.json"
lock = json.loads((root / "labels.lock.json").read_text())
assert hashlib.sha256(fixture_path.read_bytes()).hexdigest() == lock["fixture_sha256"]
for asset in json.loads(fixture_path.read_text(encoding="utf-8"))["assets"]:
destination = (root / asset["path"]).resolve()
assert destination.is_relative_to(root)
if destination.is_file() and hashlib.sha256(destination.read_bytes()).hexdigest() == asset["sha256"]:
continue
request = urllib.request.Request(asset["source_url"], headers={"User-Agent": "SearchiMediaEvaluation/1.0"})
with urllib.request.urlopen(request, timeout=60) as response:
data = response.read(asset["bytes"] + 1)
assert len(data) == asset["bytes"] and hashlib.sha256(data).hexdigest() == asset["sha256"], f"Publisher changed {asset['id']}; keep frozen corpus, investigate a new version"
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(data)
print("Verified", asset["id"])
if __name__ == "__main__":
main()
tools/media-eval/prepare.py
+62
View File
@@ -0,0 +1,62 @@
"""Prepare real CPU-decoded media candidates after verifying frozen labels.
No generated captions, placeholder vectors, or model calls occur in preparation.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
def verify(root):
fixture_path = root / "fixture.json"
lock = json.loads((root / "labels.lock.json").read_text())
assert hashlib.sha256(fixture_path.read_bytes()).hexdigest() == lock["fixture_sha256"], "Frozen labels changed"
fixture = json.loads(fixture_path.read_text(encoding="utf-8"))
for asset in fixture["assets"]:
path = (root / asset["path"]).resolve()
assert path.is_relative_to(root.resolve()), "Asset escaped corpus root"
assert path.stat().st_size == asset["bytes"], asset["id"]
assert hashlib.sha256(path.read_bytes()).hexdigest() == asset["sha256"], asset["id"]
return fixture
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--root", type=Path, default=Path("fixtures/media"))
parser.add_argument("--ffmpeg", type=Path, required=True)
parser.add_argument("--ffprobe", type=Path, required=True)
args = parser.parse_args()
root = args.root.resolve()
fixture = verify(root)
derived = root / "derived"
derived.mkdir(exist_ok=True)
candidates = []
for asset in fixture["assets"]:
source = root / asset["path"]
if asset["kind"] == "image":
candidates.append(dict(asset=asset["id"], path=str(source), modality="image", token_budget=140))
continue
probe = json.loads(subprocess.check_output([str(args.ffprobe), "-v", "error", "-show_format", "-show_streams", "-of", "json", str(source)], text=True))
duration = float(probe["format"]["duration"])
if asset["kind"] == "video":
for stamp in range(0, int(duration) + 1, 2):
output = derived / f"{asset['id']}-frame-{stamp:04}.png"
subprocess.run([str(args.ffmpeg), "-hide_banner", "-loglevel", "error", "-hwaccel", "none", "-threads", "1", "-ss", str(stamp), "-i", str(source), "-frames:v", "1", "-vf", "scale=480:-1", "-threads", "1", "-y", str(output)], check=True)
if output.is_file():
candidates.append(dict(asset=asset["id"], path=str(output), modality="image", token_budget=70, start_s=stamp, end_s=min(stamp + 2, duration)))
if any(stream["codec_type"] == "audio" for stream in probe["streams"]):
for stamp in range(0, int(duration) + 1, 30):
output = derived / f"{asset['id']}-audio-{stamp:04}.f32"
subprocess.run([str(args.ffmpeg), "-hide_banner", "-loglevel", "error", "-hwaccel", "none", "-threads", "1", "-ss", str(stamp), "-i", str(source), "-t", "30", "-vn", "-ac", "1", "-ar", "16000", "-f", "f32le", "-threads", "1", "-y", str(output)], check=True)
if output.stat().st_size:
assert output.stat().st_size <= 480000 * 4, "Audio exceeds worker budget"
candidates.append(dict(asset=asset["id"], path=str(output), modality="audio", start_s=stamp, end_s=min(stamp + 30, duration)))
manifest = dict(fixture_sha256=hashlib.sha256((root / "fixture.json").read_bytes()).hexdigest(), ffmpeg_version=subprocess.check_output([str(args.ffmpeg), "-version"], text=True).splitlines()[0], policy="CPU-only dense reference: photos140/video70 tokens, uniform2s frames, mono16kHz30s PCM", candidates=candidates)
(derived / "candidates.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
print(json.dumps(dict(candidates=len(candidates), images=sum(item["modality"] == "image" for item in candidates), audio=sum(item["modality"] == "audio" for item in candidates))))
if __name__ == "__main__":
main()
tools/media-eval/score.py
+114
View File
@@ -0,0 +1,114 @@
"""Score frozen relevance without fitting an abstention threshold."""
import argparse
from collections import defaultdict
import hashlib
import json
from pathlib import Path
def temporal_hit(candidate, query, point=False):
asset = candidate.get("asset")
if asset not in query.get("expected", []):
return False
if query.get("required_modality") and candidate.get("modality") != query["required_modality"]:
return False
ranges = query.get("intervals", {}).get(asset, [])
start = candidate.get("start_s")
end = candidate.get("end_s", start)
if start is None:
return False
if point:
return any(left <= start <= right for left, right in ranges)
return any(min(end, right) > max(start, left) for left, right in ranges)
def timestamp_distance(candidate, query):
"""Raw seconds from returned start to the nearest labelled event interval."""
if candidate.get("asset") not in query.get("expected", []) or candidate.get("start_s") is None:
return None
if query.get("required_modality") and candidate.get("modality") != query["required_modality"]:
return None
ranges = query.get("intervals", {}).get(candidate["asset"], [])
stamp = candidate["start_s"]
return min((max(left - stamp, stamp - right, 0) for left, right in ranges), default=None)
def metrics(rows):
positive = [row for row in rows if row["query"]["expected"] and not row.get("unsupported")]
negative = [row for row in rows if not row["query"]["expected"] and not row.get("unsupported")]
temporal = [row for row in positive if row["query"]["category"].startswith("video-")]
distances = sorted(distance for row in temporal if row["file_rankings"] and (distance := timestamp_distance(row["file_rankings"][0]["candidate"], row["query"])) is not None)
ranks = []
for row in positive:
expected = row["query"]["expected"]
ranks.append(next((i + 1 for i, hit in enumerate(row["file_rankings"]) if hit["candidate"]["asset"] in expected), None))
fraction = lambda count, total: count / total if total else None
return dict(
positive_queries=len(positive),
unsupported_queries=sum(bool(row.get("unsupported")) for row in rows),
recall_at1=fraction(sum(rank == 1 for rank in ranks), len(ranks)),
recall_at5=fraction(sum(rank is not None and rank <= 5 for rank in ranks), len(ranks)),
mrr=fraction(sum(1 / rank if rank else 0 for rank in ranks), len(ranks)),
temporal_queries=len(temporal),
segment_overlap_hit_at1=fraction(sum(bool(row["file_rankings"]) and temporal_hit(row["file_rankings"][0]["candidate"], row["query"]) for row in temporal), len(temporal)),
timestamp_point_hit_at1=fraction(sum(bool(row["file_rankings"]) and temporal_hit(row["file_rankings"][0]["candidate"], row["query"], point=True) for row in temporal), len(temporal)),
segment_overlap_hit_at5=fraction(sum(any(temporal_hit(hit["candidate"], row["query"]) for hit in row["segment_top5"]) for row in temporal), len(temporal)),
timestamp_distance_median_s=distances[len(distances) // 2] if distances else None,
timestamp_distance_max_s=max(distances) if distances else None,
no_answer_queries=len(negative),
no_answer_return_rate=fraction(sum(bool(row["file_rankings"]) for row in negative), len(negative)),
)
def summarize(report):
rows = report["rows"]
by_category = defaultdict(list)
by_language = defaultdict(list)
failures = []
for row in rows:
query = row["query"]
by_category[query["category"]].append(row)
by_language[query["language"]].append(row)
if row.get("unsupported"):
failures.append(dict(query=query["id"], category="unsupported-query-capability", detail=row["unsupported"]))
continue
hits = row["file_rankings"]
if not query["expected"]:
if hits:
failures.append(dict(query=query["id"], category="no-answer-nearest-neighbor-return", winner=hits[0]["candidate"]["asset"], score=hits[0]["score"]))
elif not hits or hits[0]["candidate"]["asset"] not in query["expected"]:
winner = hits[0]["candidate"]["asset"] if hits else None
failures.append(dict(query=query["id"], category="hard-negative-confusion" if winner in query.get("hard_negatives", []) else "retrieval-top1-miss", winner=winner, expected=query["expected"]))
elif query["category"].startswith("video-") and not temporal_hit(hits[0]["candidate"], query, point=True):
candidate = hits[0]["candidate"]
wrong_channel = query.get("required_modality") and candidate.get("modality") != query["required_modality"]
failures.append(dict(query=query["id"], category="wrong-media-channel" if wrong_channel else "timestamp-localization-miss", candidate=candidate, expected_intervals=query["intervals"], timestamp_distance_s=timestamp_distance(candidate, query)))
durations = sorted(row["query_ms"] for row in rows if "query_ms" in row)
return dict(
fixture_sha256=report["fixture_sha256"], worker=report["worker"],
overall=metrics(rows), by_category={key: metrics(value) for key, value in sorted(by_category.items())},
by_language={key: metrics(value) for key, value in sorted(by_language.items())},
encoding_failures=report["encoding_failures"], encoding_ms=report["encoding_ms"],
query_p50_ms=durations[len(durations) // 2] if durations else None,
query_p95_ms=durations[min(len(durations) * 95 // 100, len(durations) - 1)] if durations else None,
failures=failures,
source_scope=report.get("scope", "Direct dense model baseline"),
interpretation="Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality.",
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("report", type=Path)
parser.add_argument("output", type=Path)
args = parser.parse_args()
report = json.loads(args.report.read_text(encoding="utf-8"))
fixture = Path("fixtures/media/fixture.json")
assert report["fixture_sha256"] == hashlib.sha256(fixture.read_bytes()).hexdigest()
summary = summarize(report)
args.output.write_text(json.dumps(summary, indent=2), encoding="utf-8")
print(json.dumps(summary["overall"]))
if __name__ == "__main__":
main()
Loaded 100 of 110 files, more files were not shown because too many files have changed in this diff. Show more