feat: add multimodal search and granular performance diagnostics
Core CI / core (ubuntu-latest) (push) Failing after 3m31s
Core CI / core (macos-latest) (push) Canceled after 0s
Core CI / core (windows-latest) (push) Canceled after 0s
Core CI / desktop (ubuntu-latest) (push) Successful in 5m39s
Core CI / desktop (macos-latest) (push) Canceled after 0s
Core CI / desktop (windows-latest) (push) Canceled after 0s
No files matched your search
@@ -9,5 +9,6 @@ __pycache__/
|
|||||||
/apps/desktop/src-tauri/gen/
|
/apps/desktop/src-tauri/gen/
|
||||||
/apps/desktop/src-tauri/resources/searchi-gemma-worker*
|
/apps/desktop/src-tauri/resources/searchi-gemma-worker*
|
||||||
/apps/desktop/src-tauri/resources/embeddinggemma2/
|
/apps/desktop/src-tauri/resources/embeddinggemma2/
|
||||||
|
/apps/desktop/src-tauri/resources/media-runtime/bin/
|
||||||
/apps/desktop/src-tauri/icons/android/
|
/apps/desktop/src-tauri/icons/android/
|
||||||
/apps/desktop/src-tauri/icons/ios/
|
/apps/desktop/src-tauri/icons/ios/
|
||||||
@@ -10,6 +10,9 @@ readiness without completed release gates and recorded evidence.
|
|||||||
- Production uses available GPU/hardware acceleration with reliable CPU fallback;
|
- Production uses available GPU/hardware acceleration with reliable CPU fallback;
|
||||||
the CPU-only rule is a VM testing constraint, not a product restriction.
|
the CPU-only rule is a VM testing constraint, not a product restriction.
|
||||||
- Keep the Rust engine independent of the desktop shell; CLI and UI share its API.
|
- Keep the Rust engine independent of the desktop shell; CLI and UI share its API.
|
||||||
|
- Prefer established Rust crates and proven retrieval algorithms over custom
|
||||||
|
implementations. Keep platform adapters small, bound allocations and queues,
|
||||||
|
and measure performance before adding caches or changing ranking policies.
|
||||||
- Use bounded work, explicit errors, versioned storage, and offline defaults.
|
- Use bounded work, explicit errors, versioned storage, and offline defaults.
|
||||||
- Never substitute simulated embeddings for working semantic search.
|
- Never substitute simulated embeddings for working semantic search.
|
||||||
- Production semantic search must use Google EmbeddingGemma 2 as in DigUp.
|
- Production semantic search must use Google EmbeddingGemma 2 as in DigUp.
|
||||||
|
|||||||
@@ -207,6 +207,18 @@ version = "3.20.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bytemuck"
|
||||||
|
version = "1.25.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "byteorder-lite"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bytes"
|
name = "bytes"
|
||||||
version = "1.12.1"
|
version = "1.12.1"
|
||||||
@@ -705,6 +717,15 @@ version = "2.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fdeflate"
|
||||||
|
version = "0.3.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
|
||||||
|
dependencies = [
|
||||||
|
"simd-adler32",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "find-msvc-tools"
|
name = "find-msvc-tools"
|
||||||
version = "0.1.14"
|
version = "0.1.14"
|
||||||
@@ -718,7 +739,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crc32fast",
|
"crc32fast",
|
||||||
"miniz_oxide",
|
"miniz_oxide 0.9.1",
|
||||||
"zlib-rs",
|
"zlib-rs",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1098,6 +1119,32 @@ dependencies = [
|
|||||||
"icu_properties",
|
"icu_properties",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "image"
|
||||||
|
version = "0.25.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||||
|
dependencies = [
|
||||||
|
"bytemuck",
|
||||||
|
"byteorder-lite",
|
||||||
|
"image-webp",
|
||||||
|
"moxcms",
|
||||||
|
"num-traits",
|
||||||
|
"png",
|
||||||
|
"zune-core",
|
||||||
|
"zune-jpeg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "image-webp"
|
||||||
|
version = "0.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder-lite",
|
||||||
|
"quick-error",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "indexmap"
|
name = "indexmap"
|
||||||
version = "2.14.2"
|
version = "2.14.2"
|
||||||
@@ -1145,6 +1192,12 @@ version = "1.0.18"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "jpeg-decoder"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "js-sys"
|
name = "js-sys"
|
||||||
version = "0.3.106"
|
version = "0.3.106"
|
||||||
@@ -1300,6 +1353,16 @@ version = "0.2.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "miniz_oxide"
|
||||||
|
version = "0.8.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||||
|
dependencies = [
|
||||||
|
"adler2",
|
||||||
|
"simd-adler32",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "miniz_oxide"
|
name = "miniz_oxide"
|
||||||
version = "0.9.1"
|
version = "0.9.1"
|
||||||
@@ -1343,6 +1406,16 @@ dependencies = [
|
|||||||
"syn 2.0.119",
|
"syn 2.0.119",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "moxcms"
|
||||||
|
version = "0.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
"pxfm",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "multiversion_no_op"
|
name = "multiversion_no_op"
|
||||||
version = "1.0.0"
|
version = "1.0.0"
|
||||||
@@ -1486,6 +1559,19 @@ version = "0.3.34"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "png"
|
||||||
|
version = "0.18.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"crc32fast",
|
||||||
|
"fdeflate",
|
||||||
|
"flate2",
|
||||||
|
"miniz_oxide 0.8.9",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "portable-atomic"
|
name = "portable-atomic"
|
||||||
version = "1.15.0"
|
version = "1.15.0"
|
||||||
@@ -1528,6 +1614,18 @@ dependencies = [
|
|||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pxfm"
|
||||||
|
version = "0.1.30"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quick-error"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quick-xml"
|
name = "quick-xml"
|
||||||
version = "0.42.0"
|
version = "0.42.0"
|
||||||
@@ -1946,11 +2044,26 @@ dependencies = [
|
|||||||
"zip",
|
"zip",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "searchi-media"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"image",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"sha2 0.10.9",
|
||||||
|
"tempfile",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "searchi-semantic"
|
name = "searchi-semantic"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"base64 0.22.1",
|
||||||
|
"image",
|
||||||
|
"jpeg-decoder",
|
||||||
"ort",
|
"ort",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -1966,6 +2079,7 @@ dependencies = [
|
|||||||
"fs2",
|
"fs2",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"searchi-core",
|
"searchi-core",
|
||||||
|
"searchi-media",
|
||||||
"searchi-semantic",
|
"searchi-semantic",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -2880,3 +2994,18 @@ dependencies = [
|
|||||||
"log",
|
"log",
|
||||||
"simd-adler32",
|
"simd-adler32",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-core"
|
||||||
|
version = "0.5.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-jpeg"
|
||||||
|
version = "0.5.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
|
||||||
|
dependencies = [
|
||||||
|
"zune-core",
|
||||||
|
]
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
[workspace]
|
[workspace]
|
||||||
members = ["crates/searchi-core", "crates/searchi-cli", "crates/searchi-semantic", "crates/searchi-service", "crates/searchi-assets"]
|
members = ["crates/searchi-core", "crates/searchi-cli", "crates/searchi-semantic", "crates/searchi-service", "crates/searchi-assets", "crates/searchi-media"]
|
||||||
resolver = "2"
|
resolver = "2"
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
|
|||||||
@@ -2,89 +2,144 @@
|
|||||||
|
|
||||||
**Find what matters. Locally.**
|
**Find what matters. Locally.**
|
||||||
|
|
||||||
Local desktop search for Windows, macOS and Linux. Under active development;
|
Local desktop search for Windows, macOS and Linux, under active development.
|
||||||
production readiness is tracked in [the roadmap](docs/ROADMAP.md).
|
Production readiness is tracked in [the roadmap](docs/ROADMAP.md).
|
||||||
|
|
||||||
## Architecture
|
## What works
|
||||||
|
|
||||||
`crates/searchi-core` owns persistent indexing and retrieval. `crates/searchi-cli`
|
Search combines instant indexed filenames, SQLite FTS5 content retrieval, and
|
||||||
provides a headless interface. `apps/desktop` hosts the Tauri desktop app.
|
**real Google EmbeddingGemma 2** vectors in a persisted HNSW index. PDF pages,
|
||||||
Search combines indexed filenames, SQLite FTS5 keyword retrieval, and real
|
DOCX paragraphs and text passages retain their source locations. The same
|
||||||
Google EmbeddingGemma 2 semantic search with a persistent HNSW index. PDF pages
|
shared Rust service now indexes **images, video frames, movie soundtracks,
|
||||||
and DOCX paragraphs retain source locations. Image, audio, and video search remain
|
audio and music**: actual pixels and sound enter Google's media projector in
|
||||||
planned; the provisioned media projector is not yet used. Granola is not required.
|
the same 768-dimensional space as text queries. No generated captions,
|
||||||
The desktop UI is primary; the CLI builds separately without GUI dependencies.
|
simulated embeddings, replacement model, or Granola service is required.
|
||||||
|
Existing artist/title tags contribute keyword matches.
|
||||||
|
|
||||||
Production uses supported hardware acceleration with CPU fallback. All testing
|
The desktop is the primary interface. It automatically prepares verified local
|
||||||
in the Windows development VM uses CPU only. Indexed content stays
|
models, indexes selected folders, and prewarms its query worker. Image previews
|
||||||
local. Treat the index as sensitive: it contains extracted document text.
|
and audio/video controls use narrowly scoped local assets; timestamp results
|
||||||
|
seek inside the preview without autoplay. Open file launches the default
|
||||||
|
application, whose exact opening position is not controlled by Searchi.
|
||||||
|
The CLI provides the same retrieval engine without a GUI.
|
||||||
|
|
||||||
The desktop automatically downloads and verifies missing Google artifacts
|
**Media search is experimental.** File retrieval works on the independent seed
|
||||||
(approximately 865 MB), reuses its local cache, and warms the query session in
|
corpus, but precise spoken-topic timestamps, music hard negatives, and no-answer
|
||||||
the background. Development builds must build and stage the native worker first;
|
handling still need improvement. Nearest-neighbor results are not calibrated
|
||||||
production installers and signing remain release gates. First-query performance
|
confidence. Video frames are sampled across the movie; every fleeting scene is
|
||||||
is measured separately from repeated-query performance.
|
not guaranteed. Audio windows cover the primary stream within explicit resource
|
||||||
|
limits. See [actual media quality and failures](docs/MEDIA-QUALITY.md).
|
||||||
|
|
||||||
|
Production supports compiled hardware acceleration with CPU fallback. All
|
||||||
|
Windows VM tests use CPU only. Files, extracted text, metadata and embeddings
|
||||||
|
stay local; treat the index as sensitive. Initial verified model provisioning
|
||||||
|
requires approximately 865 MB of downloads. Search performs no remote inference.
|
||||||
|
|
||||||
## Copy-and-paste setup prompt
|
## Copy-and-paste setup prompt
|
||||||
|
|
||||||
Paste this into your coding agent:
|
|
||||||
|
|
||||||
```text
|
```text
|
||||||
Set up Searchi from https://tea.karlcom.de/karl/searchi.git in a new searchi
|
Set up Searchi from https://tea.karlcom.de/karl/searchi.git in a new searchi
|
||||||
directory, or reuse the existing checkout without overwriting local changes.
|
folder, or reuse the checkout without overwriting local changes. Read AGENTS.md,
|
||||||
Read AGENTS.md, SKILLS.md, docs/ROADMAP.md, and apps/desktop/README.md first.
|
SKILLS.md, docs/ROADMAP.md, docs/BUILDING.md and apps/desktop/README.md first.
|
||||||
Inspect the OS and installed tools. Install missing supported prerequisites:
|
Inspect the OS/tools. Install supported missing prerequisites: Git, stable Rust,
|
||||||
Git, stable Rust, Node.js 24+, Python 3, CMake, native C/C++ tools and Tauri's
|
Node.js 24 LTS, Python 3.11+, CMake, native C/C++ tools and Tauri dependencies.
|
||||||
platform dependencies. Preserve existing credentials and never print secrets.
|
Preserve credentials and never print secrets. Prefer established Rust crates
|
||||||
From the repository root, build and stage the pinned Google runtime:
|
and portable native tools; keep engine APIs shared by CLI and desktop.
|
||||||
|
For Karl's Windows VM set SEARCHI_FORCE_CPU=1 and
|
||||||
|
WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS='--disable-gpu --disable-accelerated-video-decode'
|
||||||
|
before tests or launch. Never enable a GPU provider in that VM.
|
||||||
|
From the repository root:
|
||||||
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
||||||
python tools/embeddinggemma2/stage_runtime.py --backend cpu
|
python tools/embeddinggemma2/stage_runtime.py --backend cpu
|
||||||
Run cargo test --workspace --locked, cargo fmt --all -- --check, and
|
python tools/media-runtime/provision.py --stage
|
||||||
|
The decoder manifest supports Windows/Linux x86-64 and ARM64; macOS decoder
|
||||||
|
packaging is still a gate. On unsupported hosts report the real blocker and
|
||||||
|
use explicitly verified native tools only when their provenance is known.
|
||||||
|
Run cargo test --workspace --locked, cargo fmt --all --check and
|
||||||
cargo clippy --workspace --all-targets --locked -- -D warnings.
|
cargo clippy --workspace --all-targets --locked -- -D warnings.
|
||||||
Run cargo test --manifest-path apps/desktop/src-tauri/Cargo.toml --locked.
|
Run cargo test --manifest-path apps/desktop/src-tauri/Cargo.toml --locked.
|
||||||
In apps/desktop run npm ci, npm test, npm run format:check, and npm run build.
|
In apps/desktop run npm ci, npm test, npm run format:check and npm run build.
|
||||||
For Karl's Windows VM set SEARCHI_FORCE_CPU=1 and
|
Launch the real app with npm run tauri -- dev. Allow automatic verified Google
|
||||||
WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS=--disable-gpu before tests or launch.
|
model setup and retained-worker warmup. Never substitute another model or mocks.
|
||||||
Launch the real app using npm run tauri dev from apps/desktop. Allow automatic
|
Ask which folders I want indexed; don't index my whole home automatically.
|
||||||
verified Google model setup; never substitute another model or mock results.
|
Verify filename/content/image/music/video searches and persistence on selected
|
||||||
Ask which folders I want indexed; do not index my whole home automatically.
|
files. Inspect media failure reports and actual preview seeking. Read
|
||||||
Verify filename and remembered-content searches on my selected files, including
|
MEDIA-QUALITY.md before claiming precise speech timestamps or reliable absence.
|
||||||
persistence after restart. Report actual checks and blockers. Do not publish a
|
Report actual checks and blockers. Do not publish a release or claim production
|
||||||
release or claim production readiness, GPU validation, or untested OS coverage.
|
readiness, GPU validation, native visual QA, or untested OS coverage.
|
||||||
```
|
```
|
||||||
|
|
||||||
This is developer setup, not an installer. See the [desktop guide](apps/desktop/README.md)
|
This is developer setup, not an installer. The worker builder also accepts
|
||||||
and [official Tauri prerequisites](https://v2.tauri.app/start/prerequisites/).
|
`auto`, `metal`, `vulkan` and `cuda`; each requires its platform SDK and separate
|
||||||
The worker builder also supports `auto`, `metal`, `vulkan`, and `cuda`; those
|
hardware validation. Standalone FFmpeg/FFprobe and notices are staged for the
|
||||||
require platform SDKs and separate hardware validation.
|
desktop; end users should not install tools or configure decoder paths.
|
||||||
|
|
||||||
## Development
|
## Headless setup
|
||||||
|
|
||||||
Install stable Rust and the platform's C/C++ build prerequisites. Run:
|
Build with `cargo build --release --locked -p searchi-cli`. No Node, Tauri,
|
||||||
|
WebView or display server is needed. Build the native Google worker separately,
|
||||||
|
and provision verified assets and FFmpeg tools:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
cargo test --workspace
|
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
||||||
cargo fmt --all -- --check
|
python tools/media-runtime/provision.py
|
||||||
cargo clippy --workspace --all-targets -- -D warnings
|
cargo run -p searchi-cli -- setup --model-dir .searchi/models/embeddinggemma2
|
||||||
|
cargo run -p searchi-cli -- index --db searchi.sqlite --root /path/to/files
|
||||||
|
```
|
||||||
|
|
||||||
|
On native Linux, unified document/media indexing and hybrid search use:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --ffmpeg .searchi/media-runtime/bin/ffmpeg --ffprobe .searchi/media-runtime/bin/ffprobe --cpu-threads 1
|
||||||
|
./target/release/searchi search --db searchi.sqlite 'the recording about orange soil' --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --cpu-threads 1
|
||||||
|
```
|
||||||
|
|
||||||
|
These are current interfaces, not executed Linux release evidence. Windows uses
|
||||||
|
`Release/searchi-gemma-worker.exe` and `ffmpeg.exe`/`ffprobe.exe` instead.
|
||||||
|
`embed` includes media by default and reports failures. Use `--text-only`
|
||||||
|
explicitly for document-only vectors. `setup` provisions Google assets, not
|
||||||
|
worker/decoder executables. Installed release CLIs discover decoders under
|
||||||
|
`media-runtime/bin` beside the CLI; explicit paired paths or
|
||||||
|
`SEARCHI_FFMPEG_PATH`/`SEARCHI_FFPROBE_PATH` override discovery. Search never
|
||||||
|
silently downloads missing helpers. CLI invocations load their own worker;
|
||||||
|
the desktop retains a prewarmed worker for interactive queries.
|
||||||
|
|
||||||
|
## Search performance
|
||||||
|
|
||||||
|
Normal searches show result counts, measured search time and time until results
|
||||||
|
are displayed. Enable **Performance details** in desktop settings to
|
||||||
|
inspect UI scheduling, native queue/lock time, keyword retrieval, actual Google
|
||||||
|
query inference, ANN/cache/filter work and ranking. Export the current trace as
|
||||||
|
local JSON for comparison; tracing sends no telemetry and excludes search text
|
||||||
|
and file paths. Nested stages and parallel retrieval lanes overlap.
|
||||||
|
|
||||||
|
For headless diagnostics, append `--performance --json` for basic metrics or
|
||||||
|
`--debug-performance --json` for the detailed response. Plain `--json` retains
|
||||||
|
the existing results-array format. See [timing boundaries, trace schema and
|
||||||
|
reproducible measurements](docs/PERFORMANCE.md).
|
||||||
|
|
||||||
|
## Development and release
|
||||||
|
|
||||||
|
The Rust engine, service, media adapter and CLI live under `crates/`; the Tauri
|
||||||
|
shell is `apps/desktop`. SQLite, image/jpeg-decoder, USearch/NumKong, and Tauri
|
||||||
|
provide established storage, decoding, ANN/SIMD and desktop facilities.
|
||||||
|
Extraction and inference use bounded isolated work with explicit errors.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo test --workspace --locked
|
||||||
|
cargo fmt --all --check
|
||||||
|
cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||||
cargo run -p searchi-cli -- --help
|
cargo run -p searchi-cli -- --help
|
||||||
```
|
```
|
||||||
|
|
||||||
Desktop development instructions live in `apps/desktop/README.md`.
|
See [desktop setup](apps/desktop/README.md), [build/platform constraints](docs/BUILDING.md),
|
||||||
Agent constraints and workflows live in [AGENTS.md](AGENTS.md) and
|
[semantic bounds](docs/SEMANTIC.md), [executed validation](docs/VALIDATION.md),
|
||||||
[SKILLS.md](SKILLS.md). Use Node.js 24+ for frontend state tests.
|
and [brand direction](docs/BRAND.md). Native visual QA, operating-system builds,
|
||||||
The approved visual identity and desktop direction live in [the brand guide](docs/BRAND.md).
|
macOS decoder packaging, installers, licensing distribution, signatures,
|
||||||
|
upgrades and representative production quality remain release gates.
|
||||||
|
Cross-platform source support does not mean signed macOS installers can be
|
||||||
|
built on Windows. CI declarations are not evidence of executed platform jobs.
|
||||||
|
|
||||||
For a headless build, run `cargo build --release --locked -p searchi-cli`.
|
Record benchmark hardware, corpus, cold/warm state and p95 latency. Never ship
|
||||||
No Node, Tauri, or WebView is needed. Semantic commands additionally need the
|
signing credentials in source control. Licensed under [MIT](LICENSE).
|
||||||
native worker and Google assets; `searchi setup` provisions the model cache.
|
Repository author: Karl.
|
||||||
Platform and CI details live in [the build guide](docs/BUILDING.md).
|
|
||||||
Native CI builds must pass on all supported systems before release. Cross-platform
|
|
||||||
source support does not imply signed macOS installers can be built on Windows.
|
|
||||||
|
|
||||||
## Release policy
|
|
||||||
|
|
||||||
Do not publish until the roadmap's release gates have evidence. Record benchmark
|
|
||||||
hardware, corpus size, cold/warm state and p95 latency. Signing credentials are
|
|
||||||
supplied through CI secrets, never source control.
|
|
||||||
|
|
||||||
Licensed under [MIT](LICENSE). Repository author: Karl.
|
|
||||||
@@ -42,7 +42,7 @@ Use stable Rust and Node.js 24+. Before Windows VM tests or launch:
|
|||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
$env:SEARCHI_FORCE_CPU = '1'
|
$env:SEARCHI_FORCE_CPU = '1'
|
||||||
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu'
|
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
|
||||||
```
|
```
|
||||||
|
|
||||||
From the repository root:
|
From the repository root:
|
||||||
@@ -74,6 +74,9 @@ cargo test -p searchi-semantic --test gemma_cpu -- --ignored --nocapture
|
|||||||
Record actual outputs, hardware, corpus size, background preparation, first-input
|
Record actual outputs, hardware, corpus size, background preparation, first-input
|
||||||
latency and warm latency in the relevant validation report. Unexecuted platform
|
latency and warm latency in the relevant validation report. Unexecuted platform
|
||||||
checks stay open. Compilation alone is not UI interaction verification.
|
checks stay open. Compilation alone is not UI interaction verification.
|
||||||
|
Use the opt-in measured search APIs and docs/PERFORMANCE.md for granular traces.
|
||||||
|
Keep basic and debug overhead separate; do not sum nested stages or parallel
|
||||||
|
lanes, or label DOM updates as actual paint. Retain raw measurement samples.
|
||||||
|
|
||||||
## Commit and hand off
|
## Commit and hand off
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,24 @@
|
|||||||
# Searchi desktop
|
# Searchi desktop
|
||||||
|
|
||||||
This Tauri 2 shell calls the same Rust core as the CLI and the real Google EmbeddingGemma 2 service. It provides folder indexing, native folder selection, progressive filename/content/semantic search, folder/extension/size/date filters, keyboard navigation, escaped text previews, native open/reveal actions, persisted selected folders/resource settings, and a global shortcut. There are no mocked results or embeddings. Watchers and release installers remain roadmap work.
|
This Tauri 2 shell calls the same Rust core as the CLI and the real Google EmbeddingGemma 2 service. It provides folder indexing, native folder selection, progressive filename/content/semantic search, folder/extension/size/date/media-duration filters, keyboard navigation, escaped text and scoped image/audio/video previews, native open/reveal actions, persisted selected folders/resource settings, and a global shortcut. There are no mocked results or embeddings. Watchers and release installers remain roadmap work.
|
||||||
|
|
||||||
## Develop
|
## Develop
|
||||||
|
|
||||||
Install Node.js LTS, Rust stable, and the platform prerequisites documented at https://v2.tauri.app/start/prerequisites/. Windows requires MSVC build tools and WebView2; Linux requires WebKitGTK and system development packages; macOS requires Xcode command line tools.
|
Install Node.js 24 LTS, Rust stable, and the platform prerequisites documented at https://v2.tauri.app/start/prerequisites/. Windows requires MSVC build tools and WebView2; Linux requires WebKitGTK and system development packages; macOS requires Xcode command line tools.
|
||||||
|
|
||||||
|
Before launching from a checkout, build and stage both native helpers from the
|
||||||
|
repository root. These explicit setup commands access the network and verify
|
||||||
|
pinned artifacts; they do not install tools globally:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
||||||
|
python tools/embeddinggemma2/stage_runtime.py --backend cpu
|
||||||
|
python tools/media-runtime/provision.py --stage
|
||||||
|
```
|
||||||
|
|
||||||
|
The decoder manifest currently has Windows/Linux x86-64 and ARM64 artifacts;
|
||||||
|
a vetted macOS decoder package remains a release gate. Build `cpu` in the
|
||||||
|
Windows VM. Production acceleration requires separate platform/hardware QA.
|
||||||
```sh
|
```sh
|
||||||
cd apps/desktop
|
cd apps/desktop
|
||||||
npm ci
|
npm ci
|
||||||
@@ -25,7 +38,7 @@ The desktop index is stored in the platform app-data directory under `dev.search
|
|||||||
|
|
||||||
On first launch, the native app downloads and verifies the pinned Google model artifacts (310 MB text model plus 555 MB media projector, approximately 865 MB total) into app-managed storage and prepares the packaged embedding worker. No model paths, runtime paths, or terminal setup are required. The UI shows native download bytes, verification/warming states, the actual worker device, and explicit errors with a retry control. The first download requires network access; file extraction, embeddings, and queries stay local.
|
On first launch, the native app downloads and verifies the pinned Google model artifacts (310 MB text model plus 555 MB media projector, approximately 865 MB total) into app-managed storage and prepares the packaged embedding worker. No model paths, runtime paths, or terminal setup are required. The UI shows native download bytes, verification/warming states, the actual worker device, and explicit errors with a retry control. The first download requires network access; file extraction, embeddings, and queries stay local.
|
||||||
|
|
||||||
After a folder is indexed, semantic indexing starts automatically when the model is ready. Existing keyword results stay searchable during setup/indexing. Progress is indeterminate during embedding because the native command does not expose a chunk completion count. Failed automatic builds retry only after a changed index/status revision or an explicit rebuild, avoiding repeated failure loops. Status polls run once per second while setup/indexing/warming is active and stop when idle.
|
After a folder is indexed, unified document/media semantic indexing starts automatically when the model and packaged media helpers are ready. Images use actual pixels; video frames and full primary-stream audio windows use the Google media projector. Audio/music files also retain artist/title metadata for keyword retrieval. The UI reports unavailable decoder/runtime support and per-file failures while keeping text search available. Existing keyword results stay searchable during setup/indexing. Progress is indeterminate during embedding because the native command does not expose a chunk completion count. Failed automatic builds retry only after a changed index/status revision or an explicit rebuild, avoiding repeated failure loops. Status polls run once per second while setup/indexing/warming is active and stop when idle.
|
||||||
|
|
||||||
Each query captures one immutable filter snapshot. Keyword results appear first; real hybrid results follow after the query settles. `src/search-state.ts` bounds each lane to one active native request and one latest pending request, discards old revisions, and retries a busy request at most once. Escape clears both pending lanes and their timers; already-running native work finishes without changing cleared results. Hybrid updates preserve the selected file and keyboard focus where possible.
|
Each query captures one immutable filter snapshot. Keyword results appear first; real hybrid results follow after the query settles. `src/search-state.ts` bounds each lane to one active native request and one latest pending request, discards old revisions, and retries a busy request at most once. Escape clears both pending lanes and their timers; already-running native work finishes without changing cleared results. Hybrid updates preserve the selected file and keyboard focus where possible.
|
||||||
|
|
||||||
@@ -38,7 +51,7 @@ node --test tests/search-state.test.mjs
|
|||||||
For Windows VM UI testing, disable WebView2 GPU rendering as well, before starting Tauri in PowerShell:
|
For Windows VM UI testing, disable WebView2 GPU rendering as well, before starting Tauri in PowerShell:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu'
|
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
|
||||||
$env:SEARCHI_FORCE_CPU = '1'
|
$env:SEARCHI_FORCE_CPU = '1'
|
||||||
npm.cmd run tauri dev
|
npm.cmd run tauri dev
|
||||||
```
|
```
|
||||||
@@ -56,3 +69,38 @@ Visual fidelity remains subject to native screenshot QA. Compilation and state t
|
|||||||
## Verification and packaging
|
## Verification and packaging
|
||||||
|
|
||||||
`npm run build` checks strict TypeScript and creates production frontend assets. `npm run tauri build -- --no-bundle` compiles the native shell when platform prerequisites are installed. Installer bundling is intentionally disabled until release icons, signing and native platform checks are complete. Native Windows, macOS, and Linux checks must be recorded separately; a frontend build does not prove the native shell works.
|
`npm run build` checks strict TypeScript and creates production frontend assets. `npm run tauri build -- --no-bundle` compiles the native shell when platform prerequisites are installed. Installer bundling is intentionally disabled until release icons, signing and native platform checks are complete. Native Windows, macOS, and Linux checks must be recorded separately; a frontend build does not prove the native shell works.
|
||||||
|
|
||||||
|
## Media preview and quality
|
||||||
|
|
||||||
|
The result pane renders images and audio/video controls through Tauri's scoped
|
||||||
|
asset protocol. Native preparation verifies the selected path belongs to a
|
||||||
|
current indexed file before exposing it. There is no general filesystem access
|
||||||
|
or local HTTP server. Players use metadata preload, never autoplay, and seek
|
||||||
|
matched millisecond timestamps after metadata loads. Selection changes and
|
||||||
|
Escape pause/unload playback; stale preparation responses cannot replace the
|
||||||
|
current preview. Unsupported WebView codecs show an Open file fallback.
|
||||||
|
|
||||||
|
Opening the default application does not guarantee exact-location navigation.
|
||||||
|
Preview seeking uses the returned window/frame start, which may differ from the
|
||||||
|
actual utterance. The [independent media quality report](../../docs/MEDIA-QUALITY.md)
|
||||||
|
records real file recall, speech alignment failures, music hard negatives and
|
||||||
|
unmet no-answer handling. Genuine multimodal support is implemented; production
|
||||||
|
quality and native playback/visual interaction QA remain open gates.
|
||||||
|
|
||||||
|
## Performance details
|
||||||
|
|
||||||
|
Normal results show the actual native search duration and latest result-display
|
||||||
|
latency from input. The **Performance details** checkbox in Settings is opt-in
|
||||||
|
and captured per request. Its expandable panel separates input debounce,
|
||||||
|
intentional semantic delay, pending-lane queue, failed attempts and bounded retry
|
||||||
|
wait, invocation time, native stage hierarchy and synchronous result DOM updates.
|
||||||
|
First response and hybrid-final latency are distinct; keyword-only results do not
|
||||||
|
claim semantic completion. Native spans use a command-relative clock and nested
|
||||||
|
or parallel spans must not be summed. Invocation minus native total is explicitly
|
||||||
|
an estimated transport/serialization/scheduling residual, not an exact IPC cost.
|
||||||
|
DOM update timings do not claim actual screen paint or deferred media loading.
|
||||||
|
|
||||||
|
The local clipboard report excludes queries, paths, filenames, snippets and error
|
||||||
|
strings. It captures available model/device/thread/runtime context without
|
||||||
|
guessing missing values. Histories are bounded and dropped counts remain visible.
|
||||||
|
Clipboard failure is explicit; no diagnostic telemetry is uploaded.
|
||||||
@@ -412,6 +412,12 @@ version = "1.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "byteorder-lite"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bytes"
|
name = "bytes"
|
||||||
version = "1.12.1"
|
version = "1.12.1"
|
||||||
@@ -1845,6 +1851,12 @@ dependencies = [
|
|||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "http-range"
|
||||||
|
version = "0.1.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "httparse"
|
name = "httparse"
|
||||||
version = "1.10.1"
|
version = "1.10.1"
|
||||||
@@ -2063,6 +2075,32 @@ dependencies = [
|
|||||||
"icu_properties",
|
"icu_properties",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "image"
|
||||||
|
version = "0.25.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||||
|
dependencies = [
|
||||||
|
"bytemuck",
|
||||||
|
"byteorder-lite",
|
||||||
|
"image-webp",
|
||||||
|
"moxcms",
|
||||||
|
"num-traits",
|
||||||
|
"png 0.18.1",
|
||||||
|
"zune-core",
|
||||||
|
"zune-jpeg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "image-webp"
|
||||||
|
version = "0.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder-lite",
|
||||||
|
"quick-error",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "indexmap"
|
name = "indexmap"
|
||||||
version = "1.9.3"
|
version = "1.9.3"
|
||||||
@@ -2257,6 +2295,12 @@ dependencies = [
|
|||||||
"syn 2.0.119",
|
"syn 2.0.119",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "jpeg-decoder"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "js-sys"
|
name = "js-sys"
|
||||||
version = "0.3.106"
|
version = "0.3.106"
|
||||||
@@ -2522,6 +2566,16 @@ dependencies = [
|
|||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "moxcms"
|
||||||
|
version = "0.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
"pxfm",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "muda"
|
name = "muda"
|
||||||
version = "0.20.0"
|
version = "0.20.0"
|
||||||
@@ -3157,6 +3211,18 @@ dependencies = [
|
|||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pxfm"
|
||||||
|
version = "0.1.30"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quick-error"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quick-xml"
|
name = "quick-xml"
|
||||||
version = "0.42.0"
|
version = "0.42.0"
|
||||||
@@ -3652,8 +3718,11 @@ dependencies = [
|
|||||||
name = "searchi-desktop"
|
name = "searchi-desktop"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"image",
|
||||||
"searchi-assets",
|
"searchi-assets",
|
||||||
"searchi-core",
|
"searchi-core",
|
||||||
|
"searchi-media",
|
||||||
"searchi-service",
|
"searchi-service",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -3665,11 +3734,26 @@ dependencies = [
|
|||||||
"tempfile",
|
"tempfile",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "searchi-media"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"image",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"sha2 0.10.9",
|
||||||
|
"tempfile",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "searchi-semantic"
|
name = "searchi-semantic"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"base64 0.22.1",
|
||||||
|
"image",
|
||||||
|
"jpeg-decoder",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2 0.10.9",
|
"sha2 0.10.9",
|
||||||
@@ -3683,6 +3767,7 @@ dependencies = [
|
|||||||
"fs2",
|
"fs2",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"searchi-core",
|
"searchi-core",
|
||||||
|
"searchi-media",
|
||||||
"searchi-semantic",
|
"searchi-semantic",
|
||||||
"serde",
|
"serde",
|
||||||
"sha2 0.10.9",
|
"sha2 0.10.9",
|
||||||
@@ -4218,6 +4303,7 @@ dependencies = [
|
|||||||
"gtk",
|
"gtk",
|
||||||
"heck 0.5.0",
|
"heck 0.5.0",
|
||||||
"http",
|
"http",
|
||||||
|
"http-range",
|
||||||
"jni",
|
"jni",
|
||||||
"libc",
|
"libc",
|
||||||
"log",
|
"log",
|
||||||
@@ -5993,6 +6079,21 @@ dependencies = [
|
|||||||
"simd-adler32",
|
"simd-adler32",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-core"
|
||||||
|
version = "0.5.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-jpeg"
|
||||||
|
version = "0.5.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
|
||||||
|
dependencies = [
|
||||||
|
"zune-core",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zvariant"
|
name = "zvariant"
|
||||||
version = "5.15.0"
|
version = "5.15.0"
|
||||||
|
|||||||
@@ -14,7 +14,8 @@ custom-protocol = ["tauri/custom-protocol"]
|
|||||||
tauri-build = { version = "2", features = [] }
|
tauri-build = { version = "2", features = [] }
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
tauri = { version = "2", features = [] }
|
anyhow = "1"
|
||||||
|
tauri = { version = "2", features = ["protocol-asset"] }
|
||||||
tauri-plugin-dialog = "2"
|
tauri-plugin-dialog = "2"
|
||||||
tauri-plugin-opener = "2"
|
tauri-plugin-opener = "2"
|
||||||
tauri-plugin-global-shortcut = "2"
|
tauri-plugin-global-shortcut = "2"
|
||||||
@@ -24,3 +25,5 @@ tempfile = "3"
|
|||||||
searchi-core = { path = "../../../crates/searchi-core" }
|
searchi-core = { path = "../../../crates/searchi-core" }
|
||||||
searchi-service = { path = "../../../crates/searchi-service" }
|
searchi-service = { path = "../../../crates/searchi-service" }
|
||||||
searchi-assets = { path = "../../../crates/searchi-assets" }
|
searchi-assets = { path = "../../../crates/searchi-assets" }
|
||||||
|
searchi-media = { path = "../../../crates/searchi-media" }
|
||||||
|
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp", "bmp"] }
|
||||||
@@ -0,0 +1,165 @@
|
|||||||
|
GNU LESSER GENERAL PUBLIC LICENSE
|
||||||
|
Version 3, 29 June 2007
|
||||||
|
|
||||||
|
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||||
|
Everyone is permitted to copy and distribute verbatim copies
|
||||||
|
of this license document, but changing it is not allowed.
|
||||||
|
|
||||||
|
|
||||||
|
This version of the GNU Lesser General Public License incorporates
|
||||||
|
the terms and conditions of version 3 of the GNU General Public
|
||||||
|
License, supplemented by the additional permissions listed below.
|
||||||
|
|
||||||
|
0. Additional Definitions.
|
||||||
|
|
||||||
|
As used herein, "this License" refers to version 3 of the GNU Lesser
|
||||||
|
General Public License, and the "GNU GPL" refers to version 3 of the GNU
|
||||||
|
General Public License.
|
||||||
|
|
||||||
|
"The Library" refers to a covered work governed by this License,
|
||||||
|
other than an Application or a Combined Work as defined below.
|
||||||
|
|
||||||
|
An "Application" is any work that makes use of an interface provided
|
||||||
|
by the Library, but which is not otherwise based on the Library.
|
||||||
|
Defining a subclass of a class defined by the Library is deemed a mode
|
||||||
|
of using an interface provided by the Library.
|
||||||
|
|
||||||
|
A "Combined Work" is a work produced by combining or linking an
|
||||||
|
Application with the Library. The particular version of the Library
|
||||||
|
with which the Combined Work was made is also called the "Linked
|
||||||
|
Version".
|
||||||
|
|
||||||
|
The "Minimal Corresponding Source" for a Combined Work means the
|
||||||
|
Corresponding Source for the Combined Work, excluding any source code
|
||||||
|
for portions of the Combined Work that, considered in isolation, are
|
||||||
|
based on the Application, and not on the Linked Version.
|
||||||
|
|
||||||
|
The "Corresponding Application Code" for a Combined Work means the
|
||||||
|
object code and/or source code for the Application, including any data
|
||||||
|
and utility programs needed for reproducing the Combined Work from the
|
||||||
|
Application, but excluding the System Libraries of the Combined Work.
|
||||||
|
|
||||||
|
1. Exception to Section 3 of the GNU GPL.
|
||||||
|
|
||||||
|
You may convey a covered work under sections 3 and 4 of this License
|
||||||
|
without being bound by section 3 of the GNU GPL.
|
||||||
|
|
||||||
|
2. Conveying Modified Versions.
|
||||||
|
|
||||||
|
If you modify a copy of the Library, and, in your modifications, a
|
||||||
|
facility refers to a function or data to be supplied by an Application
|
||||||
|
that uses the facility (other than as an argument passed when the
|
||||||
|
facility is invoked), then you may convey a copy of the modified
|
||||||
|
version:
|
||||||
|
|
||||||
|
a) under this License, provided that you make a good faith effort to
|
||||||
|
ensure that, in the event an Application does not supply the
|
||||||
|
function or data, the facility still operates, and performs
|
||||||
|
whatever part of its purpose remains meaningful, or
|
||||||
|
|
||||||
|
b) under the GNU GPL, with none of the additional permissions of
|
||||||
|
this License applicable to that copy.
|
||||||
|
|
||||||
|
3. Object Code Incorporating Material from Library Header Files.
|
||||||
|
|
||||||
|
The object code form of an Application may incorporate material from
|
||||||
|
a header file that is part of the Library. You may convey such object
|
||||||
|
code under terms of your choice, provided that, if the incorporated
|
||||||
|
material is not limited to numerical parameters, data structure
|
||||||
|
layouts and accessors, or small macros, inline functions and templates
|
||||||
|
(ten or fewer lines in length), you do both of the following:
|
||||||
|
|
||||||
|
a) Give prominent notice with each copy of the object code that the
|
||||||
|
Library is used in it and that the Library and its use are
|
||||||
|
covered by this License.
|
||||||
|
|
||||||
|
b) Accompany the object code with a copy of the GNU GPL and this license
|
||||||
|
document.
|
||||||
|
|
||||||
|
4. Combined Works.
|
||||||
|
|
||||||
|
You may convey a Combined Work under terms of your choice that,
|
||||||
|
taken together, effectively do not restrict modification of the
|
||||||
|
portions of the Library contained in the Combined Work and reverse
|
||||||
|
engineering for debugging such modifications, if you also do each of
|
||||||
|
the following:
|
||||||
|
|
||||||
|
a) Give prominent notice with each copy of the Combined Work that
|
||||||
|
the Library is used in it and that the Library and its use are
|
||||||
|
covered by this License.
|
||||||
|
|
||||||
|
b) Accompany the Combined Work with a copy of the GNU GPL and this license
|
||||||
|
document.
|
||||||
|
|
||||||
|
c) For a Combined Work that displays copyright notices during
|
||||||
|
execution, include the copyright notice for the Library among
|
||||||
|
these notices, as well as a reference directing the user to the
|
||||||
|
copies of the GNU GPL and this license document.
|
||||||
|
|
||||||
|
d) Do one of the following:
|
||||||
|
|
||||||
|
0) Convey the Minimal Corresponding Source under the terms of this
|
||||||
|
License, and the Corresponding Application Code in a form
|
||||||
|
suitable for, and under terms that permit, the user to
|
||||||
|
recombine or relink the Application with a modified version of
|
||||||
|
the Linked Version to produce a modified Combined Work, in the
|
||||||
|
manner specified by section 6 of the GNU GPL for conveying
|
||||||
|
Corresponding Source.
|
||||||
|
|
||||||
|
1) Use a suitable shared library mechanism for linking with the
|
||||||
|
Library. A suitable mechanism is one that (a) uses at run time
|
||||||
|
a copy of the Library already present on the user's computer
|
||||||
|
system, and (b) will operate properly with a modified version
|
||||||
|
of the Library that is interface-compatible with the Linked
|
||||||
|
Version.
|
||||||
|
|
||||||
|
e) Provide Installation Information, but only if you would otherwise
|
||||||
|
be required to provide such information under section 6 of the
|
||||||
|
GNU GPL, and only to the extent that such information is
|
||||||
|
necessary to install and execute a modified version of the
|
||||||
|
Combined Work produced by recombining or relinking the
|
||||||
|
Application with a modified version of the Linked Version. (If
|
||||||
|
you use option 4d0, the Installation Information must accompany
|
||||||
|
the Minimal Corresponding Source and Corresponding Application
|
||||||
|
Code. If you use option 4d1, you must provide the Installation
|
||||||
|
Information in the manner specified by section 6 of the GNU GPL
|
||||||
|
for conveying Corresponding Source.)
|
||||||
|
|
||||||
|
5. Combined Libraries.
|
||||||
|
|
||||||
|
You may place library facilities that are a work based on the
|
||||||
|
Library side by side in a single library together with other library
|
||||||
|
facilities that are not Applications and are not covered by this
|
||||||
|
License, and convey such a combined library under terms of your
|
||||||
|
choice, if you do both of the following:
|
||||||
|
|
||||||
|
a) Accompany the combined library with a copy of the same work based
|
||||||
|
on the Library, uncombined with any other library facilities,
|
||||||
|
conveyed under the terms of this License.
|
||||||
|
|
||||||
|
b) Give prominent notice with the combined library that part of it
|
||||||
|
is a work based on the Library, and explaining where to find the
|
||||||
|
accompanying uncombined form of the same work.
|
||||||
|
|
||||||
|
6. Revised Versions of the GNU Lesser General Public License.
|
||||||
|
|
||||||
|
The Free Software Foundation may publish revised and/or new versions
|
||||||
|
of the GNU Lesser General Public License from time to time. Such new
|
||||||
|
versions will be similar in spirit to the present version, but may
|
||||||
|
differ in detail to address new problems or concerns.
|
||||||
|
|
||||||
|
Each version is given a distinguishing version number. If the
|
||||||
|
Library as you received it specifies that a certain numbered version
|
||||||
|
of the GNU Lesser General Public License "or any later version"
|
||||||
|
applies to it, you have the option of following the terms and
|
||||||
|
conditions either of that published version or of any later version
|
||||||
|
published by the Free Software Foundation. If the Library as you
|
||||||
|
received it does not specify a version number of the GNU Lesser
|
||||||
|
General Public License, you may choose any version of the GNU Lesser
|
||||||
|
General Public License ever published by the Free Software Foundation.
|
||||||
|
|
||||||
|
If the Library as you received it specifies that a proxy can decide
|
||||||
|
whether future versions of the GNU Lesser General Public License shall
|
||||||
|
apply, that proxy's public statement of acceptance of any version is
|
||||||
|
permanent authorization for you to choose that version for the
|
||||||
|
Library.
|
||||||
@@ -0,0 +1,165 @@
|
|||||||
|
GNU LESSER GENERAL PUBLIC LICENSE
|
||||||
|
Version 3, 29 June 2007
|
||||||
|
|
||||||
|
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||||
|
Everyone is permitted to copy and distribute verbatim copies
|
||||||
|
of this license document, but changing it is not allowed.
|
||||||
|
|
||||||
|
|
||||||
|
This version of the GNU Lesser General Public License incorporates
|
||||||
|
the terms and conditions of version 3 of the GNU General Public
|
||||||
|
License, supplemented by the additional permissions listed below.
|
||||||
|
|
||||||
|
0. Additional Definitions.
|
||||||
|
|
||||||
|
As used herein, "this License" refers to version 3 of the GNU Lesser
|
||||||
|
General Public License, and the "GNU GPL" refers to version 3 of the GNU
|
||||||
|
General Public License.
|
||||||
|
|
||||||
|
"The Library" refers to a covered work governed by this License,
|
||||||
|
other than an Application or a Combined Work as defined below.
|
||||||
|
|
||||||
|
An "Application" is any work that makes use of an interface provided
|
||||||
|
by the Library, but which is not otherwise based on the Library.
|
||||||
|
Defining a subclass of a class defined by the Library is deemed a mode
|
||||||
|
of using an interface provided by the Library.
|
||||||
|
|
||||||
|
A "Combined Work" is a work produced by combining or linking an
|
||||||
|
Application with the Library. The particular version of the Library
|
||||||
|
with which the Combined Work was made is also called the "Linked
|
||||||
|
Version".
|
||||||
|
|
||||||
|
The "Minimal Corresponding Source" for a Combined Work means the
|
||||||
|
Corresponding Source for the Combined Work, excluding any source code
|
||||||
|
for portions of the Combined Work that, considered in isolation, are
|
||||||
|
based on the Application, and not on the Linked Version.
|
||||||
|
|
||||||
|
The "Corresponding Application Code" for a Combined Work means the
|
||||||
|
object code and/or source code for the Application, including any data
|
||||||
|
and utility programs needed for reproducing the Combined Work from the
|
||||||
|
Application, but excluding the System Libraries of the Combined Work.
|
||||||
|
|
||||||
|
1. Exception to Section 3 of the GNU GPL.
|
||||||
|
|
||||||
|
You may convey a covered work under sections 3 and 4 of this License
|
||||||
|
without being bound by section 3 of the GNU GPL.
|
||||||
|
|
||||||
|
2. Conveying Modified Versions.
|
||||||
|
|
||||||
|
If you modify a copy of the Library, and, in your modifications, a
|
||||||
|
facility refers to a function or data to be supplied by an Application
|
||||||
|
that uses the facility (other than as an argument passed when the
|
||||||
|
facility is invoked), then you may convey a copy of the modified
|
||||||
|
version:
|
||||||
|
|
||||||
|
a) under this License, provided that you make a good faith effort to
|
||||||
|
ensure that, in the event an Application does not supply the
|
||||||
|
function or data, the facility still operates, and performs
|
||||||
|
whatever part of its purpose remains meaningful, or
|
||||||
|
|
||||||
|
b) under the GNU GPL, with none of the additional permissions of
|
||||||
|
this License applicable to that copy.
|
||||||
|
|
||||||
|
3. Object Code Incorporating Material from Library Header Files.
|
||||||
|
|
||||||
|
The object code form of an Application may incorporate material from
|
||||||
|
a header file that is part of the Library. You may convey such object
|
||||||
|
code under terms of your choice, provided that, if the incorporated
|
||||||
|
material is not limited to numerical parameters, data structure
|
||||||
|
layouts and accessors, or small macros, inline functions and templates
|
||||||
|
(ten or fewer lines in length), you do both of the following:
|
||||||
|
|
||||||
|
a) Give prominent notice with each copy of the object code that the
|
||||||
|
Library is used in it and that the Library and its use are
|
||||||
|
covered by this License.
|
||||||
|
|
||||||
|
b) Accompany the object code with a copy of the GNU GPL and this license
|
||||||
|
document.
|
||||||
|
|
||||||
|
4. Combined Works.
|
||||||
|
|
||||||
|
You may convey a Combined Work under terms of your choice that,
|
||||||
|
taken together, effectively do not restrict modification of the
|
||||||
|
portions of the Library contained in the Combined Work and reverse
|
||||||
|
engineering for debugging such modifications, if you also do each of
|
||||||
|
the following:
|
||||||
|
|
||||||
|
a) Give prominent notice with each copy of the Combined Work that
|
||||||
|
the Library is used in it and that the Library and its use are
|
||||||
|
covered by this License.
|
||||||
|
|
||||||
|
b) Accompany the Combined Work with a copy of the GNU GPL and this license
|
||||||
|
document.
|
||||||
|
|
||||||
|
c) For a Combined Work that displays copyright notices during
|
||||||
|
execution, include the copyright notice for the Library among
|
||||||
|
these notices, as well as a reference directing the user to the
|
||||||
|
copies of the GNU GPL and this license document.
|
||||||
|
|
||||||
|
d) Do one of the following:
|
||||||
|
|
||||||
|
0) Convey the Minimal Corresponding Source under the terms of this
|
||||||
|
License, and the Corresponding Application Code in a form
|
||||||
|
suitable for, and under terms that permit, the user to
|
||||||
|
recombine or relink the Application with a modified version of
|
||||||
|
the Linked Version to produce a modified Combined Work, in the
|
||||||
|
manner specified by section 6 of the GNU GPL for conveying
|
||||||
|
Corresponding Source.
|
||||||
|
|
||||||
|
1) Use a suitable shared library mechanism for linking with the
|
||||||
|
Library. A suitable mechanism is one that (a) uses at run time
|
||||||
|
a copy of the Library already present on the user's computer
|
||||||
|
system, and (b) will operate properly with a modified version
|
||||||
|
of the Library that is interface-compatible with the Linked
|
||||||
|
Version.
|
||||||
|
|
||||||
|
e) Provide Installation Information, but only if you would otherwise
|
||||||
|
be required to provide such information under section 6 of the
|
||||||
|
GNU GPL, and only to the extent that such information is
|
||||||
|
necessary to install and execute a modified version of the
|
||||||
|
Combined Work produced by recombining or relinking the
|
||||||
|
Application with a modified version of the Linked Version. (If
|
||||||
|
you use option 4d0, the Installation Information must accompany
|
||||||
|
the Minimal Corresponding Source and Corresponding Application
|
||||||
|
Code. If you use option 4d1, you must provide the Installation
|
||||||
|
Information in the manner specified by section 6 of the GNU GPL
|
||||||
|
for conveying Corresponding Source.)
|
||||||
|
|
||||||
|
5. Combined Libraries.
|
||||||
|
|
||||||
|
You may place library facilities that are a work based on the
|
||||||
|
Library side by side in a single library together with other library
|
||||||
|
facilities that are not Applications and are not covered by this
|
||||||
|
License, and convey such a combined library under terms of your
|
||||||
|
choice, if you do both of the following:
|
||||||
|
|
||||||
|
a) Accompany the combined library with a copy of the same work based
|
||||||
|
on the Library, uncombined with any other library facilities,
|
||||||
|
conveyed under the terms of this License.
|
||||||
|
|
||||||
|
b) Give prominent notice with the combined library that part of it
|
||||||
|
is a work based on the Library, and explaining where to find the
|
||||||
|
accompanying uncombined form of the same work.
|
||||||
|
|
||||||
|
6. Revised Versions of the GNU Lesser General Public License.
|
||||||
|
|
||||||
|
The Free Software Foundation may publish revised and/or new versions
|
||||||
|
of the GNU Lesser General Public License from time to time. Such new
|
||||||
|
versions will be similar in spirit to the present version, but may
|
||||||
|
differ in detail to address new problems or concerns.
|
||||||
|
|
||||||
|
Each version is given a distinguishing version number. If the
|
||||||
|
Library as you received it specifies that a certain numbered version
|
||||||
|
of the GNU Lesser General Public License "or any later version"
|
||||||
|
applies to it, you have the option of following the terms and
|
||||||
|
conditions either of that published version or of any later version
|
||||||
|
published by the Free Software Foundation. If the Library as you
|
||||||
|
received it does not specify a version number of the GNU Lesser
|
||||||
|
General Public License, you may choose any version of the GNU Lesser
|
||||||
|
General Public License ever published by the Free Software Foundation.
|
||||||
|
|
||||||
|
If the Library as you received it specifies that a proxy can decide
|
||||||
|
whether future versions of the GNU Lesser General Public License shall
|
||||||
|
apply, that proxy's public statement of acceptance of any version is
|
||||||
|
permanent authorization for you to choose that version for the
|
||||||
|
Library.
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"provider": "BtbN/FFmpeg-Builds",
|
||||||
|
"release": "autobuild-2026-10-12-13-12",
|
||||||
|
"version": "9.0.2-27-g7e93014c5b",
|
||||||
|
"platforms": {
|
||||||
|
"windows-x86_64": {"archive": "ffmpeg-n9.0.2-27-g7e93014c5b-win64-lgpl-9.0.zip", "bytes": 171476229, "sha256": "104a019fe34c0747b208bde4842a0bac8b70a307cc2bb33aa4648fc15ddb2e39"},
|
||||||
|
"windows-aarch64": {"archive": "ffmpeg-n9.0.2-27-g7e93014c5b-winarm64-lgpl-9.0.zip", "bytes": 118006941, "sha256": "68d3bac585b80b6f17e459eb4a203ed1f01b7635d4c611f1685de1ae65ca0987"},
|
||||||
|
"linux-x86_64": {"archive": "ffmpeg-n9.0.2-27-g7e93014c5b-linux64-lgpl-9.0.tar.xz", "bytes": 137901812, "sha256": "12445b38a4700ba1ae7c9c7e57062f602c927dc4187555bfadc9f3e2f75f05b0"},
|
||||||
|
"linux-aarch64": {"archive": "ffmpeg-n9.0.2-27-g7e93014c5b-linuxarm64-lgpl-9.0.tar.xz", "bytes": 117008524, "sha256": "c9160ae8440c17c016fb2f6b3fcce4bb9d633a15845e908b2303948a7724f227"}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,9 +4,11 @@ mod model_download;
|
|||||||
mod semantic_commands;
|
mod semantic_commands;
|
||||||
mod settings;
|
mod settings;
|
||||||
|
|
||||||
|
use searchi_core::metrics::{MetricsMode, SearchMetrics, SearchResponse, TraceCollector};
|
||||||
use searchi_core::{IndexReport, IndexStats, SearchIndex, SearchOptions, SearchResult};
|
use searchi_core::{IndexReport, IndexStats, SearchIndex, SearchOptions, SearchResult};
|
||||||
use semantic_commands::{
|
use semantic_commands::{
|
||||||
hybrid_search, index_embeddings, semantic_status, setup_model, SemanticState,
|
hybrid_search, hybrid_search_measured, index_embeddings, semantic_status, setup_model,
|
||||||
|
SemanticState,
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use settings::{Settings, SettingsStore};
|
use settings::{Settings, SettingsStore};
|
||||||
@@ -31,6 +33,46 @@ struct AppState {
|
|||||||
semantic: Arc<SemanticState>,
|
semantic: Arc<SemanticState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct MeasuredFailure {
|
||||||
|
message: String,
|
||||||
|
metrics: SearchMetrics,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MeasuredFailure {
|
||||||
|
fn new(message: impl Into<String>, trace: TraceCollector) -> Self {
|
||||||
|
Self {
|
||||||
|
message: message.into(),
|
||||||
|
metrics: trace.finish(0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn task(message: impl Into<String>, started: std::time::Instant) -> Self {
|
||||||
|
Self {
|
||||||
|
message: message.into(),
|
||||||
|
metrics: SearchMetrics {
|
||||||
|
total_ms: started.elapsed().as_secs_f64() * 1000.0,
|
||||||
|
result_count: 0,
|
||||||
|
trace: None,
|
||||||
|
dropped_spans: 1,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn measured_outcome(
|
||||||
|
result: Result<Vec<SearchResult>, String>,
|
||||||
|
trace: TraceCollector,
|
||||||
|
) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
|
||||||
|
match result {
|
||||||
|
Ok(results) => Ok(SearchResponse {
|
||||||
|
metrics: trace.finish(results.len()),
|
||||||
|
results,
|
||||||
|
}),
|
||||||
|
Err(message) => Err(MeasuredFailure::new(message, trace)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
struct DesktopStatus {
|
struct DesktopStatus {
|
||||||
settings: Settings,
|
settings: Settings,
|
||||||
@@ -78,6 +120,8 @@ struct Filters {
|
|||||||
extension: Option<String>,
|
extension: Option<String>,
|
||||||
min_size: Option<u64>,
|
min_size: Option<u64>,
|
||||||
max_size: Option<u64>,
|
max_size: Option<u64>,
|
||||||
|
min_duration_ms: Option<u32>,
|
||||||
|
max_duration_ms: Option<u32>,
|
||||||
modified_after_unix_ms: Option<i64>,
|
modified_after_unix_ms: Option<i64>,
|
||||||
modified_before_unix_ms: Option<i64>,
|
modified_before_unix_ms: Option<i64>,
|
||||||
}
|
}
|
||||||
@@ -89,6 +133,8 @@ impl From<Filters> for SearchOptions {
|
|||||||
extension: value.extension,
|
extension: value.extension,
|
||||||
min_size: value.min_size,
|
min_size: value.min_size,
|
||||||
max_size: value.max_size,
|
max_size: value.max_size,
|
||||||
|
min_duration_ms: value.min_duration_ms,
|
||||||
|
max_duration_ms: value.max_duration_ms,
|
||||||
modified_after_unix_ms: value.modified_after_unix_ms,
|
modified_after_unix_ms: value.modified_after_unix_ms,
|
||||||
modified_before_unix_ms: value.modified_before_unix_ms,
|
modified_before_unix_ms: value.modified_before_unix_ms,
|
||||||
}
|
}
|
||||||
@@ -114,6 +160,46 @@ async fn search(
|
|||||||
.map_err(|e| e.to_string())?
|
.map_err(|e| e.to_string())?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Measured command preserves the legacy array-returning API for integrations.
|
||||||
|
#[tauri::command]
|
||||||
|
async fn search_measured(
|
||||||
|
query: String,
|
||||||
|
filters: Option<Filters>,
|
||||||
|
debug_performance: bool,
|
||||||
|
state: State<'_, AppState>,
|
||||||
|
) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
|
||||||
|
let started = std::time::Instant::now();
|
||||||
|
let mut trace = TraceCollector::new(if debug_performance {
|
||||||
|
MetricsMode::Debug
|
||||||
|
} else {
|
||||||
|
MetricsMode::Basic
|
||||||
|
});
|
||||||
|
let queue = trace.begin("desktop.worker_queue", None);
|
||||||
|
let lexical = state.lexical.clone();
|
||||||
|
tauri::async_runtime::spawn_blocking(move || {
|
||||||
|
trace.end(queue, Default::default());
|
||||||
|
let result = (|| {
|
||||||
|
let index = trace.timed("desktop.lexical_lock", None, |_, _| {
|
||||||
|
lexical
|
||||||
|
.try_lock()
|
||||||
|
.map_err(|_| "Keyword worker is busy; try the latest query again.".to_owned())
|
||||||
|
})?;
|
||||||
|
index
|
||||||
|
.search_with_options_traced(
|
||||||
|
&query,
|
||||||
|
50,
|
||||||
|
&filters.unwrap_or_default().into(),
|
||||||
|
&mut trace,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
})();
|
||||||
|
measured_outcome(result, trace)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| MeasuredFailure::task(error.to_string(), started))?
|
||||||
|
}
|
||||||
|
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
async fn pick_folder(app: tauri::AppHandle) -> Result<Option<String>, String> {
|
async fn pick_folder(app: tauri::AppHandle) -> Result<Option<String>, String> {
|
||||||
tauri::async_runtime::spawn_blocking(move || {
|
tauri::async_runtime::spawn_blocking(move || {
|
||||||
@@ -148,6 +234,68 @@ fn validated_result(index: &SearchIndex, path: &str) -> Result<PathBuf, String>
|
|||||||
Ok(canonical)
|
Ok(canonical)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct MediaPreview {
|
||||||
|
path: String,
|
||||||
|
kind: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
// The asset protocol starts with an empty scope. Only a validated indexed
|
||||||
|
// media file is granted access; the webview receives no general filesystem API.
|
||||||
|
#[tauri::command]
|
||||||
|
async fn prepare_media_preview(
|
||||||
|
path: String,
|
||||||
|
app: tauri::AppHandle,
|
||||||
|
state: State<'_, AppState>,
|
||||||
|
) -> Result<MediaPreview, String> {
|
||||||
|
let database = state.database.clone();
|
||||||
|
tauri::async_runtime::spawn_blocking(move || {
|
||||||
|
let index = SearchIndex::open(database).map_err(|error| error.to_string())?;
|
||||||
|
let canonical = validated_result(&index, &path)?;
|
||||||
|
let kind = match searchi_media::classify(&canonical) {
|
||||||
|
Some(searchi_media::MediaKind::Image) => "image",
|
||||||
|
Some(searchi_media::MediaKind::Audio) => "audio",
|
||||||
|
Some(searchi_media::MediaKind::Video) => "video",
|
||||||
|
None => return Err("This file is not a supported media preview.".into()),
|
||||||
|
};
|
||||||
|
if kind == "image" {
|
||||||
|
let bytes = std::fs::metadata(&canonical)
|
||||||
|
.map_err(|error| error.to_string())?
|
||||||
|
.len();
|
||||||
|
if bytes > 64 * 1024 * 1024 {
|
||||||
|
return Err("Image exceeds the 64 MiB preview limit.".into());
|
||||||
|
}
|
||||||
|
let (width, height) = image::ImageReader::open(&canonical)
|
||||||
|
.map_err(|error| error.to_string())?
|
||||||
|
.with_guessed_format()
|
||||||
|
.map_err(|error| error.to_string())?
|
||||||
|
.into_dimensions()
|
||||||
|
.map_err(|error| format!("Image preview is unavailable: {error}"))?;
|
||||||
|
let jpeg = canonical
|
||||||
|
.extension()
|
||||||
|
.and_then(|value| value.to_str())
|
||||||
|
.is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "jpg" | "jpeg"));
|
||||||
|
let max_pixels = if jpeg {
|
||||||
|
64 * 1024 * 1024
|
||||||
|
} else {
|
||||||
|
16 * 1024 * 1024
|
||||||
|
};
|
||||||
|
if width == 0 || height == 0 || u64::from(width) * u64::from(height) > max_pixels {
|
||||||
|
return Err("Image exceeds the supported preview dimensions.".into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app.asset_protocol_scope()
|
||||||
|
.allow_file(&canonical)
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
Ok(MediaPreview {
|
||||||
|
path: canonical.to_string_lossy().into_owned(),
|
||||||
|
kind,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())?
|
||||||
|
}
|
||||||
|
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
async fn open_result(
|
async fn open_result(
|
||||||
path: String,
|
path: String,
|
||||||
@@ -279,7 +427,24 @@ fn main() {
|
|||||||
app.path().resource_dir().unwrap_or_else(|_| directory.clone()).join(worker_name)
|
app.path().resource_dir().unwrap_or_else(|_| directory.clone()).join(worker_name)
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
let semantic = Arc::new(SemanticState::new(model_directory, worker));
|
let media_tools = (|| -> Result<searchi_media::FfmpegTools, String> {
|
||||||
|
if let Some(tools) = searchi_media::FfmpegTools::from_environment()
|
||||||
|
.map_err(|error| error.to_string())? {
|
||||||
|
return Ok(tools);
|
||||||
|
}
|
||||||
|
let development_media = development_root.join(".searchi/media-runtime/bin");
|
||||||
|
let media_directory = if cfg!(debug_assertions) && development_media.is_dir() {
|
||||||
|
development_media
|
||||||
|
} else {
|
||||||
|
app.path().resource_dir().map_err(|error| error.to_string())?.join("media-runtime/bin")
|
||||||
|
};
|
||||||
|
let ffmpeg = media_directory.join(if cfg!(windows) { "ffmpeg.exe" } else { "ffmpeg" });
|
||||||
|
let ffprobe = media_directory.join(if cfg!(windows) { "ffprobe.exe" } else { "ffprobe" });
|
||||||
|
searchi_media::FfmpegTools::new(ffmpeg, ffprobe).map_err(|_| {
|
||||||
|
"The media decoder is missing from this installation. Repair or reinstall searchi.".into()
|
||||||
|
})
|
||||||
|
})();
|
||||||
|
let semantic = Arc::new(SemanticState::new(model_directory, worker, media_tools));
|
||||||
semantic.begin_automatic_setup(database.clone(), settings.value.cpu_threads);
|
semantic.begin_automatic_setup(database.clone(), settings.value.cpu_threads);
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
let shortcut = "Shift+Super+Space";
|
let shortcut = "Shift+Super+Space";
|
||||||
@@ -307,16 +472,19 @@ fn main() {
|
|||||||
})
|
})
|
||||||
.invoke_handler(tauri::generate_handler![
|
.invoke_handler(tauri::generate_handler![
|
||||||
search,
|
search,
|
||||||
|
search_measured,
|
||||||
stats,
|
stats,
|
||||||
index_folder,
|
index_folder,
|
||||||
pick_folder,
|
pick_folder,
|
||||||
open_result,
|
open_result,
|
||||||
|
prepare_media_preview,
|
||||||
desktop_status,
|
desktop_status,
|
||||||
set_cpu_threads,
|
set_cpu_threads,
|
||||||
semantic_status,
|
semantic_status,
|
||||||
setup_model,
|
setup_model,
|
||||||
index_embeddings,
|
index_embeddings,
|
||||||
hybrid_search
|
hybrid_search,
|
||||||
|
hybrid_search_measured
|
||||||
])
|
])
|
||||||
.run(tauri::generate_context!())
|
.run(tauri::generate_context!())
|
||||||
.expect("Searchi desktop could not start");
|
.expect("Searchi desktop could not start");
|
||||||
@@ -326,6 +494,32 @@ fn main() {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn measured_failure_retains_closed_trace_without_inventing_results() {
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
let parent = trace.begin("desktop.request", None);
|
||||||
|
trace.begin("desktop.lexical_lock", parent);
|
||||||
|
let failed = measured_outcome(Err("Keyword worker is busy".into()), trace)
|
||||||
|
.err()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(failed.message, "Keyword worker is busy");
|
||||||
|
assert_eq!(failed.metrics.result_count, 0);
|
||||||
|
let spans = failed.metrics.trace.unwrap();
|
||||||
|
assert_eq!(spans.len(), 2);
|
||||||
|
assert_eq!(spans[1].parent, Some(0));
|
||||||
|
assert!(spans[1].start_ms + spans[1].duration_ms <= failed.metrics.total_ms + 0.001);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn measured_empty_success_is_distinct_from_failure_and_basic_has_no_trace() {
|
||||||
|
let response = measured_outcome(Ok(Vec::new()), TraceCollector::new(MetricsMode::Basic))
|
||||||
|
.ok()
|
||||||
|
.unwrap();
|
||||||
|
assert!(response.results.is_empty());
|
||||||
|
assert_eq!(response.metrics.result_count, 0);
|
||||||
|
assert!(response.metrics.trace.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn indexing_gate_is_released_when_worker_unwinds() {
|
fn indexing_gate_is_released_when_worker_unwinds() {
|
||||||
let indexing = Arc::new(AtomicBool::new(true));
|
let indexing = Arc::new(AtomicBool::new(true));
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
//! Desktop-owned orchestration: managed model setup and persistent hybrid queries.
|
//! Desktop-owned orchestration: managed model setup and persistent hybrid queries.
|
||||||
use crate::{model_download, AppState, Filters};
|
use crate::{measured_outcome, model_download, AppState, Filters, MeasuredFailure};
|
||||||
|
use searchi_core::metrics::{MetricsMode, SearchResponse, TraceCollector};
|
||||||
use searchi_core::SearchResult;
|
use searchi_core::SearchResult;
|
||||||
use searchi_service::{EmbeddingReport, HybridSearch};
|
use searchi_service::{EmbeddingReport, HybridSearch};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
@@ -24,10 +25,20 @@ pub struct SemanticState {
|
|||||||
downloaded_bytes: AtomicU64,
|
downloaded_bytes: AtomicU64,
|
||||||
error: Mutex<Option<String>>,
|
error: Mutex<Option<String>>,
|
||||||
service: Mutex<Option<HybridSearch>>,
|
service: Mutex<Option<HybridSearch>>,
|
||||||
|
media_tools: Option<searchi_media::FfmpegTools>,
|
||||||
|
media_setup_error: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SemanticState {
|
impl SemanticState {
|
||||||
pub fn new(model_directory: PathBuf, worker: PathBuf) -> Self {
|
pub fn new(
|
||||||
|
model_directory: PathBuf,
|
||||||
|
worker: PathBuf,
|
||||||
|
media_tools: Result<searchi_media::FfmpegTools, String>,
|
||||||
|
) -> Self {
|
||||||
|
let (media_tools, media_setup_error) = match media_tools {
|
||||||
|
Ok(tools) => (Some(tools), None),
|
||||||
|
Err(error) => (None, Some(error)),
|
||||||
|
};
|
||||||
Self {
|
Self {
|
||||||
model_directory,
|
model_directory,
|
||||||
worker,
|
worker,
|
||||||
@@ -40,6 +51,8 @@ impl SemanticState {
|
|||||||
downloaded_bytes: AtomicU64::new(0),
|
downloaded_bytes: AtomicU64::new(0),
|
||||||
error: Mutex::new(None),
|
error: Mutex::new(None),
|
||||||
service: Mutex::new(None),
|
service: Mutex::new(None),
|
||||||
|
media_tools,
|
||||||
|
media_setup_error,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,6 +131,9 @@ pub struct SemanticStatus {
|
|||||||
pending: usize,
|
pending: usize,
|
||||||
warming: bool,
|
warming: bool,
|
||||||
worker_info: Option<searchi_service::WorkerInfo>,
|
worker_info: Option<searchi_service::WorkerInfo>,
|
||||||
|
media_tools_ready: bool,
|
||||||
|
media_setup_error: Option<String>,
|
||||||
|
media_failures: Vec<searchi_service::MediaFailure>,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct WorkGuard<'a>(&'a AtomicBool);
|
struct WorkGuard<'a>(&'a AtomicBool);
|
||||||
@@ -158,6 +174,9 @@ pub fn semantic_status(state: State<'_, AppState>) -> Result<SemanticStatus, Str
|
|||||||
.try_lock()
|
.try_lock()
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|service| service.as_ref().and_then(HybridSearch::worker_info)),
|
.and_then(|service| service.as_ref().and_then(HybridSearch::worker_info)),
|
||||||
|
media_tools_ready: semantic.media_tools.is_some(),
|
||||||
|
media_setup_error: semantic.media_setup_error.clone(),
|
||||||
|
media_failures: index.failures,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -217,13 +236,42 @@ pub async fn index_embeddings(state: State<'_, AppState>) -> Result<EmbeddingRep
|
|||||||
semantic.record_error(None);
|
semantic.record_error(None);
|
||||||
// Indexing uses its own isolated worker; interactive queries keep their own session.
|
// Indexing uses its own isolated worker; interactive queries keep their own session.
|
||||||
let result = HybridSearch::open(
|
let result = HybridSearch::open(
|
||||||
database,
|
&database,
|
||||||
&semantic.model_directory,
|
&semantic.model_directory,
|
||||||
&semantic.worker,
|
&semantic.worker,
|
||||||
threads,
|
threads,
|
||||||
)
|
)
|
||||||
.and_then(|mut service| service.index_embeddings())
|
.and_then(|mut service| {
|
||||||
|
let tools = semantic.media_tools.as_ref().ok_or_else(|| {
|
||||||
|
anyhow::anyhow!("The media decoder is missing from this installation. Repair or reinstall searchi.")
|
||||||
|
})?;
|
||||||
|
service.index_embeddings_with_media(tools, &searchi_media::MediaLimits::default())
|
||||||
|
})
|
||||||
.map_err(|error| error.to_string());
|
.map_err(|error| error.to_string());
|
||||||
|
// Prepare the newly activated graph before advertising semantic readiness.
|
||||||
|
// Keyword search stays available while the cached query session catches up.
|
||||||
|
if result.is_ok() {
|
||||||
|
semantic.warmed.store(false, Ordering::Release);
|
||||||
|
semantic.warming.store(true, Ordering::Release);
|
||||||
|
let _warming_guard = WorkGuard(&semantic.warming);
|
||||||
|
let preparation = (|| {
|
||||||
|
let mut cached = semantic
|
||||||
|
.service
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "Semantic session is unavailable.".to_string())?;
|
||||||
|
let service = cached
|
||||||
|
.as_mut()
|
||||||
|
.ok_or_else(|| "Semantic query session has not been prepared.".to_string())?;
|
||||||
|
service.warmup().map_err(|error| error.to_string())
|
||||||
|
})();
|
||||||
|
semantic.warmed.store(preparation.is_ok(), Ordering::Release);
|
||||||
|
if let Err(error) = preparation {
|
||||||
|
semantic.record_error(Some(format!(
|
||||||
|
"The index was saved, but search preparation failed: {error}"
|
||||||
|
)));
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
semantic.record_error(result.as_ref().err().cloned());
|
semantic.record_error(result.as_ref().err().cloned());
|
||||||
result
|
result
|
||||||
})
|
})
|
||||||
@@ -283,3 +331,85 @@ pub async fn hybrid_search(
|
|||||||
.await
|
.await
|
||||||
.map_err(|error| error.to_string())?
|
.map_err(|error| error.to_string())?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tauri::command]
|
||||||
|
pub async fn hybrid_search_measured(
|
||||||
|
query: String,
|
||||||
|
filters: Option<Filters>,
|
||||||
|
debug_performance: bool,
|
||||||
|
state: State<'_, AppState>,
|
||||||
|
) -> Result<SearchResponse<SearchResult>, MeasuredFailure> {
|
||||||
|
let started = std::time::Instant::now();
|
||||||
|
let mut trace = TraceCollector::new(if debug_performance {
|
||||||
|
MetricsMode::Debug
|
||||||
|
} else {
|
||||||
|
MetricsMode::Basic
|
||||||
|
});
|
||||||
|
let semantic = state.semantic.clone();
|
||||||
|
if !semantic.warmed.load(Ordering::Acquire) {
|
||||||
|
return Err(MeasuredFailure::new(
|
||||||
|
"Semantic search is preparing in the background; keyword results are available.",
|
||||||
|
trace,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let threads = trace.timed("desktop.settings_lock", None, |_, _| {
|
||||||
|
state
|
||||||
|
.settings
|
||||||
|
.lock()
|
||||||
|
.map(|settings| settings.value.cpu_threads)
|
||||||
|
.map_err(|_| "Settings are unavailable.")
|
||||||
|
});
|
||||||
|
let threads = match threads {
|
||||||
|
Ok(threads) => threads,
|
||||||
|
Err(error) => return Err(MeasuredFailure::new(error, trace)),
|
||||||
|
};
|
||||||
|
if semantic
|
||||||
|
.querying
|
||||||
|
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return Err(MeasuredFailure::new(
|
||||||
|
"Semantic worker is busy; keyword results remain available.",
|
||||||
|
trace,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let database = state.database.clone();
|
||||||
|
let queue = trace.begin("desktop.worker_queue", None);
|
||||||
|
tauri::async_runtime::spawn_blocking(move || {
|
||||||
|
let _guard = WorkGuard(&semantic.querying);
|
||||||
|
trace.end(queue, Default::default());
|
||||||
|
let result = (|| {
|
||||||
|
let mut cached = trace.timed("desktop.semantic_lock", None, |_, _| {
|
||||||
|
semantic
|
||||||
|
.service
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "Semantic session is unavailable.")
|
||||||
|
})?;
|
||||||
|
if cached.is_none() {
|
||||||
|
*cached = Some(trace.timed("desktop.session_initialize", None, |_, _| {
|
||||||
|
HybridSearch::open(
|
||||||
|
database,
|
||||||
|
&semantic.model_directory,
|
||||||
|
&semantic.worker,
|
||||||
|
threads,
|
||||||
|
)
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
})?);
|
||||||
|
}
|
||||||
|
cached
|
||||||
|
.as_mut()
|
||||||
|
.ok_or("Semantic session is unavailable.")?
|
||||||
|
.search_traced(
|
||||||
|
&query,
|
||||||
|
50,
|
||||||
|
&filters.unwrap_or_default().into(),
|
||||||
|
&mut trace,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
})();
|
||||||
|
measured_outcome(result, trace)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| MeasuredFailure::task(error.to_string(), started))?
|
||||||
|
}
|
||||||
@@ -20,7 +20,8 @@
|
|||||||
}
|
}
|
||||||
],
|
],
|
||||||
"security": {
|
"security": {
|
||||||
"csp": "default-src 'self'; connect-src ipc: http://ipc.localhost; style-src 'self' 'unsafe-inline'; img-src 'self' data:"
|
"csp": "default-src 'self'; connect-src ipc: http://ipc.localhost; style-src 'self' 'unsafe-inline'; img-src 'self' data: asset: http://asset.localhost https://asset.localhost; media-src 'self' asset: http://asset.localhost https://asset.localhost",
|
||||||
|
"assetProtocol": { "enable": true, "scope": [] }
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"bundle": {
|
"bundle": {
|
||||||
|
|||||||
@@ -14,6 +14,10 @@ import arrowDown from "@fluentui/svg-icons/icons/arrow_down_16_regular.svg?raw";
|
|||||||
import open from "@fluentui/svg-icons/icons/open_20_regular.svg?raw";
|
import open from "@fluentui/svg-icons/icons/open_20_regular.svg?raw";
|
||||||
import chevron from "@fluentui/svg-icons/icons/chevron_down_16_regular.svg?raw";
|
import chevron from "@fluentui/svg-icons/icons/chevron_down_16_regular.svg?raw";
|
||||||
import sparkles from "@fluentui/svg-icons/icons/sparkle_20_regular.svg?raw";
|
import sparkles from "@fluentui/svg-icons/icons/sparkle_20_regular.svg?raw";
|
||||||
|
import image from "@fluentui/svg-icons/icons/image_24_regular.svg?raw";
|
||||||
|
import video from "@fluentui/svg-icons/icons/video_24_regular.svg?raw";
|
||||||
|
import audio from "@fluentui/svg-icons/icons/music_note_2_24_regular.svg?raw";
|
||||||
|
import { mediaKind } from "./media-display";
|
||||||
|
|
||||||
const assets = {
|
const assets = {
|
||||||
search,
|
search,
|
||||||
@@ -31,11 +35,16 @@ const assets = {
|
|||||||
open,
|
open,
|
||||||
chevron,
|
chevron,
|
||||||
sparkles,
|
sparkles,
|
||||||
|
image,
|
||||||
|
video,
|
||||||
|
audio,
|
||||||
};
|
};
|
||||||
export function icon(name: keyof typeof assets) {
|
export function icon(name: keyof typeof assets) {
|
||||||
return `<span class="icon" aria-hidden="true">${assets[name]}</span>`;
|
return `<span class="icon" aria-hidden="true">${assets[name]}</span>`;
|
||||||
}
|
}
|
||||||
export function fileIcon(filename: string) {
|
export function fileIcon(filename: string) {
|
||||||
|
const media = mediaKind(filename);
|
||||||
|
if (media) return { html: icon(media), type: media };
|
||||||
const extension = filename.split(".").pop()?.toLowerCase();
|
const extension = filename.split(".").pop()?.toLowerCase();
|
||||||
if (extension === "pdf") return { html: icon("pdf"), type: "pdf" };
|
if (extension === "pdf") return { html: icon("pdf"), type: "pdf" };
|
||||||
if (
|
if (
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { invoke, isTauri } from "@tauri-apps/api/core";
|
import { invoke, isTauri, convertFileSrc } from "@tauri-apps/api/core";
|
||||||
import { listen } from "@tauri-apps/api/event";
|
import { listen } from "@tauri-apps/api/event";
|
||||||
import {
|
import {
|
||||||
AutomaticIndexGate,
|
AutomaticIndexGate,
|
||||||
@@ -7,8 +7,22 @@ import {
|
|||||||
} from "./search-state";
|
} from "./search-state";
|
||||||
import type { SemanticPhase } from "./search-state";
|
import type { SemanticPhase } from "./search-state";
|
||||||
import { fileIcon, icon } from "./icons";
|
import { fileIcon, icon } from "./icons";
|
||||||
|
import {
|
||||||
|
SearchMeasurement,
|
||||||
|
measuredFailure,
|
||||||
|
spanDepth,
|
||||||
|
briefMs,
|
||||||
|
} from "./search-performance";
|
||||||
|
import type { MeasuredResponse, SearchLane } from "./search-performance";
|
||||||
import "@fontsource-variable/inter/wght.css";
|
import "@fontsource-variable/inter/wght.css";
|
||||||
import "./style.css";
|
import "./style.css";
|
||||||
|
import {
|
||||||
|
mediaKind,
|
||||||
|
matchLabel,
|
||||||
|
sourceLabel,
|
||||||
|
seekSeconds,
|
||||||
|
PreviewRevision,
|
||||||
|
} from "./media-display";
|
||||||
|
|
||||||
interface SearchResult {
|
interface SearchResult {
|
||||||
path: string;
|
path: string;
|
||||||
@@ -16,7 +30,7 @@ interface SearchResult {
|
|||||||
snippet: string;
|
snippet: string;
|
||||||
score: number;
|
score: number;
|
||||||
match_kind: string;
|
match_kind: string;
|
||||||
source?: { kind: string; number: number };
|
source?: { kind: string; number: number; matched_query?: string };
|
||||||
extraction_error?: string;
|
extraction_error?: string;
|
||||||
}
|
}
|
||||||
interface IndexReport {
|
interface IndexReport {
|
||||||
@@ -55,11 +69,16 @@ interface SemanticStatus {
|
|||||||
stale: boolean;
|
stale: boolean;
|
||||||
pending: number;
|
pending: number;
|
||||||
warming: boolean;
|
warming: boolean;
|
||||||
|
media_tools_ready?: boolean;
|
||||||
|
media_setup_error?: string | null;
|
||||||
|
media_failures?: { path: string; message: string }[];
|
||||||
worker_info?: {
|
worker_info?: {
|
||||||
model?: string;
|
model?: string;
|
||||||
device?: string;
|
device?: string;
|
||||||
runtime?: string;
|
runtime?: string;
|
||||||
threads?: number;
|
threads?: number;
|
||||||
|
dimensions?: number;
|
||||||
|
media_supported?: boolean;
|
||||||
} | null;
|
} | null;
|
||||||
}
|
}
|
||||||
interface SearchRequest {
|
interface SearchRequest {
|
||||||
@@ -67,15 +86,16 @@ interface SearchRequest {
|
|||||||
query: string;
|
query: string;
|
||||||
filters: Record<string, string | number | null>;
|
filters: Record<string, string | number | null>;
|
||||||
started: number;
|
started: number;
|
||||||
|
measurement: SearchMeasurement;
|
||||||
}
|
}
|
||||||
|
|
||||||
document.querySelector<HTMLDivElement>("#app")!.innerHTML = `
|
document.querySelector<HTMLDivElement>("#app")!.innerHTML = `
|
||||||
<header><div class="brand"><img class="brand-symbol" src="/searchi-symbol.png" alt=""><span class="wordmark">searchi</span></div><span id="index-status" class="index-status">Loading your index…</span><div class="header-actions"><button id="add-folder" class="quiet-button" type="button">${icon("folderAdd")}<span>Add folder</span></button><details id="filters" class="popover"><summary>${icon("filter")}<span>All files</span>${icon("chevron")}</summary><div class="popover-panel"><h2>Filter your search</h2><div class="filters-grid"><label>Within folder<input id="filter-folder" placeholder="Absolute folder path"></label><label>Extension<input id="filter-extension" placeholder="pdf, md, txt…"></label><label>Minimum size (bytes)<input id="filter-min" type="number" min="0" step="1"></label><label>Maximum size (bytes)<input id="filter-max" type="number" min="0" step="1"></label><label>Modified from<input id="filter-after" type="date"></label><label>Modified through<input id="filter-before" type="date"></label></div></div></details><details id="settings" class="popover"><summary class="icon-button" aria-label="Settings" title="Settings">${icon("settings")}</summary><div class="popover-panel"><h2>Folders & resources</h2><p id="shortcut-status" class="hint"></p><label class="resource-label" for="cpu-threads">CPU thread budget</label><select id="cpu-threads">${Array.from({ length: 8 }, (_, i) => `<option value="${i + 1}">${i + 1} ${i ? "threads" : "thread (default)"}</option>`).join("")}</select><p class="hint">Search automatically uses supported acceleration with CPU fallback.</p><ul id="roots" aria-label="Selected folders"></ul></div></details></div></header>
|
<header><div class="brand"><img class="brand-symbol" src="/searchi-symbol.png" alt=""><span class="wordmark">searchi</span></div><span id="index-status" class="index-status">Loading your index…</span><div class="header-actions"><button id="add-folder" class="quiet-button" type="button">${icon("folderAdd")}<span>Add folder</span></button><details id="filters" class="popover"><summary>${icon("filter")}<span>All files</span>${icon("chevron")}</summary><div class="popover-panel"><h2>Filter your search</h2><div class="filters-grid"><label>Within folder<input id="filter-folder" placeholder="Absolute folder path"></label><label>Extension<input id="filter-extension" placeholder="pdf, md, txt…"></label><label>Minimum size (bytes)<input id="filter-min" type="number" min="0" step="1"></label><label>Maximum size (bytes)<input id="filter-max" type="number" min="0" step="1"></label><label>Minimum duration (seconds)<input id="filter-duration-min" type="number" min="0" step="0.1"></label><label>Maximum duration (seconds)<input id="filter-duration-max" type="number" min="0" step="0.1"></label><label>Modified from<input id="filter-after" type="date"></label><label>Modified through<input id="filter-before" type="date"></label></div></div></details><details id="settings" class="popover"><summary class="icon-button" aria-label="Settings" title="Settings">${icon("settings")}</summary><div class="popover-panel"><h2>Folders & resources</h2><p id="shortcut-status" class="hint"></p><label class="resource-label" for="cpu-threads">CPU thread budget</label><select id="cpu-threads">${Array.from({ length: 8 }, (_, i) => `<option value="${i + 1}">${i + 1} ${i ? "threads" : "thread (default)"}</option>`).join("")}</select><p class="hint">Search automatically uses supported acceleration with CPU fallback.</p><ul id="roots" aria-label="Selected folders"></ul><label class="performance-toggle"><input id="performance-toggle" type="checkbox">Performance details</label><p class="hint">Local timing diagnostics for new searches. Queries and file paths are excluded from copied reports.</p></div></details></div></header>
|
||||||
<main>
|
<main>
|
||||||
<form id="search-form" role="search"><label class="sr-only" for="query">Search your files</label>${icon("search")}<input id="query" type="search" placeholder="Search a filename, phrase, or what you remember…" autocomplete="off" spellcheck="false" maxlength="1000"><kbd id="focus-shortcut">Ctrl K</kbd><button id="clear-query" class="icon-button" type="button" aria-label="Clear search" title="Clear search">${icon("dismiss")}</button></form>
|
<form id="search-form" role="search"><label class="sr-only" for="query">Search your files</label>${icon("search")}<input id="query" type="search" placeholder="Search a filename, phrase, or what you remember…" autocomplete="off" spellcheck="false" maxlength="1000"><kbd id="focus-shortcut">Ctrl K</kbd><button id="clear-query" class="icon-button" type="button" aria-label="Clear search" title="Clear search">${icon("dismiss")}</button></form>
|
||||||
<details id="semantic-panel"><summary>${icon("sparkles")}<span id="semantic-status" role="status" aria-live="polite">Preparing local semantic search…</span><span id="semantic-device" class="kind">Local model</span>${icon("chevron")}</summary><div class="semantic-content"><strong>Google EmbeddingGemma 2</strong><progress id="semantic-progress" aria-label="Local model setup progress"></progress><p id="semantic-detail" class="hint">The verified local model downloads automatically on first launch (865 MB). Your files stay on this device.</p><div class="semantic-actions"><button id="retry-model" type="button" hidden>Retry model setup</button><button id="rebuild-semantic" type="button" hidden>Rebuild semantic index</button></div></div></details>
|
<details id="semantic-panel"><summary>${icon("sparkles")}<span id="semantic-status" role="status" aria-live="polite">Preparing local semantic search…</span><span id="semantic-device" class="kind">Local model</span>${icon("chevron")}</summary><div class="semantic-content"><strong>Google EmbeddingGemma 2</strong><progress id="semantic-progress" aria-label="Local model setup progress"></progress><p id="semantic-detail" class="hint">The verified local model downloads automatically on first launch (865 MB). Your files stay on this device.</p><div class="semantic-actions"><button id="retry-model" type="button" hidden>Retry model setup</button><button id="rebuild-semantic" type="button" hidden>Rebuild semantic index</button></div></div></details>
|
||||||
<form id="folder-form" hidden><label for="folder">Folder to index</label><div class="folder-row"><input id="folder" placeholder="Absolute folder path" required><button id="browse-folder" type="button">Browse…</button><button type="submit">Index folder</button><button id="cancel-folder" type="button">Cancel</button></div><p class="hint">Only choose folders you want searched. Large folders may take a while.</p></form>
|
<form id="folder-form" hidden><label for="folder">Folder to index</label><div class="folder-row"><input id="folder" placeholder="Absolute folder path" required><button id="browse-folder" type="button">Browse…</button><button type="submit">Index folder</button><button id="cancel-folder" type="button">Cancel</button></div><p class="hint">Only choose folders you want searched. Large folders may take a while.</p></form>
|
||||||
<p id="notice" role="status" aria-live="polite"></p><div class="workspace"><section class="results-pane" aria-label="Search results"><p id="result-summary" class="eyebrow">READY WHEN YOU ARE</p><ul id="results" aria-label="Matching files"></ul><div id="empty">${icon("search")}<h2>Find what matters. Locally.</h2><p>Index a folder, then search a filename, phrase, or what you remember.</p></div></section><aside id="preview" aria-label="Result preview" hidden><div class="preview-header"><span id="preview-icon" class="file-icon"></span><div class="preview-heading"><h2 id="preview-title"></h2><p id="preview-path" class="path"></p></div></div><div class="preview-toolbar"><span id="preview-kind"></span><div class="preview-actions"><button id="open-result" type="button">${icon("open")}<span>Open file</span></button><button id="reveal-result" type="button" title="Show in folder" aria-label="Show in folder">${icon("folder")}</button></div></div><div class="passage"><p class="eyebrow">EXTRACTED PASSAGE</p><pre id="preview-text"></pre></div></aside></div>
|
<p id="notice" role="status" aria-live="polite"></p><details id="performance-panel" hidden><summary>Performance details</summary><div id="performance-detail"></div><button id="copy-performance" type="button">Copy local timing report</button><span id="copy-performance-status" role="status"></span></details><div class="workspace"><section class="results-pane" aria-label="Search results"><p id="result-summary" class="eyebrow">READY WHEN YOU ARE</p><ul id="results" aria-label="Matching files"></ul><div id="empty">${icon("search")}<h2>Find what matters. Locally.</h2><p>Index a folder, then search a filename, phrase, or what you remember.</p></div></section><aside id="preview" aria-label="Result preview" hidden><div class="preview-header"><span id="preview-icon" class="file-icon"></span><div class="preview-heading"><h2 id="preview-title"></h2><p id="preview-path" class="path"></p></div></div><div class="preview-toolbar"><span id="preview-kind"></span><div class="preview-actions"><button id="open-result" type="button">${icon("open")}<span>Open file</span></button><button id="reveal-result" type="button" title="Show in folder" aria-label="Show in folder">${icon("folder")}</button></div></div><div id="media-preview" class="media-preview" hidden></div><p id="media-preview-status" class="hint" role="status" hidden></p><div class="passage"><p id="passage-label" class="eyebrow">EXTRACTED PASSAGE</p><pre id="preview-text"></pre></div></aside></div>
|
||||||
</main><footer><span class="footer-privacy">${icon("lock")}<span>Private search. Everything stays on your device.</span><span id="privacy" class="privacy">Local</span></span><span class="keyboard-hints"><kbd>${icon("arrowUp")}</kbd><kbd>${icon("arrowDown")}</kbd><span>Navigate</span><kbd>Enter</kbd><span>Open</span><kbd>Esc</kbd><span>Clear</span></span></footer>`;
|
</main><footer><span class="footer-privacy">${icon("lock")}<span>Private search. Everything stays on your device.</span><span id="privacy" class="privacy">Local</span></span><span class="keyboard-hints"><kbd>${icon("arrowUp")}</kbd><kbd>${icon("arrowDown")}</kbd><span>Navigate</span><kbd>Enter</kbd><span>Open</span><kbd>Esc</kbd><span>Clear</span></span></footer>`;
|
||||||
|
|
||||||
const element = <T extends HTMLElement>(id: string) =>
|
const element = <T extends HTMLElement>(id: string) =>
|
||||||
@@ -83,6 +103,12 @@ const element = <T extends HTMLElement>(id: string) =>
|
|||||||
const query = element<HTMLInputElement>("query");
|
const query = element<HTMLInputElement>("query");
|
||||||
const list = element<HTMLUListElement>("results");
|
const list = element<HTMLUListElement>("results");
|
||||||
const notice = element<HTMLParagraphElement>("notice");
|
const notice = element<HTMLParagraphElement>("notice");
|
||||||
|
try {
|
||||||
|
element<HTMLInputElement>("performance-toggle").checked =
|
||||||
|
localStorage.getItem("searchi-performance-details") === "true";
|
||||||
|
} catch {
|
||||||
|
/* Diagnostics remain opt-in when storage is unavailable. */
|
||||||
|
}
|
||||||
let results: SearchResult[] = [];
|
let results: SearchResult[] = [];
|
||||||
let selected = -1;
|
let selected = -1;
|
||||||
let requestId = 0;
|
let requestId = 0;
|
||||||
@@ -99,6 +125,89 @@ let folderRevision = 0;
|
|||||||
let localSemanticError: string | null = null;
|
let localSemanticError: string | null = null;
|
||||||
let querySemanticFailed = false;
|
let querySemanticFailed = false;
|
||||||
let previewPath: string | null = null;
|
let previewPath: string | null = null;
|
||||||
|
const previewRevision = new PreviewRevision();
|
||||||
|
interface PreviewRequest {
|
||||||
|
revision: number;
|
||||||
|
result: SearchResult;
|
||||||
|
}
|
||||||
|
function unloadMedia() {
|
||||||
|
element("media-preview")
|
||||||
|
.querySelectorAll<HTMLMediaElement>("audio,video")
|
||||||
|
.forEach((player) => {
|
||||||
|
player.pause();
|
||||||
|
player.removeAttribute("src");
|
||||||
|
player.load();
|
||||||
|
});
|
||||||
|
element("media-preview").replaceChildren();
|
||||||
|
element("media-preview").hidden = true;
|
||||||
|
element("media-preview-status").hidden = true;
|
||||||
|
}
|
||||||
|
function clearMedia() {
|
||||||
|
previewRevision.clear();
|
||||||
|
mediaLane.clear();
|
||||||
|
unloadMedia();
|
||||||
|
}
|
||||||
|
const mediaLane = new LatestLane<PreviewRequest>(
|
||||||
|
async (request) => {
|
||||||
|
const prepared = await invoke<{
|
||||||
|
path: string;
|
||||||
|
kind: "image" | "audio" | "video";
|
||||||
|
}>("prepare_media_preview", { path: request.result.path });
|
||||||
|
if (!previewRevision.accepts(request.revision)) return;
|
||||||
|
const host = element("media-preview");
|
||||||
|
const status = element("media-preview-status");
|
||||||
|
const failed = () => {
|
||||||
|
if (!previewRevision.accepts(request.revision)) return;
|
||||||
|
status.hidden = false;
|
||||||
|
status.textContent =
|
||||||
|
"This format cannot be previewed here. Use Open file to view it in your default application.";
|
||||||
|
};
|
||||||
|
if (prepared.kind === "image") {
|
||||||
|
const image = document.createElement("img");
|
||||||
|
image.alt = request.result.filename;
|
||||||
|
image.addEventListener("error", failed, { once: true });
|
||||||
|
image.src = convertFileSrc(prepared.path);
|
||||||
|
host.append(image);
|
||||||
|
} else {
|
||||||
|
const player = document.createElement(prepared.kind);
|
||||||
|
player.controls = true;
|
||||||
|
player.preload = "metadata";
|
||||||
|
player.addEventListener("error", failed, { once: true });
|
||||||
|
player.addEventListener(
|
||||||
|
"loadedmetadata",
|
||||||
|
() => {
|
||||||
|
if (!previewRevision.accepts(request.revision)) return;
|
||||||
|
const target = seekSeconds(request.result.source);
|
||||||
|
if (target !== undefined) {
|
||||||
|
if (!Number.isFinite(player.duration) || target > player.duration) {
|
||||||
|
status.hidden = false;
|
||||||
|
status.textContent =
|
||||||
|
"The matched timestamp is unavailable in this preview. Use Open file to check the recording.";
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
player.currentTime = target;
|
||||||
|
} catch {
|
||||||
|
failed();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
player.src = convertFileSrc(prepared.path);
|
||||||
|
host.append(player);
|
||||||
|
}
|
||||||
|
host.hidden = false;
|
||||||
|
status.hidden = true;
|
||||||
|
},
|
||||||
|
(request) => previewRevision.accepts(request.revision),
|
||||||
|
(request, error) => {
|
||||||
|
if (!previewRevision.accepts(request.revision)) return;
|
||||||
|
element("media-preview-status").hidden = false;
|
||||||
|
element("media-preview-status").textContent =
|
||||||
|
`Preview unavailable: ${String(error)}. Use Open file.`;
|
||||||
|
},
|
||||||
|
);
|
||||||
let displayedRequestId: number | null = null;
|
let displayedRequestId: number | null = null;
|
||||||
let expandedError: string | null = null;
|
let expandedError: string | null = null;
|
||||||
const reducedMotion = matchMedia("(prefers-reduced-motion: reduce)");
|
const reducedMotion = matchMedia("(prefers-reduced-motion: reduce)");
|
||||||
@@ -168,7 +277,7 @@ async function buildSemanticIndex() {
|
|||||||
"index_embeddings",
|
"index_embeddings",
|
||||||
);
|
);
|
||||||
element("semantic-status").textContent =
|
element("semantic-status").textContent =
|
||||||
`Semantic search indexed ${report.chunks.toLocaleString()} passages.`;
|
`Semantic search indexed ${report.chunks.toLocaleString()} content segments.`;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
localSemanticError = String(error);
|
localSemanticError = String(error);
|
||||||
} finally {
|
} finally {
|
||||||
@@ -229,7 +338,7 @@ async function refreshSemanticStatus() {
|
|||||||
message =
|
message =
|
||||||
"Building your semantic index… Keyword search remains available.";
|
"Building your semantic index… Keyword search remains available.";
|
||||||
else if (status.ready && status.chunks > 0)
|
else if (status.ready && status.chunks > 0)
|
||||||
message = `Semantic search ready · ${status.chunks.toLocaleString()} passages${status.stale ? " · Updates pending" : ""}`;
|
message = `Semantic search ready · ${status.chunks.toLocaleString()} content segments${status.stale ? " · Updates pending" : ""}${status.media_failures?.length ? ` · ${status.media_failures.length} files could not be indexed` : ""}`;
|
||||||
else if (status.ready)
|
else if (status.ready)
|
||||||
message = "Google model ready. Index a folder to search by meaning.";
|
message = "Google model ready. Index a folder to search by meaning.";
|
||||||
else
|
else
|
||||||
@@ -240,6 +349,18 @@ async function refreshSemanticStatus() {
|
|||||||
element("semantic-detail").textContent = status.worker_info
|
element("semantic-detail").textContent = status.worker_info
|
||||||
? `${status.model} · ${status.worker_info.runtime || "Local inference"} · Your files stay on this device.`
|
? `${status.model} · ${status.worker_info.runtime || "Local inference"} · Your files stay on this device.`
|
||||||
: "The verified Google model downloads automatically once (865 MB). After setup, inference runs locally.";
|
: "The verified Google model downloads automatically once (865 MB). After setup, inference runs locally.";
|
||||||
|
const mediaDetail = status.media_setup_error
|
||||||
|
? `Media decoder setup needs attention: ${status.media_setup_error}. Text search remains available.`
|
||||||
|
: status.worker_info && !status.worker_info.media_supported
|
||||||
|
? "The installed worker needs a media-capable runtime update. Text search remains available."
|
||||||
|
: status.media_tools_ready && status.worker_info?.media_supported
|
||||||
|
? "Image, audio, and video encoders are available locally."
|
||||||
|
: "Preparing local image, audio, and video support.";
|
||||||
|
element("semantic-detail").textContent += ` ${mediaDetail}`;
|
||||||
|
if (status.media_failures?.length) {
|
||||||
|
element("semantic-detail").textContent +=
|
||||||
|
` ${status.media_failures.length} files could not be indexed. The first issue: ${status.media_failures[0].path}: ${status.media_failures[0].message}. Filename search remains available; rebuild after fixing the file.`;
|
||||||
|
}
|
||||||
const retry = element<HTMLButtonElement>("retry-model");
|
const retry = element<HTMLButtonElement>("retry-model");
|
||||||
retry.hidden = !error || status.ready;
|
retry.hidden = !error || status.ready;
|
||||||
retry.disabled = working;
|
retry.disabled = working;
|
||||||
@@ -266,7 +387,7 @@ async function refreshSemanticStatus() {
|
|||||||
canSearchSemantically() &&
|
canSearchSemantically() &&
|
||||||
latestSearch?.id === requestId
|
latestSearch?.id === requestId
|
||||||
)
|
)
|
||||||
hybridLane.submit(latestSearch);
|
submitSearch(latestSearch, "hybrid");
|
||||||
if (working || (!status.ready && !error)) queueSemanticRefresh();
|
if (working || (!status.ready && !error)) queueSemanticRefresh();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
element("semantic-status").textContent =
|
element("semantic-status").textContent =
|
||||||
@@ -349,15 +470,33 @@ function select(index: number, focus = false) {
|
|||||||
element("preview-title").textContent = result.filename;
|
element("preview-title").textContent = result.filename;
|
||||||
element("preview-path").textContent = result.path;
|
element("preview-path").textContent = result.path;
|
||||||
element("preview-kind").textContent =
|
element("preview-kind").textContent =
|
||||||
`${result.match_kind} match${result.source ? ` · ${result.source.kind} ${result.source.number}` : ""}`;
|
`${matchLabel(result.match_kind)}${result.source ? ` · ${sourceLabel(result.source)}` : ""}`;
|
||||||
|
const media = mediaKind(result.filename);
|
||||||
|
element("passage-label").textContent = media
|
||||||
|
? "MATCHED CONTENT"
|
||||||
|
: "EXTRACTED PASSAGE";
|
||||||
element("preview-text").textContent =
|
element("preview-text").textContent =
|
||||||
result.snippet ||
|
result.snippet ||
|
||||||
(result.extraction_error
|
(result.extraction_error
|
||||||
? `Content could not be extracted: ${result.extraction_error}`
|
? `Content could not be extracted: ${result.extraction_error}`
|
||||||
: "No extracted text is available for this file.");
|
: media
|
||||||
|
? result.source?.matched_query || "Preview the matched media below."
|
||||||
|
: "No extracted text is available for this file.");
|
||||||
element<HTMLButtonElement>("open-result").title = result.source
|
element<HTMLButtonElement>("open-result").title = result.source
|
||||||
? `Opens this file in your default application. ${result.source.kind} ${result.source.number} is shown here; exact-location opening is not supported by the default application interface.`
|
? `Open in your default application. The matched location ${sourceLabel(result.source)} is shown in Searchi; default application opening starts at its own location.`
|
||||||
: "Open in your default application";
|
: "Open in your default application";
|
||||||
|
const selection = previewRevision.select(
|
||||||
|
JSON.stringify([result.path, result.source?.kind, result.source?.number]),
|
||||||
|
);
|
||||||
|
if (selection.changed) {
|
||||||
|
mediaLane.clear();
|
||||||
|
unloadMedia();
|
||||||
|
if (media && isTauri()) {
|
||||||
|
element("media-preview-status").hidden = false;
|
||||||
|
element("media-preview-status").textContent = "Preparing media preview…";
|
||||||
|
mediaLane.submit({ revision: selection.revision, result });
|
||||||
|
}
|
||||||
|
}
|
||||||
if (previewPath !== result.path) {
|
if (previewPath !== result.path) {
|
||||||
const passage = document.querySelector<HTMLElement>(".passage")!;
|
const passage = document.querySelector<HTMLElement>(".passage")!;
|
||||||
passage.scrollTop = 0;
|
passage.scrollTop = 0;
|
||||||
@@ -388,6 +527,7 @@ function render() {
|
|||||||
if (!results.length) {
|
if (!results.length) {
|
||||||
element("preview").hidden = true;
|
element("preview").hidden = true;
|
||||||
previewPath = null;
|
previewPath = null;
|
||||||
|
clearMedia();
|
||||||
}
|
}
|
||||||
results.forEach((result, i) => {
|
results.forEach((result, i) => {
|
||||||
let entry = resultRows.get(result.path);
|
let entry = resultRows.get(result.path);
|
||||||
@@ -419,9 +559,7 @@ function render() {
|
|||||||
entry.title.textContent = result.filename;
|
entry.title.textContent = result.filename;
|
||||||
entry.path.textContent = result.path;
|
entry.path.textContent = result.path;
|
||||||
entry.snippet.textContent = result.snippet;
|
entry.snippet.textContent = result.snippet;
|
||||||
entry.kind.textContent = result.source
|
entry.kind.textContent = `${matchLabel(result.match_kind)}${result.source ? ` · ${sourceLabel(result.source)}` : ""}`;
|
||||||
? `${result.source.kind} ${result.source.number}`
|
|
||||||
: result.match_kind;
|
|
||||||
const asset = fileIcon(result.filename);
|
const asset = fileIcon(result.filename);
|
||||||
if (entry.file.dataset.type !== asset.type) {
|
if (entry.file.dataset.type !== asset.type) {
|
||||||
entry.file.innerHTML = asset.html;
|
entry.file.innerHTML = asset.html;
|
||||||
@@ -443,6 +581,14 @@ function captureFilters() {
|
|||||||
throw new Error("File size must be a nonnegative whole number of bytes.");
|
throw new Error("File size must be a nonnegative whole number of bytes.");
|
||||||
return value;
|
return value;
|
||||||
};
|
};
|
||||||
|
const duration = (id: string) => {
|
||||||
|
const text = input(id);
|
||||||
|
if (!text) return null;
|
||||||
|
const ms = Math.round(Number(text) * 1000);
|
||||||
|
if (!Number.isFinite(ms) || ms < 0 || ms > 4_294_967_295)
|
||||||
|
throw new Error("Enter a valid nonnegative media duration in seconds.");
|
||||||
|
return ms;
|
||||||
|
};
|
||||||
// Date controls represent the user's local calendar day, including its end.
|
// Date controls represent the user's local calendar day, including its end.
|
||||||
const date = (id: string, end: boolean) => {
|
const date = (id: string, end: boolean) => {
|
||||||
const text = input(id);
|
const text = input(id);
|
||||||
@@ -457,6 +603,8 @@ function captureFilters() {
|
|||||||
return {
|
return {
|
||||||
folder: input("filter-folder") || null,
|
folder: input("filter-folder") || null,
|
||||||
extension: input("filter-extension") || null,
|
extension: input("filter-extension") || null,
|
||||||
|
min_duration_ms: duration("filter-duration-min"),
|
||||||
|
max_duration_ms: duration("filter-duration-max"),
|
||||||
min_size: size("filter-min"),
|
min_size: size("filter-min"),
|
||||||
max_size: size("filter-max"),
|
max_size: size("filter-max"),
|
||||||
modified_after_unix_ms: date("filter-after", false),
|
modified_after_unix_ms: date("filter-after", false),
|
||||||
@@ -474,6 +622,7 @@ function applyResults(
|
|||||||
(lane === "lexical" && shownLane === "hybrid")
|
(lane === "lexical" && shownLane === "hybrid")
|
||||||
)
|
)
|
||||||
return;
|
return;
|
||||||
|
const renderStarted = performance.now();
|
||||||
const selectedPath = results[selected]?.path;
|
const selectedPath = results[selected]?.path;
|
||||||
const resultsHadFocus = list.contains(document.activeElement);
|
const resultsHadFocus = list.contains(document.activeElement);
|
||||||
shownLane = lane;
|
shownLane = lane;
|
||||||
@@ -484,18 +633,131 @@ function applyResults(
|
|||||||
const preserved = results.findIndex((result) => result.path === selectedPath);
|
const preserved = results.findIndex((result) => result.path === selectedPath);
|
||||||
if (preserved >= 0) select(preserved, resultsHadFocus);
|
if (preserved >= 0) select(preserved, resultsHadFocus);
|
||||||
else if (results.length) select(0, resultsHadFocus);
|
else if (results.length) select(0, resultsHadFocus);
|
||||||
element("result-summary").textContent =
|
|
||||||
`${found.length} RESULTS · ${lane === "hybrid" ? "KEYWORD + MEANING" : "KEYWORD"} · ${Math.round(performance.now() - request.started)} MS`;
|
|
||||||
if (!found.length) refreshEmptySearchState();
|
if (!found.length) refreshEmptySearchState();
|
||||||
|
request.measurement.displayed(
|
||||||
|
lane,
|
||||||
|
renderStarted,
|
||||||
|
performance.now(),
|
||||||
|
found.length,
|
||||||
|
);
|
||||||
|
const metric = request.measurement.latestNative(lane);
|
||||||
|
const trace = request.measurement.export();
|
||||||
|
element("result-summary").textContent =
|
||||||
|
`${found.length} RESULTS · ${lane === "hybrid" ? "KEYWORD + MEANING" : "KEYWORD"} · NATIVE ${briefMs(metric?.total_ms ?? 0)} MS · ${lane === "hybrid" ? "FINAL" : "FIRST RESPONSE"} ${Math.round(lane === "hybrid" ? (trace.hybrid_final_results_ms ?? 0) : (trace.first_results_ms ?? 0))} MS`;
|
||||||
|
refreshPerformance(request);
|
||||||
|
}
|
||||||
|
|
||||||
|
function refreshPerformance(request: SearchRequest) {
|
||||||
|
if (request.id !== requestId) return;
|
||||||
|
const panel = element<HTMLDetailsElement>("performance-panel");
|
||||||
|
panel.hidden = !request.measurement.debug;
|
||||||
|
if (!request.measurement.debug) return;
|
||||||
|
const body = element("performance-detail");
|
||||||
|
body.replaceChildren();
|
||||||
|
const report = request.measurement.export();
|
||||||
|
const append = (text: string, className = "") => {
|
||||||
|
const row = document.createElement("p");
|
||||||
|
row.textContent = text;
|
||||||
|
row.className = className;
|
||||||
|
body.append(row);
|
||||||
|
return row;
|
||||||
|
};
|
||||||
|
append(
|
||||||
|
"Times below use separate UI and native clocks. Nested spans and parallel lanes must not be added together. DOM update measures synchronous changes, not paint or deferred media preview loading.",
|
||||||
|
"hint",
|
||||||
|
);
|
||||||
|
append(
|
||||||
|
`Input debounce ${report.debounce_ms.toFixed(1)} ms · First displayed response ${report.first_results_ms?.toFixed(1) ?? "pending"} ms · Latest displayed ${report.latest_displayed_ms?.toFixed(1) ?? "pending"} ms · Hybrid final ${report.hybrid_final_results_ms?.toFixed(1) ?? "pending / unavailable"} ms`,
|
||||||
|
);
|
||||||
|
for (const [lane, delay] of Object.entries(report.lane_schedule_ms))
|
||||||
|
append(
|
||||||
|
`${lane}: scheduled ${delay.toFixed(1)} ms after debounce${lane === "hybrid" ? " (includes intentional 350 ms semantic delay or waiting for readiness)" : ""}`,
|
||||||
|
);
|
||||||
|
for (const attempt of report.attempts) {
|
||||||
|
append(
|
||||||
|
`${attempt.lane} attempt ${attempt.attempt}${attempt.failed ? " · FAILED" : ""}: pending queue ${attempt.queued_ms.toFixed(1)} ms · retry wait ${attempt.retry_wait_ms.toFixed(1)} ms · invoke ${attempt.invoke_ms.toFixed(1)} ms`,
|
||||||
|
"performance-attempt",
|
||||||
|
);
|
||||||
|
if (!attempt.native) {
|
||||||
|
append("Native timing unavailable for this failure.", "hint");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
append(
|
||||||
|
`Native command ${attempt.native.total_ms.toFixed(1)} ms · ${attempt.native.result_count} results · estimated transport / serialization / scheduling ${attempt.transport_estimate_ms?.toFixed(1)} ms`,
|
||||||
|
"hint",
|
||||||
|
);
|
||||||
|
if (attempt.native.trace === null)
|
||||||
|
append("Detailed native trace unavailable for this attempt.", "hint");
|
||||||
|
const spans = attempt.native.trace ?? [];
|
||||||
|
spans.forEach((span, index) => {
|
||||||
|
const counts = Object.entries(span.counters)
|
||||||
|
.map(([name, value]) =>
|
||||||
|
name.endsWith("_us")
|
||||||
|
? `${name.slice(0, -3).replaceAll("_", " ")}: ${(value / 1000).toFixed(3)} ms`
|
||||||
|
: `${name.replaceAll("_", " ")}: ${value}`,
|
||||||
|
)
|
||||||
|
.join(" · ");
|
||||||
|
const row = append(
|
||||||
|
`${span.name}: ${span.duration_ms.toFixed(2)} ms · starts +${span.start_ms.toFixed(2)} ms${counts ? ` · ${counts}` : ""}`,
|
||||||
|
"performance-span",
|
||||||
|
);
|
||||||
|
row.style.paddingLeft = `${spanDepth(spans, index) * 14}px`;
|
||||||
|
});
|
||||||
|
if (attempt.native.dropped_spans)
|
||||||
|
append(`${attempt.native.dropped_spans} native spans dropped.`, "hint");
|
||||||
|
}
|
||||||
|
for (const display of report.displays)
|
||||||
|
append(
|
||||||
|
`${display.lane} synchronous DOM update ${display.dom_update_ms.toFixed(2)} ms · ${display.count} displayed results`,
|
||||||
|
"hint",
|
||||||
|
);
|
||||||
|
if (report.dropped_attempts || report.dropped_displays)
|
||||||
|
append(
|
||||||
|
`History capped: ${report.dropped_attempts} attempts and ${report.dropped_displays} displays dropped.`,
|
||||||
|
"hint",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function invokeMeasured(request: SearchRequest, lane: SearchLane) {
|
||||||
|
const attempt = request.measurement.begin(lane, performance.now());
|
||||||
|
try {
|
||||||
|
const response = await invoke<MeasuredResponse<SearchResult>>(
|
||||||
|
lane === "lexical" ? "search_measured" : "hybrid_search_measured",
|
||||||
|
{
|
||||||
|
query: request.query,
|
||||||
|
filters: request.filters,
|
||||||
|
debugPerformance: request.measurement.debug,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
request.measurement.completed(
|
||||||
|
attempt,
|
||||||
|
performance.now(),
|
||||||
|
response.metrics,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
return response.results;
|
||||||
|
} catch (error) {
|
||||||
|
const failure = measuredFailure(error);
|
||||||
|
request.measurement.completed(
|
||||||
|
attempt,
|
||||||
|
performance.now(),
|
||||||
|
failure?.metrics ?? null,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
refreshPerformance(request);
|
||||||
|
throw failure ? new Error(failure.message) : error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function submitSearch(request: SearchRequest, lane: SearchLane) {
|
||||||
|
request.measurement.submitted(lane, performance.now());
|
||||||
|
(lane === "lexical" ? lexicalLane : hybridLane).submit(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
const isCurrent = (request: SearchRequest) => request.id === requestId;
|
const isCurrent = (request: SearchRequest) => request.id === requestId;
|
||||||
const lexicalLane = new LatestLane<SearchRequest>(
|
const lexicalLane = new LatestLane<SearchRequest>(
|
||||||
async (request) => {
|
async (request) => {
|
||||||
const found = await invoke<SearchResult[]>("search", {
|
const found = await invokeMeasured(request, "lexical");
|
||||||
query: request.query,
|
|
||||||
filters: request.filters,
|
|
||||||
});
|
|
||||||
applyResults(found, request, "lexical");
|
applyResults(found, request, "lexical");
|
||||||
},
|
},
|
||||||
isCurrent,
|
isCurrent,
|
||||||
@@ -509,10 +771,7 @@ const lexicalLane = new LatestLane<SearchRequest>(
|
|||||||
const hybridLane = new LatestLane<SearchRequest>(
|
const hybridLane = new LatestLane<SearchRequest>(
|
||||||
async (request) => {
|
async (request) => {
|
||||||
if (!canSearchSemantically()) return;
|
if (!canSearchSemantically()) return;
|
||||||
const found = await invoke<SearchResult[]>("hybrid_search", {
|
const found = await invokeMeasured(request, "hybrid");
|
||||||
query: request.query,
|
|
||||||
filters: request.filters,
|
|
||||||
});
|
|
||||||
applyResults(found, request, "hybrid");
|
applyResults(found, request, "hybrid");
|
||||||
},
|
},
|
||||||
isCurrent,
|
isCurrent,
|
||||||
@@ -534,12 +793,13 @@ function invalidateSearch() {
|
|||||||
latestSearch = null;
|
latestSearch = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function search() {
|
async function search(inputAt = performance.now()) {
|
||||||
invalidateSearch();
|
invalidateSearch();
|
||||||
querySemanticFailed = false;
|
querySemanticFailed = false;
|
||||||
shownLane = "lexical";
|
shownLane = "lexical";
|
||||||
const value = query.value.trim();
|
const value = query.value.trim();
|
||||||
if (!value) {
|
if (!value) {
|
||||||
|
element("performance-panel").hidden = true;
|
||||||
results = [];
|
results = [];
|
||||||
displayedRequestId = null;
|
displayedRequestId = null;
|
||||||
selected = -1;
|
selected = -1;
|
||||||
@@ -557,13 +817,28 @@ async function search() {
|
|||||||
query: value,
|
query: value,
|
||||||
filters: captureFilters(),
|
filters: captureFilters(),
|
||||||
started: performance.now(),
|
started: performance.now(),
|
||||||
|
measurement: new SearchMeasurement(
|
||||||
|
inputAt,
|
||||||
|
performance.now(),
|
||||||
|
element<HTMLInputElement>("performance-toggle").checked,
|
||||||
|
{
|
||||||
|
model: semanticStatus?.worker_info?.model ?? null,
|
||||||
|
device: semanticStatus?.worker_info?.device ?? null,
|
||||||
|
runtime: semanticStatus?.worker_info?.runtime ?? null,
|
||||||
|
threads: semanticStatus?.worker_info?.threads ?? null,
|
||||||
|
dimensions: semanticStatus?.worker_info?.dimensions ?? null,
|
||||||
|
semantic_ready: canSearchSemantically(),
|
||||||
|
indexed_chunks: semanticStatus?.chunks ?? null,
|
||||||
|
},
|
||||||
|
),
|
||||||
};
|
};
|
||||||
latestSearch = request;
|
latestSearch = request;
|
||||||
|
refreshPerformance(request);
|
||||||
element("result-summary").textContent = "SEARCHING…";
|
element("result-summary").textContent = "SEARCHING…";
|
||||||
lexicalLane.submit(request);
|
submitSearch(request, "lexical");
|
||||||
hybridTimer = setTimeout(() => {
|
hybridTimer = setTimeout(() => {
|
||||||
if (isCurrent(request) && canSearchSemantically())
|
if (isCurrent(request) && canSearchSemantically())
|
||||||
hybridLane.submit(request);
|
submitSearch(request, "hybrid");
|
||||||
}, 350);
|
}, 350);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
showError(error);
|
showError(error);
|
||||||
@@ -573,12 +848,41 @@ async function search() {
|
|||||||
async function refreshStats() {
|
async function refreshStats() {
|
||||||
const stats = await invoke<IndexStats>("stats");
|
const stats = await invoke<IndexStats>("stats");
|
||||||
element("index-status").textContent =
|
element("index-status").textContent =
|
||||||
`${stats.files.toLocaleString()} files · ${stats.content_files.toLocaleString()} with searchable content`;
|
`${stats.files.toLocaleString()} files · ${stats.content_files.toLocaleString()} with extracted text`;
|
||||||
}
|
}
|
||||||
|
|
||||||
query.addEventListener("input", () => {
|
query.addEventListener("input", () => {
|
||||||
|
const inputAt = performance.now();
|
||||||
invalidateSearch();
|
invalidateSearch();
|
||||||
timer = setTimeout(search, 60);
|
timer = setTimeout(() => void search(inputAt), 60);
|
||||||
|
});
|
||||||
|
element("performance-toggle").addEventListener("change", () => {
|
||||||
|
const enabled = element<HTMLInputElement>("performance-toggle").checked;
|
||||||
|
try {
|
||||||
|
localStorage.setItem("searchi-performance-details", String(enabled));
|
||||||
|
} catch {
|
||||||
|
/* Session preference still works. */
|
||||||
|
}
|
||||||
|
void search();
|
||||||
|
});
|
||||||
|
element("copy-performance").addEventListener("click", () => {
|
||||||
|
if (!latestSearch?.measurement.debug) return;
|
||||||
|
if (!navigator.clipboard) {
|
||||||
|
element("copy-performance-status").textContent =
|
||||||
|
"Clipboard unavailable. Timings remain visible here.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const report = JSON.stringify(latestSearch.measurement.export(), null, 2);
|
||||||
|
void navigator.clipboard.writeText(report).then(
|
||||||
|
() => {
|
||||||
|
element("copy-performance-status").textContent =
|
||||||
|
"Timing report copied. No query or file paths included.";
|
||||||
|
},
|
||||||
|
() => {
|
||||||
|
element("copy-performance-status").textContent =
|
||||||
|
"Clipboard unavailable. Timings remain visible here.";
|
||||||
|
},
|
||||||
|
);
|
||||||
});
|
});
|
||||||
element("search-form").addEventListener("submit", (event) => {
|
element("search-form").addEventListener("submit", (event) => {
|
||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
@@ -714,6 +1018,7 @@ document.addEventListener("keydown", (event) => {
|
|||||||
query.select();
|
query.select();
|
||||||
}
|
}
|
||||||
if (event.key === "Escape") {
|
if (event.key === "Escape") {
|
||||||
|
clearMedia();
|
||||||
document
|
document
|
||||||
.querySelectorAll<HTMLDetailsElement>(".popover[open]")
|
.querySelectorAll<HTMLDetailsElement>(".popover[open]")
|
||||||
.forEach((popover) => {
|
.forEach((popover) => {
|
||||||
|
|||||||
@@ -0,0 +1,98 @@
|
|||||||
|
/** Media formatting and selection revisions shared by the preview and tests. */
|
||||||
|
export interface MediaSource {
|
||||||
|
kind: string;
|
||||||
|
number: number;
|
||||||
|
}
|
||||||
|
export type MediaKind = "image" | "audio" | "video";
|
||||||
|
export function timestampLabel(milliseconds: number): string {
|
||||||
|
if (!Number.isFinite(milliseconds) || milliseconds < 0) return "Unknown time";
|
||||||
|
const seconds = Math.floor(milliseconds / 1000);
|
||||||
|
return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
|
||||||
|
}
|
||||||
|
export function seekSeconds(source?: MediaSource): number | undefined {
|
||||||
|
return source?.kind === "timestamp_ms" &&
|
||||||
|
Number.isFinite(source.number) &&
|
||||||
|
source.number >= 0
|
||||||
|
? source.number / 1000
|
||||||
|
: undefined;
|
||||||
|
}
|
||||||
|
export function sourceLabel(source?: MediaSource): string {
|
||||||
|
if (!source) return "";
|
||||||
|
if (source.kind === "image") return "Image";
|
||||||
|
if (source.kind === "timestamp_ms") return timestampLabel(source.number);
|
||||||
|
return `${source.kind} ${source.number}`;
|
||||||
|
}
|
||||||
|
export function matchLabel(kind: string): string {
|
||||||
|
const labels: Record<string, string> = {
|
||||||
|
semantic_image: "Image meaning",
|
||||||
|
semantic_video: "Video frame meaning",
|
||||||
|
semantic_audio: "Audio meaning",
|
||||||
|
semantic: "Meaning",
|
||||||
|
hybrid: "Keyword + meaning",
|
||||||
|
exact_filename: "Exact filename",
|
||||||
|
exact_identifier: "Exact identifier",
|
||||||
|
filename: "Filename",
|
||||||
|
keyword: "Content",
|
||||||
|
};
|
||||||
|
return labels[kind] ?? kind.replaceAll("_", " ");
|
||||||
|
}
|
||||||
|
export function mediaKind(filename: string): MediaKind | undefined {
|
||||||
|
const extension = filename.split(".").pop()?.toLowerCase() ?? "";
|
||||||
|
if (
|
||||||
|
[
|
||||||
|
"png",
|
||||||
|
"jpg",
|
||||||
|
"jpeg",
|
||||||
|
"webp",
|
||||||
|
"bmp",
|
||||||
|
"gif",
|
||||||
|
"tiff",
|
||||||
|
"tif",
|
||||||
|
"heic",
|
||||||
|
"avif",
|
||||||
|
].includes(extension)
|
||||||
|
)
|
||||||
|
return "image";
|
||||||
|
if (
|
||||||
|
["mp4", "mkv", "mov", "webm", "avi", "m4v", "mpeg", "mpg"].includes(
|
||||||
|
extension,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return "video";
|
||||||
|
if (
|
||||||
|
[
|
||||||
|
"mp3",
|
||||||
|
"wav",
|
||||||
|
"flac",
|
||||||
|
"ogg",
|
||||||
|
"m4a",
|
||||||
|
"aac",
|
||||||
|
"opus",
|
||||||
|
"wma",
|
||||||
|
"aiff",
|
||||||
|
"aif",
|
||||||
|
].includes(extension)
|
||||||
|
)
|
||||||
|
return "audio";
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
/** Even selecting the same file at a new timestamp invalidates old responses. */
|
||||||
|
export class PreviewRevision {
|
||||||
|
private revision = 0;
|
||||||
|
private key = "";
|
||||||
|
select(key: string): { revision: number; changed: boolean } {
|
||||||
|
const changed = this.key !== key;
|
||||||
|
if (changed) {
|
||||||
|
this.key = key;
|
||||||
|
this.revision++;
|
||||||
|
}
|
||||||
|
return { revision: this.revision, changed };
|
||||||
|
}
|
||||||
|
accepts(revision: number) {
|
||||||
|
return revision === this.revision && this.key !== "";
|
||||||
|
}
|
||||||
|
clear() {
|
||||||
|
this.key = "";
|
||||||
|
this.revision++;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
/** UI times use performance.now(); native spans use their own command clock. */
|
||||||
|
export interface NativeSpan {
|
||||||
|
name: string;
|
||||||
|
start_ms: number;
|
||||||
|
duration_ms: number;
|
||||||
|
parent: number | null;
|
||||||
|
counters: Record<string, number>;
|
||||||
|
}
|
||||||
|
export interface NativeMetrics {
|
||||||
|
total_ms: number;
|
||||||
|
result_count: number;
|
||||||
|
trace: NativeSpan[] | null;
|
||||||
|
dropped_spans: number;
|
||||||
|
}
|
||||||
|
export interface MeasuredResponse<T> {
|
||||||
|
results: T[];
|
||||||
|
metrics: NativeMetrics;
|
||||||
|
}
|
||||||
|
export type SearchLane = "lexical" | "hybrid";
|
||||||
|
export interface AttemptStart {
|
||||||
|
lane: SearchLane;
|
||||||
|
startedAt: number;
|
||||||
|
submittedAt: number;
|
||||||
|
submission: number;
|
||||||
|
}
|
||||||
|
interface Attempt {
|
||||||
|
lane: SearchLane;
|
||||||
|
submission: number;
|
||||||
|
attempt: number;
|
||||||
|
queued_ms: number;
|
||||||
|
retry_wait_ms: number;
|
||||||
|
invoke_ms: number;
|
||||||
|
native: NativeMetrics | null;
|
||||||
|
transport_estimate_ms: number | null;
|
||||||
|
failed: boolean;
|
||||||
|
ended_at: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SearchMeasurement {
|
||||||
|
debug: boolean;
|
||||||
|
inputAt: number;
|
||||||
|
scheduledAt: number;
|
||||||
|
context: Record<string, string | number | boolean | null>;
|
||||||
|
submissions = new Map<SearchLane, number>();
|
||||||
|
latestSubmission = new Map<SearchLane, number>();
|
||||||
|
submissionSequence = new Map<SearchLane, number>();
|
||||||
|
laneAttempts = new Map<SearchLane, number>();
|
||||||
|
attempts: Attempt[] = [];
|
||||||
|
droppedAttempts = 0;
|
||||||
|
droppedDisplays = 0;
|
||||||
|
firstDisplayMs: number | null = null;
|
||||||
|
displays: {
|
||||||
|
lane: SearchLane;
|
||||||
|
elapsed_ms: number;
|
||||||
|
dom_update_ms: number;
|
||||||
|
count: number;
|
||||||
|
}[] = [];
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
inputAt: number,
|
||||||
|
scheduledAt: number,
|
||||||
|
debug: boolean,
|
||||||
|
context: Record<string, string | number | boolean | null> = {},
|
||||||
|
) {
|
||||||
|
this.inputAt = inputAt;
|
||||||
|
this.scheduledAt = scheduledAt;
|
||||||
|
this.debug = debug;
|
||||||
|
this.context = { ...context };
|
||||||
|
}
|
||||||
|
submitted(lane: SearchLane, at: number) {
|
||||||
|
if (!this.submissions.has(lane)) this.submissions.set(lane, at);
|
||||||
|
this.latestSubmission.set(lane, at);
|
||||||
|
this.submissionSequence.set(
|
||||||
|
lane,
|
||||||
|
(this.submissionSequence.get(lane) ?? 0) + 1,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
begin(lane: SearchLane, startedAt: number): AttemptStart {
|
||||||
|
return {
|
||||||
|
lane,
|
||||||
|
startedAt,
|
||||||
|
submittedAt: this.latestSubmission.get(lane) ?? startedAt,
|
||||||
|
submission: this.submissionSequence.get(lane) ?? 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
completed(
|
||||||
|
token: AttemptStart,
|
||||||
|
end: number,
|
||||||
|
native: NativeMetrics | null,
|
||||||
|
failed: boolean,
|
||||||
|
) {
|
||||||
|
const { lane, startedAt: start, submittedAt, submission } = token;
|
||||||
|
const prior = this.attempts.filter((attempt) => attempt.lane === lane);
|
||||||
|
const previous = prior.at(-1);
|
||||||
|
const retry = previous?.failed && previous.submission === submission;
|
||||||
|
const attemptNumber = (this.laneAttempts.get(lane) ?? 0) + 1;
|
||||||
|
this.laneAttempts.set(lane, attemptNumber);
|
||||||
|
const elapsed = Math.max(0, end - start);
|
||||||
|
this.attempts.push({
|
||||||
|
lane,
|
||||||
|
submission,
|
||||||
|
attempt: attemptNumber,
|
||||||
|
queued_ms: retry ? 0 : Math.max(0, start - submittedAt),
|
||||||
|
retry_wait_ms: retry ? Math.max(0, start - previous.ended_at) : 0,
|
||||||
|
invoke_ms: elapsed,
|
||||||
|
native,
|
||||||
|
transport_estimate_ms: native
|
||||||
|
? Math.max(0, elapsed - native.total_ms)
|
||||||
|
: null,
|
||||||
|
failed,
|
||||||
|
ended_at: end,
|
||||||
|
});
|
||||||
|
if (this.attempts.length > (this.debug ? 32 : 4)) {
|
||||||
|
this.attempts.shift();
|
||||||
|
this.droppedAttempts++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
displayed(lane: SearchLane, start: number, end: number, count: number) {
|
||||||
|
if (this.firstDisplayMs === null)
|
||||||
|
this.firstDisplayMs = Math.max(0, end - this.inputAt);
|
||||||
|
this.displays.push({
|
||||||
|
lane,
|
||||||
|
elapsed_ms: Math.max(0, end - this.inputAt),
|
||||||
|
dom_update_ms: Math.max(0, end - start),
|
||||||
|
count,
|
||||||
|
});
|
||||||
|
if (this.displays.length > (this.debug ? 16 : 2)) {
|
||||||
|
this.displays.shift();
|
||||||
|
this.droppedDisplays++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
latestNative(lane: SearchLane) {
|
||||||
|
return (
|
||||||
|
this.attempts
|
||||||
|
.filter((attempt) => attempt.lane === lane && !attempt.failed)
|
||||||
|
.at(-1)?.native ?? null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
/** Deliberately exclude query text, filenames, paths, snippets and errors. */
|
||||||
|
export() {
|
||||||
|
return {
|
||||||
|
version: 1,
|
||||||
|
context: this.context,
|
||||||
|
clock_note:
|
||||||
|
"UI monotonic elapsed times; native command-relative spans. Nested spans and parallel lanes must not be summed.",
|
||||||
|
debounce_ms: Math.max(0, this.scheduledAt - this.inputAt),
|
||||||
|
lane_schedule_ms: Object.fromEntries(
|
||||||
|
[...this.submissions].map(([lane, at]) => [
|
||||||
|
lane,
|
||||||
|
Math.max(0, at - this.scheduledAt),
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
attempts: this.attempts.map(({ ended_at: _end, ...attempt }) => attempt),
|
||||||
|
displays: this.displays,
|
||||||
|
first_results_ms: this.firstDisplayMs,
|
||||||
|
latest_displayed_ms: this.displays.at(-1)?.elapsed_ms ?? null,
|
||||||
|
hybrid_final_results_ms:
|
||||||
|
[...this.displays]
|
||||||
|
.reverse()
|
||||||
|
.find((display) => display.lane === "hybrid")?.elapsed_ms ?? null,
|
||||||
|
dropped_attempts: this.droppedAttempts,
|
||||||
|
dropped_displays: this.droppedDisplays,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function measuredFailure(
|
||||||
|
error: unknown,
|
||||||
|
): { message: string; metrics: NativeMetrics } | null {
|
||||||
|
if (!error || typeof error !== "object") return null;
|
||||||
|
const value = error as { message?: unknown; metrics?: NativeMetrics };
|
||||||
|
return typeof value.message === "string" &&
|
||||||
|
value.metrics &&
|
||||||
|
Number.isFinite(value.metrics.total_ms)
|
||||||
|
? { message: value.message, metrics: value.metrics }
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Malformed/cyclic parent chains cannot trap the debug renderer. */
|
||||||
|
export function spanDepth(spans: NativeSpan[], index: number) {
|
||||||
|
let depth = 0;
|
||||||
|
const visited = new Set([index]);
|
||||||
|
let parent = spans[index]?.parent;
|
||||||
|
while (
|
||||||
|
parent != null &&
|
||||||
|
spans[parent] &&
|
||||||
|
!visited.has(parent) &&
|
||||||
|
depth < 16
|
||||||
|
) {
|
||||||
|
visited.add(parent);
|
||||||
|
depth++;
|
||||||
|
parent = spans[parent].parent;
|
||||||
|
}
|
||||||
|
return depth;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function briefMs(value: number) {
|
||||||
|
return value < 1
|
||||||
|
? "<1"
|
||||||
|
: value < 10
|
||||||
|
? value.toFixed(1)
|
||||||
|
: String(Math.round(value));
|
||||||
|
}
|
||||||
@@ -766,3 +766,73 @@ footer {
|
|||||||
outline-color: Highlight;
|
outline-color: Highlight;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/* Keep one scoped media player within the existing result preview. */
|
||||||
|
.media-preview {
|
||||||
|
margin: 0 20px 14px;
|
||||||
|
overflow: hidden;
|
||||||
|
border-radius: 10px;
|
||||||
|
background: #f3f4f7;
|
||||||
|
}
|
||||||
|
.media-preview img,
|
||||||
|
.media-preview video {
|
||||||
|
display: block;
|
||||||
|
width: 100%;
|
||||||
|
max-height: 340px;
|
||||||
|
object-fit: contain;
|
||||||
|
}
|
||||||
|
.media-preview audio {
|
||||||
|
display: block;
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
#media-preview-status {
|
||||||
|
margin: 0 20px 14px;
|
||||||
|
}
|
||||||
|
.file-icon[data-type="image"] {
|
||||||
|
color: #6c62b5;
|
||||||
|
}
|
||||||
|
.file-icon[data-type="video"] {
|
||||||
|
color: #a16a48;
|
||||||
|
}
|
||||||
|
.file-icon[data-type="audio"] {
|
||||||
|
color: #608271;
|
||||||
|
}
|
||||||
|
.performance-toggle {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
margin-top: 14px;
|
||||||
|
}
|
||||||
|
.performance-toggle input {
|
||||||
|
width: auto;
|
||||||
|
accent-color: var(--brand);
|
||||||
|
}
|
||||||
|
#performance-panel {
|
||||||
|
margin: 0 20px 12px;
|
||||||
|
padding: 10px 14px;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: 10px;
|
||||||
|
background: var(--surface);
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
#performance-panel summary {
|
||||||
|
cursor: pointer;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
#performance-detail {
|
||||||
|
max-height: 280px;
|
||||||
|
overflow: auto;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
#performance-detail p {
|
||||||
|
margin: 8px 0;
|
||||||
|
}
|
||||||
|
.performance-attempt {
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.performance-span {
|
||||||
|
border-left: 2px solid var(--line);
|
||||||
|
}
|
||||||
|
#copy-performance-status {
|
||||||
|
margin-left: 8px;
|
||||||
|
color: var(--muted);
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import test from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import {
|
||||||
|
timestampLabel,
|
||||||
|
seekSeconds,
|
||||||
|
sourceLabel,
|
||||||
|
matchLabel,
|
||||||
|
PreviewRevision,
|
||||||
|
} from "../src/media-display.ts";
|
||||||
|
import { LatestLane } from "../src/search-state.ts";
|
||||||
|
const settle = () => new Promise((resolve) => setImmediate(resolve));
|
||||||
|
|
||||||
|
test("millisecond seeks preserve fractions and readable minute labels", () => {
|
||||||
|
assert.equal(timestampLabel(125450), "2:05");
|
||||||
|
assert.equal(timestampLabel(3600000), "60:00");
|
||||||
|
assert.equal(seekSeconds({ kind: "timestamp_ms", number: 125450 }), 125.45);
|
||||||
|
assert.equal(seekSeconds({ kind: "page", number: 4 }), undefined);
|
||||||
|
assert.equal(seekSeconds({ kind: "timestamp_ms", number: -1 }), undefined);
|
||||||
|
assert.equal(timestampLabel(Infinity), "Unknown time");
|
||||||
|
assert.equal(sourceLabel({ kind: "image", number: 0 }), "Image");
|
||||||
|
assert.equal(matchLabel("semantic_audio"), "Audio meaning");
|
||||||
|
assert.equal(matchLabel("semantic_video"), "Video frame meaning");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("another timestamp in the same movie invalidates older preparations", () => {
|
||||||
|
const gate = new PreviewRevision();
|
||||||
|
const first = gate.select("movie:1000");
|
||||||
|
assert.equal(gate.select("movie:1000").changed, false);
|
||||||
|
const second = gate.select("movie:3000");
|
||||||
|
assert.equal(gate.accepts(first.revision), false);
|
||||||
|
assert.equal(gate.accepts(second.revision), true);
|
||||||
|
gate.clear();
|
||||||
|
assert.equal(gate.accepts(second.revision), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("one active preview and only latest pending reject stale output", async () => {
|
||||||
|
const gate = new PreviewRevision();
|
||||||
|
const executed = [];
|
||||||
|
const accepted = [];
|
||||||
|
let release;
|
||||||
|
const lane = new LatestLane(
|
||||||
|
async (request) => {
|
||||||
|
executed.push(request.path);
|
||||||
|
if (request.path === "first")
|
||||||
|
await new Promise((resolve) => {
|
||||||
|
release = resolve;
|
||||||
|
});
|
||||||
|
if (gate.accepts(request.revision)) accepted.push(request.path);
|
||||||
|
},
|
||||||
|
(request) => gate.accepts(request.revision),
|
||||||
|
() => assert.fail("Unexpected preview failure"),
|
||||||
|
);
|
||||||
|
lane.submit({ path: "first", revision: gate.select("first").revision });
|
||||||
|
lane.submit({ path: "middle", revision: gate.select("middle").revision });
|
||||||
|
lane.submit({ path: "latest", revision: gate.select("latest").revision });
|
||||||
|
release();
|
||||||
|
await settle();
|
||||||
|
assert.deepEqual(executed, ["first", "latest"]);
|
||||||
|
assert.deepEqual(accepted, ["latest"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Escape clears queued preparation and rejects active output", async () => {
|
||||||
|
const gate = new PreviewRevision();
|
||||||
|
let release;
|
||||||
|
const accepted = [];
|
||||||
|
const lane = new LatestLane(
|
||||||
|
async (request) => {
|
||||||
|
await new Promise((resolve) => {
|
||||||
|
release = resolve;
|
||||||
|
});
|
||||||
|
if (gate.accepts(request.revision)) accepted.push(request.path);
|
||||||
|
},
|
||||||
|
(request) => gate.accepts(request.revision),
|
||||||
|
() => assert.fail("Unexpected preview failure"),
|
||||||
|
);
|
||||||
|
lane.submit({ path: "one", revision: gate.select("one").revision });
|
||||||
|
lane.submit({ path: "two", revision: gate.select("two").revision });
|
||||||
|
gate.clear();
|
||||||
|
lane.clear();
|
||||||
|
release();
|
||||||
|
await settle();
|
||||||
|
assert.deepEqual(accepted, []);
|
||||||
|
});
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { SearchMeasurement, measuredFailure, spanDepth, briefMs } from '../src/search-performance.ts';
|
||||||
|
|
||||||
|
const finish = (trace, lane, start, end, metrics, failed) => trace.completed(trace.begin(lane, start), end, metrics, failed);
|
||||||
|
|
||||||
|
const native = { total_ms: 12, result_count: 2, trace: [], dropped_spans: 0 };
|
||||||
|
|
||||||
|
test('separates debounce, intentional semantic delay, queue, invoke and DOM update', () => {
|
||||||
|
const trace = new SearchMeasurement(100, 165, true);
|
||||||
|
trace.submitted('lexical', 165);
|
||||||
|
finish(trace, 'lexical', 180, 200, native, false);
|
||||||
|
trace.displayed('lexical', 200, 203, 2);
|
||||||
|
trace.submitted('hybrid', 515);
|
||||||
|
finish(trace, 'hybrid', 520, 550, native, false);
|
||||||
|
trace.displayed('hybrid', 550, 554, 3);
|
||||||
|
const report = trace.export();
|
||||||
|
assert.equal(report.debounce_ms, 65);
|
||||||
|
assert.equal(report.lane_schedule_ms.hybrid, 350);
|
||||||
|
assert.equal(report.attempts[0].queued_ms, 15);
|
||||||
|
assert.equal(report.attempts[0].invoke_ms, 20);
|
||||||
|
assert.equal(report.attempts[0].transport_estimate_ms, 8);
|
||||||
|
assert.equal(report.displays[0].dom_update_ms, 3);
|
||||||
|
assert.equal(report.first_results_ms, 103);
|
||||||
|
assert.equal(report.hybrid_final_results_ms, 454);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('failed attempt timings survive retry and lexical response does not pretend hybrid completion', () => {
|
||||||
|
const trace = new SearchMeasurement(0, 0, true);
|
||||||
|
trace.submitted('lexical', 0);
|
||||||
|
finish(trace, 'lexical', 0, 2, native, true);
|
||||||
|
finish(trace, 'lexical', 402, 414, native, false);
|
||||||
|
trace.displayed('lexical', 414, 415, 2);
|
||||||
|
const report = trace.export();
|
||||||
|
assert.equal(report.attempts[1].retry_wait_ms, 400);
|
||||||
|
assert.equal(report.attempts[0].failed, true);
|
||||||
|
assert.equal(report.attempts[0].transport_estimate_ms, 0);
|
||||||
|
assert.equal(report.hybrid_final_results_ms, null);
|
||||||
|
assert.equal(report.latest_displayed_ms, 415);
|
||||||
|
assert.equal(measuredFailure({message: 'worker busy', metrics: native}).message, 'worker busy');
|
||||||
|
assert.equal(measuredFailure('legacy failure'), null);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('debug history is bounded and first response survives dropped displays', () => {
|
||||||
|
const context = { device: 'cpu', threads: 1 };
|
||||||
|
const trace = new SearchMeasurement(0, 0, true, context);
|
||||||
|
context.threads = 8;
|
||||||
|
for (let i = 0; i < 40; i++) {
|
||||||
|
finish(trace, 'hybrid', i, i + 1, native, false);
|
||||||
|
trace.displayed('hybrid', i, i + 1, 1);
|
||||||
|
}
|
||||||
|
const report = trace.export();
|
||||||
|
assert.equal(report.attempts.length, 32);
|
||||||
|
assert.equal(report.displays.length, 16);
|
||||||
|
assert.equal(report.dropped_attempts, 8);
|
||||||
|
assert.equal(report.dropped_displays, 24);
|
||||||
|
assert.equal(report.first_results_ms, 1);
|
||||||
|
assert.equal(report.context.threads, 1);
|
||||||
|
assert.doesNotMatch(JSON.stringify(report), /query|filename|snippet|ended_at/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('cyclic or invalid native span parents cannot hang hierarchy rendering', () => {
|
||||||
|
const spans = [{parent: 1}, {parent: 0}, {parent: 99}];
|
||||||
|
assert.equal(spanDepth(spans, 0), 1);
|
||||||
|
assert.equal(spanDepth(spans, 2), 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a later refresh has queue timing rather than being misreported as a busy retry', () => {
|
||||||
|
const trace = new SearchMeasurement(0, 0, true);
|
||||||
|
trace.submitted('hybrid', 350);
|
||||||
|
finish(trace, 'hybrid', 350, 360, native, false);
|
||||||
|
trace.submitted('hybrid', 1000);
|
||||||
|
finish(trace, 'hybrid', 1010, 1020, native, false);
|
||||||
|
assert.equal(trace.export().attempts[1].queued_ms, 10);
|
||||||
|
assert.equal(trace.export().attempts[1].retry_wait_ms, 0);
|
||||||
|
assert.equal(briefMs(.44), '<1');
|
||||||
|
assert.equal(briefMs(3.14), '3.1');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('new submission after a failure measures pending queue rather than retry wait', () => {
|
||||||
|
const trace = new SearchMeasurement(0, 0, true);
|
||||||
|
trace.submitted('hybrid', 350);
|
||||||
|
finish(trace, 'hybrid', 350, 360, native, true);
|
||||||
|
trace.submitted('hybrid', 1000);
|
||||||
|
finish(trace, 'hybrid', 1010, 1020, native, false);
|
||||||
|
assert.equal(trace.export().attempts[1].queued_ms, 10);
|
||||||
|
assert.equal(trace.export().attempts[1].retry_wait_ms, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('an in-flight attempt keeps its captured submission when a refresh queues', () => {
|
||||||
|
const trace = new SearchMeasurement(0, 0, true);
|
||||||
|
trace.submitted('hybrid', 350);
|
||||||
|
const active = trace.begin('hybrid', 355);
|
||||||
|
trace.submitted('hybrid', 1000);
|
||||||
|
trace.completed(active, 1005, native, true);
|
||||||
|
finish(trace, 'hybrid', 1010, 1020, native, false);
|
||||||
|
const attempts = trace.export().attempts;
|
||||||
|
assert.equal(attempts[0].queued_ms, 5);
|
||||||
|
assert.equal(attempts[1].queued_ms, 10);
|
||||||
|
assert.equal(attempts[1].retry_wait_ms, 0);
|
||||||
|
assert.notEqual(attempts[0].submission, attempts[1].submission);
|
||||||
|
});
|
||||||
@@ -1,9 +1,10 @@
|
|||||||
//! Thin command-line adapter around the shared, CPU-only search engine.
|
//! Thin headless adapter around the shared local search engine.
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
use anyhow::{bail, Context, Result};
|
use anyhow::{bail, Context, Result};
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
|
use searchi_core::metrics::{MetricsMode, SearchResponse, TraceCollector};
|
||||||
use searchi_core::{IndexOptions, SearchIndex, SearchOptions};
|
use searchi_core::{IndexOptions, SearchIndex, SearchOptions};
|
||||||
use searchi_service::HybridSearch;
|
use searchi_service::HybridSearch;
|
||||||
|
|
||||||
@@ -46,6 +47,12 @@ enum Command {
|
|||||||
/// Emit a JSON array for scripting.
|
/// Emit a JSON array for scripting.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
json: bool,
|
json: bool,
|
||||||
|
/// Include basic measured statistics in JSON output.
|
||||||
|
#[arg(long)]
|
||||||
|
performance: bool,
|
||||||
|
/// Include bounded granular stage timings and diagnostic counters.
|
||||||
|
#[arg(long)]
|
||||||
|
debug_performance: bool,
|
||||||
/// Restrict results to this folder and descendants.
|
/// Restrict results to this folder and descendants.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
folder: Option<PathBuf>,
|
folder: Option<PathBuf>,
|
||||||
@@ -56,6 +63,11 @@ enum Command {
|
|||||||
min_size: Option<u64>,
|
min_size: Option<u64>,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
max_size: Option<u64>,
|
max_size: Option<u64>,
|
||||||
|
/// Inclusive media duration limit in milliseconds.
|
||||||
|
#[arg(long)]
|
||||||
|
min_duration_ms: Option<u32>,
|
||||||
|
#[arg(long)]
|
||||||
|
max_duration_ms: Option<u32>,
|
||||||
/// Inclusive modification time in Unix milliseconds.
|
/// Inclusive modification time in Unix milliseconds.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
modified_after_ms: Option<i64>,
|
modified_after_ms: Option<i64>,
|
||||||
@@ -80,6 +92,14 @@ enum Command {
|
|||||||
embedding_worker: PathBuf,
|
embedding_worker: PathBuf,
|
||||||
#[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u8).range(1..=8))]
|
#[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u8).range(1..=8))]
|
||||||
cpu_threads: u8,
|
cpu_threads: u8,
|
||||||
|
/// Standalone decoder; normally discovered beside the installed CLI.
|
||||||
|
#[arg(long, requires = "ffprobe")]
|
||||||
|
ffmpeg: Option<PathBuf>,
|
||||||
|
#[arg(long, requires = "ffmpeg")]
|
||||||
|
ffprobe: Option<PathBuf>,
|
||||||
|
/// Explicitly build only document vectors, without media.
|
||||||
|
#[arg(long)]
|
||||||
|
text_only: bool,
|
||||||
},
|
},
|
||||||
/// Show the indexed file and content counts.
|
/// Show the indexed file and content counts.
|
||||||
Stats {
|
Stats {
|
||||||
@@ -122,10 +142,14 @@ fn main() -> Result<()> {
|
|||||||
query,
|
query,
|
||||||
limit,
|
limit,
|
||||||
json,
|
json,
|
||||||
|
performance,
|
||||||
|
debug_performance,
|
||||||
folder,
|
folder,
|
||||||
extension,
|
extension,
|
||||||
min_size,
|
min_size,
|
||||||
max_size,
|
max_size,
|
||||||
|
min_duration_ms,
|
||||||
|
max_duration_ms,
|
||||||
modified_after_ms,
|
modified_after_ms,
|
||||||
modified_before_ms,
|
modified_before_ms,
|
||||||
model_dir,
|
model_dir,
|
||||||
@@ -141,30 +165,56 @@ fn main() -> Result<()> {
|
|||||||
extension,
|
extension,
|
||||||
min_size,
|
min_size,
|
||||||
max_size,
|
max_size,
|
||||||
|
min_duration_ms,
|
||||||
|
max_duration_ms,
|
||||||
modified_after_unix_ms: modified_after_ms,
|
modified_after_unix_ms: modified_after_ms,
|
||||||
modified_before_unix_ms: modified_before_ms,
|
modified_before_unix_ms: modified_before_ms,
|
||||||
};
|
};
|
||||||
|
let mut trace = TraceCollector::new(if debug_performance {
|
||||||
|
MetricsMode::Debug
|
||||||
|
} else {
|
||||||
|
MetricsMode::Basic
|
||||||
|
});
|
||||||
let results = if let (Some(model), Some(runtime)) = (model_dir, embedding_worker) {
|
let results = if let (Some(model), Some(runtime)) = (model_dir, embedding_worker) {
|
||||||
HybridSearch::open(&db, model, runtime, usize::from(cpu_threads))?
|
let mut service = trace.timed("cli.session_initialize", None, |_, _| {
|
||||||
.search(&query, usize::from(limit), &options)
|
HybridSearch::open(&db, model, runtime, usize::from(cpu_threads))
|
||||||
|
})?;
|
||||||
|
service
|
||||||
|
.search_traced(&query, usize::from(limit), &options, &mut trace, None)
|
||||||
.context("Hybrid search failed")?
|
.context("Hybrid search failed")?
|
||||||
} else {
|
} else {
|
||||||
SearchIndex::open(&db)
|
let index = trace
|
||||||
.with_context(|| format!("Cannot open index {}", db.display()))?
|
.timed("cli.index_open", None, |_, _| SearchIndex::open(&db))
|
||||||
.search_with_options(&query, usize::from(limit), &options)
|
.with_context(|| format!("Cannot open index {}", db.display()))?;
|
||||||
|
index
|
||||||
|
.search_with_options_traced(
|
||||||
|
&query,
|
||||||
|
usize::from(limit),
|
||||||
|
&options,
|
||||||
|
&mut trace,
|
||||||
|
None,
|
||||||
|
)
|
||||||
.context("Search failed")?
|
.context("Search failed")?
|
||||||
};
|
};
|
||||||
|
let response = SearchResponse {
|
||||||
|
metrics: trace.finish(results.len()),
|
||||||
|
results,
|
||||||
|
};
|
||||||
if json {
|
if json {
|
||||||
println!("{}", serde_json::to_string_pretty(&results)?);
|
if performance || debug_performance {
|
||||||
} else if results.is_empty() {
|
println!("{}", serde_json::to_string_pretty(&response)?);
|
||||||
|
} else {
|
||||||
|
println!("{}", serde_json::to_string_pretty(&response.results)?);
|
||||||
|
}
|
||||||
|
} else if response.results.is_empty() {
|
||||||
println!("No matching files.");
|
println!("No matching files.");
|
||||||
} else {
|
} else {
|
||||||
for result in results {
|
for result in &response.results {
|
||||||
println!("{} [{}]", result.path, result.match_kind);
|
println!("{} [{}]", result.path, result.match_kind);
|
||||||
if let Some(source) = result.source {
|
if let Some(source) = &result.source {
|
||||||
println!(" {} {}", source.kind, source.number);
|
println!(" {} {}", source.kind, source.number);
|
||||||
}
|
}
|
||||||
if let Some(error) = result.extraction_error {
|
if let Some(error) = &result.extraction_error {
|
||||||
println!(" Text extraction unavailable: {error}");
|
println!(" Text extraction unavailable: {error}");
|
||||||
}
|
}
|
||||||
if !result.snippet.is_empty() {
|
if !result.snippet.is_empty() {
|
||||||
@@ -172,18 +222,36 @@ fn main() -> Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !json {
|
||||||
|
eprintln!(
|
||||||
|
"{} results · {:.2} ms (includes CLI initialization)",
|
||||||
|
response.metrics.result_count, response.metrics.total_ms
|
||||||
|
);
|
||||||
|
if debug_performance {
|
||||||
|
eprintln!("{}", serde_json::to_string_pretty(&response.metrics)?);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Command::Embed {
|
Command::Embed {
|
||||||
db,
|
db,
|
||||||
model_dir,
|
model_dir,
|
||||||
embedding_worker,
|
embedding_worker,
|
||||||
cpu_threads,
|
cpu_threads,
|
||||||
|
ffmpeg,
|
||||||
|
ffprobe,
|
||||||
|
text_only,
|
||||||
} => {
|
} => {
|
||||||
require_existing_index(&db)?;
|
require_existing_index(&db)?;
|
||||||
let report =
|
let mut service =
|
||||||
HybridSearch::open(&db, model_dir, embedding_worker, usize::from(cpu_threads))?
|
HybridSearch::open(&db, model_dir, embedding_worker, usize::from(cpu_threads))?;
|
||||||
.index_embeddings()
|
let report = if text_only {
|
||||||
.context("Semantic indexing failed; prior generation remains active")?;
|
service.index_embeddings()
|
||||||
|
} else {
|
||||||
|
let tools = resolve_media_tools(ffmpeg, ffprobe)?;
|
||||||
|
service
|
||||||
|
.index_embeddings_with_media(&tools, &searchi_service::MediaLimits::default())
|
||||||
|
}
|
||||||
|
.context("Semantic indexing failed; prior generation remains active")?;
|
||||||
println!("{}", serde_json::to_string_pretty(&report)?);
|
println!("{}", serde_json::to_string_pretty(&report)?);
|
||||||
}
|
}
|
||||||
Command::Stats { db, json } => {
|
Command::Stats { db, json } => {
|
||||||
@@ -204,6 +272,32 @@ fn main() -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn resolve_media_tools(
|
||||||
|
ffmpeg: Option<PathBuf>,
|
||||||
|
ffprobe: Option<PathBuf>,
|
||||||
|
) -> Result<searchi_service::FfmpegTools> {
|
||||||
|
if let (Some(ffmpeg), Some(ffprobe)) = (ffmpeg, ffprobe) {
|
||||||
|
return searchi_service::FfmpegTools::new(ffmpeg, ffprobe);
|
||||||
|
}
|
||||||
|
if let Some(tools) = searchi_service::FfmpegTools::from_environment()? {
|
||||||
|
return Ok(tools);
|
||||||
|
}
|
||||||
|
let development =
|
||||||
|
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.searchi/media-runtime/bin");
|
||||||
|
let directory = if cfg!(debug_assertions) && development.is_dir() {
|
||||||
|
development
|
||||||
|
} else {
|
||||||
|
std::env::current_exe()?
|
||||||
|
.parent()
|
||||||
|
.context("CLI installation directory is unavailable")?
|
||||||
|
.join("media-runtime/bin")
|
||||||
|
};
|
||||||
|
searchi_service::FfmpegTools::new(
|
||||||
|
directory.join(if cfg!(windows) { "ffmpeg.exe" } else { "ffmpeg" }),
|
||||||
|
directory.join(if cfg!(windows) { "ffprobe.exe" } else { "ffprobe" }),
|
||||||
|
).context("Media tools are missing from this installation; repair the headless package or explicitly use --text-only")
|
||||||
|
}
|
||||||
|
|
||||||
/// Read commands must not silently create a new empty database after a typo.
|
/// Read commands must not silently create a new empty database after a typo.
|
||||||
fn require_existing_index(db: &std::path::Path) -> Result<()> {
|
fn require_existing_index(db: &std::path::Path) -> Result<()> {
|
||||||
if !db.is_file() {
|
if !db.is_file() {
|
||||||
|
|||||||
@@ -68,6 +68,36 @@ fn indexes_persists_and_returns_search_and_stats_json() {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
.contains("termination"));
|
.contains("termination"));
|
||||||
|
|
||||||
|
for flag in ["--performance", "--debug-performance"] {
|
||||||
|
let measured = fixture.run(&["search", "--db", "index.db", "termination", "--json", flag]);
|
||||||
|
assert!(
|
||||||
|
measured.status.success(),
|
||||||
|
"{}",
|
||||||
|
String::from_utf8_lossy(&measured.stderr)
|
||||||
|
);
|
||||||
|
let envelope: serde_json::Value = serde_json::from_slice(&measured.stdout).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
envelope["results"],
|
||||||
|
serde_json::Value::Array(results.clone())
|
||||||
|
);
|
||||||
|
assert_eq!(envelope["metrics"]["result_count"], 1);
|
||||||
|
let total = envelope["metrics"]["total_ms"].as_f64().unwrap();
|
||||||
|
assert!(total.is_finite() && total >= 0.0);
|
||||||
|
if flag == "--performance" {
|
||||||
|
assert!(envelope["metrics"]["trace"].is_null());
|
||||||
|
} else {
|
||||||
|
let spans = envelope["metrics"]["trace"].as_array().unwrap();
|
||||||
|
assert!(spans.iter().any(|span| span["name"] == "cli.index_open"));
|
||||||
|
assert!(spans.len() > 2);
|
||||||
|
for span in spans {
|
||||||
|
let start = span["start_ms"].as_f64().unwrap();
|
||||||
|
let duration = span["duration_ms"].as_f64().unwrap();
|
||||||
|
assert!(start >= 0.0 && duration >= 0.0 && start + duration <= total + 0.001);
|
||||||
|
assert!(!span["name"].as_str().unwrap().contains("termination"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let stats = fixture.run(&["stats", "--db", "index.db", "--json"]);
|
let stats = fixture.run(&["stats", "--db", "index.db", "--json"]);
|
||||||
assert!(stats.status.success());
|
assert!(stats.status.success());
|
||||||
let stats: serde_json::Value = serde_json::from_slice(&stats.stdout).unwrap();
|
let stats: serde_json::Value = serde_json::from_slice(&stats.stdout).unwrap();
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
//! Small prepared SQLite instrumentation diagnostic; not a production benchmark.
|
||||||
|
use anyhow::{ensure, Result};
|
||||||
|
use searchi_core::SearchIndex;
|
||||||
|
use std::{fs, path::Path, time::Instant};
|
||||||
|
|
||||||
|
fn distribution(samples: &[f64]) -> serde_json::Value {
|
||||||
|
let mut sorted = samples.to_vec();
|
||||||
|
sorted.sort_by(f64::total_cmp);
|
||||||
|
let percentile = |fraction: f64| {
|
||||||
|
sorted[((sorted.len() as f64 * fraction).ceil() as usize).saturating_sub(1)]
|
||||||
|
};
|
||||||
|
serde_json::json!({"samples":sorted.len(),"p50_ms":percentile(0.50),"p95_ms":percentile(0.95),"mean_ms":sorted.iter().sum::<f64>()/sorted.len() as f64})
|
||||||
|
}
|
||||||
|
fn main() -> Result<()> {
|
||||||
|
let fixture = tempfile::tempdir()?;
|
||||||
|
let root = fixture.path().join("files");
|
||||||
|
fs::create_dir(&root)?;
|
||||||
|
for index in 0..1000 {
|
||||||
|
fs::write(
|
||||||
|
root.join(format!("record-{index:04}.txt")),
|
||||||
|
format!(
|
||||||
|
"Supplier agreement {index}. Cancellation needs written notice. Group {}.",
|
||||||
|
index % 31
|
||||||
|
),
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
let mut index = SearchIndex::open(fixture.path().join("index.sqlite"))?;
|
||||||
|
let report = index.index_folder(&root)?;
|
||||||
|
index.prewarm()?;
|
||||||
|
ensure!(report.indexed == 1000, "Unexpected fixture indexing count");
|
||||||
|
let queries: Vec<String> = (0..24)
|
||||||
|
.map(|index| format!("record-{:04}.txt", index * 37))
|
||||||
|
.collect();
|
||||||
|
for query in &queries {
|
||||||
|
index.search_measured(query, 10, false)?;
|
||||||
|
index.search_measured(query, 10, true)?;
|
||||||
|
}
|
||||||
|
let mut basic = Vec::new();
|
||||||
|
let mut debug = Vec::new();
|
||||||
|
let mut trace_spans = Vec::new();
|
||||||
|
// Alternate order to reduce monotonic cache/CPU-order effects.
|
||||||
|
for repetition in 0..20 {
|
||||||
|
for query in &queries {
|
||||||
|
for detailed in if repetition % 2 == 0 {
|
||||||
|
[false, true]
|
||||||
|
} else {
|
||||||
|
[true, false]
|
||||||
|
} {
|
||||||
|
let started = Instant::now();
|
||||||
|
let response = index.search_measured(query, 10, detailed)?;
|
||||||
|
let elapsed = started.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
ensure!(
|
||||||
|
response
|
||||||
|
.results
|
||||||
|
.first()
|
||||||
|
.is_some_and(|result| &result.filename == query),
|
||||||
|
"Fixture retrieval changed"
|
||||||
|
);
|
||||||
|
if detailed {
|
||||||
|
trace_spans.push(response.metrics.trace.as_ref().unwrap().len());
|
||||||
|
debug.push(elapsed);
|
||||||
|
} else {
|
||||||
|
ensure!(
|
||||||
|
response.metrics.trace.is_none(),
|
||||||
|
"Basic unexpectedly stored traces"
|
||||||
|
);
|
||||||
|
basic.push(elapsed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let output = serde_json::json!({"kind":"prepared_sqlite_trace_overhead","production_claim":false,"profile":if cfg!(debug_assertions){"debug"}else{"release"},"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"files":1000,"queries":24,"repetitions":20,"measurement":"outer Rust call including metrics assembly; excludes fixture setup/JSON serialization","basic":distribution(&basic),"debug":distribution(&debug),"basic_samples_ms":basic,"debug_samples_ms":debug,"debug_span_count_min":trace_spans.iter().min(),"debug_span_count_max":trace_spans.iter().max(),"limits":"tiny warm lexical corpus; harness serial/no model/media/GPU; other VM work including native photo inference may contend; not isolated production performance"});
|
||||||
|
let serialized = serde_json::to_string_pretty(&output)?;
|
||||||
|
if let Some(path) = std::env::args_os().nth(1) {
|
||||||
|
fs::write(Path::new(&path), serialized)?;
|
||||||
|
} else {
|
||||||
|
println!("{serialized}");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -4,7 +4,10 @@
|
|||||||
//! Text and document extraction have byte limits, but still run in this process.
|
//! Text and document extraction have byte limits, but still run in this process.
|
||||||
//! Worker isolation remains a production milestone.
|
//! Worker isolation remains a production milestone.
|
||||||
mod extraction;
|
mod extraction;
|
||||||
|
pub mod metrics;
|
||||||
use anyhow::{bail, Context, Result};
|
use anyhow::{bail, Context, Result};
|
||||||
|
pub use metrics::{MetricsMode, SearchMetrics, SearchResponse, SearchSpan};
|
||||||
|
use metrics::{SpanToken, TraceCollector};
|
||||||
use rusqlite::{params, params_from_iter, types::Value, Connection, OptionalExtension};
|
use rusqlite::{params, params_from_iter, types::Value, Connection, OptionalExtension};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use std::{
|
use std::{
|
||||||
@@ -16,7 +19,7 @@ use std::{
|
|||||||
};
|
};
|
||||||
use walkdir::WalkDir;
|
use walkdir::WalkDir;
|
||||||
|
|
||||||
const SCHEMA_VERSION: i64 = 9;
|
const SCHEMA_VERSION: i64 = 10;
|
||||||
const MAX_RESULTS: usize = 1000;
|
const MAX_RESULTS: usize = 1000;
|
||||||
|
|
||||||
#[derive(Debug, Default, Serialize)]
|
#[derive(Debug, Default, Serialize)]
|
||||||
@@ -53,6 +56,8 @@ pub struct SearchOptions {
|
|||||||
pub max_size: Option<u64>,
|
pub max_size: Option<u64>,
|
||||||
pub modified_after_unix_ms: Option<i64>,
|
pub modified_after_unix_ms: Option<i64>,
|
||||||
pub modified_before_unix_ms: Option<i64>,
|
pub modified_before_unix_ms: Option<i64>,
|
||||||
|
pub min_duration_ms: Option<u32>,
|
||||||
|
pub max_duration_ms: Option<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
@@ -66,7 +71,8 @@ pub struct SearchResult {
|
|||||||
pub extraction_error: Option<String>,
|
pub extraction_error: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A verified matching passage location; numbers are one based.
|
/// Verified source location. Document numbers are one based; timestamp_ms
|
||||||
|
/// numbers are milliseconds from media start, and image numbers are zero.
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
pub struct SourceLocation {
|
pub struct SourceLocation {
|
||||||
pub kind: String,
|
pub kind: String,
|
||||||
@@ -227,6 +233,15 @@ impl SearchIndex {
|
|||||||
UPDATE name_statistics_state SET dirty=1 WHERE singleton=1; END;
|
UPDATE name_statistics_state SET dirty=1 WHERE singleton=1; END;
|
||||||
PRAGMA user_version=9;",
|
PRAGMA user_version=9;",
|
||||||
)?;
|
)?;
|
||||||
|
initialization.execute_batch("CREATE INDEX IF NOT EXISTS files_extension ON files(extension,id);
|
||||||
|
CREATE TABLE IF NOT EXISTS media_metadata(path TEXT PRIMARY KEY, file_id INTEGER NOT NULL UNIQUE, tags TEXT NOT NULL,duration_ms INTEGER NOT NULL);
|
||||||
|
CREATE VIRTUAL TABLE IF NOT EXISTS media_tags USING fts5(tags,content='media_metadata',content_rowid='file_id');
|
||||||
|
CREATE TRIGGER IF NOT EXISTS media_metadata_insert AFTER INSERT ON media_metadata BEGIN INSERT INTO media_tags(rowid,tags) VALUES(new.file_id,new.tags); END;
|
||||||
|
CREATE TRIGGER IF NOT EXISTS media_metadata_delete AFTER DELETE ON media_metadata BEGIN INSERT INTO media_tags(media_tags,rowid,tags) VALUES('delete',old.file_id,old.tags); END;
|
||||||
|
CREATE TRIGGER IF NOT EXISTS media_metadata_update AFTER UPDATE ON media_metadata BEGIN INSERT INTO media_tags(media_tags,rowid,tags) VALUES('delete',old.file_id,old.tags); INSERT INTO media_tags(rowid,tags) VALUES(new.file_id,new.tags); END;
|
||||||
|
CREATE TRIGGER IF NOT EXISTS files_media_delete AFTER DELETE ON files BEGIN DELETE FROM media_metadata WHERE path=old.path; END;
|
||||||
|
CREATE TRIGGER IF NOT EXISTS files_media_update AFTER UPDATE ON files WHEN new.path!=old.path OR new.size!=old.size OR new.mtime!=old.mtime BEGIN DELETE FROM media_metadata WHERE path=old.path; END;
|
||||||
|
PRAGMA user_version=10;")?;
|
||||||
initialization.commit()?;
|
initialization.commit()?;
|
||||||
}
|
}
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
@@ -236,6 +251,32 @@ impl SearchIndex {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Persist genuine extracted media tags, guarded against filesystem changes.
|
||||||
|
pub fn set_media_metadata(
|
||||||
|
&mut self,
|
||||||
|
path: &str,
|
||||||
|
tags: &str,
|
||||||
|
duration_ms: u32,
|
||||||
|
size: u64,
|
||||||
|
mtime_ns: i64,
|
||||||
|
) -> Result<()> {
|
||||||
|
if tags.len() > 32768 {
|
||||||
|
bail!("Media tags exceed 32 KiB bound");
|
||||||
|
}
|
||||||
|
let tx = self.connection.transaction()?;
|
||||||
|
let id: Option<i64> = tx
|
||||||
|
.query_row(
|
||||||
|
"SELECT id FROM files WHERE path=?1 AND size=?2 AND mtime=?3",
|
||||||
|
params![path, i64::try_from(size)?, mtime_ns],
|
||||||
|
|r| r.get(0),
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
let id = id.context("Media metadata file changed since indexing")?;
|
||||||
|
tx.execute("INSERT INTO media_metadata(path,file_id,tags,duration_ms) VALUES(?1,?2,?3,?4) ON CONFLICT(path) DO UPDATE SET tags=excluded.tags,duration_ms=excluded.duration_ms",params![path,id,tags,duration_ms])?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Refresh candidate-frequency statistics off the foreground search path.
|
/// Refresh candidate-frequency statistics off the foreground search path.
|
||||||
/// Stale/missing estimates can affect speed, never literal-match correctness.
|
/// Stale/missing estimates can affect speed, never literal-match correctness.
|
||||||
/// The app invokes this during background startup and after reconciliation.
|
/// The app invokes this during background startup and after reconciliation.
|
||||||
@@ -465,6 +506,56 @@ impl SearchIndex {
|
|||||||
limit: usize,
|
limit: usize,
|
||||||
options: &SearchOptions,
|
options: &SearchOptions,
|
||||||
) -> Result<Vec<SearchResult>> {
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Basic);
|
||||||
|
self.search_with_options_traced(query, limit, options, &mut trace, None)
|
||||||
|
}
|
||||||
|
pub fn search_measured(
|
||||||
|
&self,
|
||||||
|
query: &str,
|
||||||
|
limit: usize,
|
||||||
|
debug: bool,
|
||||||
|
) -> Result<SearchResponse<SearchResult>> {
|
||||||
|
self.search_with_options_measured(query, limit, &SearchOptions::default(), debug)
|
||||||
|
}
|
||||||
|
pub fn search_with_options_measured(
|
||||||
|
&self,
|
||||||
|
query: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
debug: bool,
|
||||||
|
) -> Result<SearchResponse<SearchResult>> {
|
||||||
|
let mut trace = TraceCollector::new(if debug {
|
||||||
|
MetricsMode::Debug
|
||||||
|
} else {
|
||||||
|
MetricsMode::Basic
|
||||||
|
});
|
||||||
|
let results = self.search_with_options_traced(query, limit, options, &mut trace, None)?;
|
||||||
|
let metrics = trace.finish(results.len());
|
||||||
|
Ok(SearchResponse { results, metrics })
|
||||||
|
}
|
||||||
|
/// Share one monotonic request collector with downstream retrieval lanes.
|
||||||
|
pub fn search_with_options_traced(
|
||||||
|
&self,
|
||||||
|
query: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
trace: &mut TraceCollector,
|
||||||
|
parent: Option<SpanToken>,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
trace.timed("lexical.search", parent, |trace, parent| {
|
||||||
|
self.search_inner(query, limit, options, trace, parent)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
fn search_inner(
|
||||||
|
&self,
|
||||||
|
query: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
trace: &mut TraceCollector,
|
||||||
|
parent: Option<SpanToken>,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
let normalization = trace.begin("lexical.normalization", parent);
|
||||||
|
|
||||||
if options
|
if options
|
||||||
.min_size
|
.min_size
|
||||||
.zip(options.max_size)
|
.zip(options.max_size)
|
||||||
@@ -479,6 +570,13 @@ impl SearchIndex {
|
|||||||
{
|
{
|
||||||
bail!("modified-after exceeds modified-before");
|
bail!("modified-after exceeds modified-before");
|
||||||
}
|
}
|
||||||
|
if options
|
||||||
|
.min_duration_ms
|
||||||
|
.zip(options.max_duration_ms)
|
||||||
|
.is_some_and(|(a, b)| a > b)
|
||||||
|
{
|
||||||
|
bail!("Minimum duration exceeds maximum duration");
|
||||||
|
}
|
||||||
let folder = options
|
let folder = options
|
||||||
.folder
|
.folder
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -518,11 +616,16 @@ impl SearchIndex {
|
|||||||
.map(|term| format!("\"{}\"", term.replace('"', "\"\"")))
|
.map(|term| format!("\"{}\"", term.replace('"', "\"\"")))
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(" AND ");
|
.join(" AND ");
|
||||||
|
trace.counter(normalization, "query_bytes", query.len() as u64);
|
||||||
|
trace.end(normalization, Default::default());
|
||||||
|
let planning = trace.begin("lexical.candidate_plan", parent);
|
||||||
let limit = limit.min(MAX_RESULTS);
|
let limit = limit.min(MAX_RESULTS);
|
||||||
let filters = " AND (?3 IS NULL OR substr(f.path,1,length(?3))=?3)
|
let filters = " AND (?3 IS NULL OR substr(f.path,1,length(?3))=?3)
|
||||||
AND (?4 IS NULL OR f.extension=?4)
|
AND (?4 IS NULL OR f.extension=?4)
|
||||||
AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6)
|
AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6)
|
||||||
AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8)";
|
AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8)
|
||||||
|
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms>=?10))
|
||||||
|
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))";
|
||||||
let filter_values = [
|
let filter_values = [
|
||||||
folder_prefix.map_or(Value::Null, Value::Text),
|
folder_prefix.map_or(Value::Null, Value::Text),
|
||||||
extension.map_or(Value::Null, Value::Text),
|
extension.map_or(Value::Null, Value::Text),
|
||||||
@@ -535,7 +638,9 @@ impl SearchIndex {
|
|||||||
.modified_before_unix_ms
|
.modified_before_unix_ms
|
||||||
.map_or(Value::Null, Value::Integer),
|
.map_or(Value::Null, Value::Integer),
|
||||||
];
|
];
|
||||||
let has_filters = filter_values.iter().any(|value| *value != Value::Null);
|
let has_filters = filter_values.iter().any(|value| *value != Value::Null)
|
||||||
|
|| options.min_duration_ms.is_some()
|
||||||
|
|| options.max_duration_ms.is_some();
|
||||||
let mut results = Vec::new();
|
let mut results = Vec::new();
|
||||||
let mut selected = Vec::<i64>::new();
|
let mut selected = Vec::<i64>::new();
|
||||||
// Each lane filters its entire candidate set before LIMIT. There is no
|
// Each lane filters its entire candidate set before LIMIT. There is no
|
||||||
@@ -561,13 +666,17 @@ impl SearchIndex {
|
|||||||
if cache.len() + grams.len() > 4096 {
|
if cache.len() + grams.len() > 4096 {
|
||||||
cache.clear();
|
cache.clear();
|
||||||
}
|
}
|
||||||
|
let mut frequency_hits = 0u64;
|
||||||
|
let mut frequency_misses = 0u64;
|
||||||
let mut rare_first = grams
|
let mut rare_first = grams
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|gram| {
|
.map(|gram| {
|
||||||
let term = gram.to_lowercase();
|
let term = gram.to_lowercase();
|
||||||
let frequency = if let Some(frequency) = cache.get(&term) {
|
let frequency = if let Some(frequency) = cache.get(&term) {
|
||||||
|
frequency_hits += 1;
|
||||||
*frequency
|
*frequency
|
||||||
} else {
|
} else {
|
||||||
|
frequency_misses += 1;
|
||||||
let frequency = vocabulary
|
let frequency = vocabulary
|
||||||
.query_row([&term], |row| row.get::<_, i64>(0))
|
.query_row([&term], |row| row.get::<_, i64>(0))
|
||||||
.optional()?
|
.optional()?
|
||||||
@@ -581,6 +690,9 @@ impl SearchIndex {
|
|||||||
rare_first.sort();
|
rare_first.sort();
|
||||||
drop(cache);
|
drop(cache);
|
||||||
let anchor = substring_anchor(&characters, &rare_first);
|
let anchor = substring_anchor(&characters, &rare_first);
|
||||||
|
trace.counter(planning, "frequency_cache_hits", frequency_hits);
|
||||||
|
trace.counter(planning, "frequency_cache_misses", frequency_misses);
|
||||||
|
trace.counter(planning, "trigram_index", u64::from(trigram));
|
||||||
let grams = format!("\"{}\"", anchor.replace('"', "\"\""));
|
let grams = format!("\"{}\"", anchor.replace('"', "\"\""));
|
||||||
let mut lanes = vec![
|
let mut lanes = vec![
|
||||||
(
|
(
|
||||||
@@ -648,10 +760,30 @@ impl SearchIndex {
|
|||||||
"rank,f.id",
|
"rank,f.id",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
lanes.push((
|
||||||
|
"media_metadata",
|
||||||
|
"media_tags CROSS JOIN files f ON f.id=media_tags.rowid".into(),
|
||||||
|
"media_tags MATCH ?1".into(),
|
||||||
|
fts.clone(),
|
||||||
|
"rank,f.id",
|
||||||
|
));
|
||||||
|
trace.end(planning, Default::default());
|
||||||
for (kind, from, condition, expression, ordering) in lanes {
|
for (kind, from, condition, expression, ordering) in lanes {
|
||||||
if results.len() == limit {
|
if results.len() == limit {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
let lane = trace.begin(
|
||||||
|
match kind {
|
||||||
|
"exact_filename" => "lexical.exact_filename",
|
||||||
|
"filename" => "lexical.filename_substring",
|
||||||
|
"path" => "lexical.path_substring",
|
||||||
|
"name_terms" => "lexical.filename_terms",
|
||||||
|
"content" => "lexical.content",
|
||||||
|
"media_metadata" => "lexical.media_metadata",
|
||||||
|
_ => "lexical.other",
|
||||||
|
},
|
||||||
|
parent,
|
||||||
|
);
|
||||||
let exclusions = if selected.is_empty() {
|
let exclusions = if selected.is_empty() {
|
||||||
String::new()
|
String::new()
|
||||||
} else {
|
} else {
|
||||||
@@ -664,12 +796,14 @@ impl SearchIndex {
|
|||||||
.join(",")
|
.join(",")
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
let rank = if matches!(kind, "content" | "name_terms") {
|
let rank = if matches!(kind, "content" | "name_terms" | "media_metadata") {
|
||||||
"rank"
|
"rank"
|
||||||
} else {
|
} else {
|
||||||
"0.0"
|
"0.0"
|
||||||
};
|
};
|
||||||
let snippet = if kind == "content" {
|
let snippet = if kind == "media_metadata" {
|
||||||
|
"snippet(media_tags,0,'','',' … ',24)"
|
||||||
|
} else if kind == "content" {
|
||||||
"snippet(content_search,0,'','',' … ',24)"
|
"snippet(content_search,0,'','',' … ',24)"
|
||||||
} else if kind == "name_terms" {
|
} else if kind == "name_terms" {
|
||||||
"snippet(search,2,'','',' … ',24)"
|
"snippet(search,2,'','',' … ',24)"
|
||||||
@@ -698,6 +832,20 @@ impl SearchIndex {
|
|||||||
let mut values = vec![Value::Text(expression), Value::Text(query.into())];
|
let mut values = vec![Value::Text(expression), Value::Text(query.into())];
|
||||||
values.extend(filter_values.iter().cloned());
|
values.extend(filter_values.iter().cloned());
|
||||||
values.push(Value::Integer((limit - results.len()) as i64));
|
values.push(Value::Integer((limit - results.len()) as i64));
|
||||||
|
if sql.contains("?10") {
|
||||||
|
values.push(
|
||||||
|
options
|
||||||
|
.min_duration_ms
|
||||||
|
.map_or(Value::Null, |v| Value::Integer(i64::from(v))),
|
||||||
|
);
|
||||||
|
values.push(
|
||||||
|
options
|
||||||
|
.max_duration_ms
|
||||||
|
.map_or(Value::Null, |v| Value::Integer(i64::from(v))),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let sql_span = trace.begin("lexical.filtered_sql", lane);
|
||||||
|
trace.counter(sql_span, "filters_active", u64::from(has_filters));
|
||||||
let mut statement = self.connection.prepare(&sql)?;
|
let mut statement = self.connection.prepare(&sql)?;
|
||||||
let matches = statement
|
let matches = statement
|
||||||
.query_map(params_from_iter(values), |row| {
|
.query_map(params_from_iter(values), |row| {
|
||||||
@@ -715,12 +863,16 @@ impl SearchIndex {
|
|||||||
))
|
))
|
||||||
})?
|
})?
|
||||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
trace.counter(sql_span, "rows", matches.len() as u64);
|
||||||
|
trace.end(sql_span, Default::default());
|
||||||
for (id, result) in matches {
|
for (id, result) in matches {
|
||||||
selected.push(id);
|
selected.push(id);
|
||||||
results.push(result);
|
results.push(result);
|
||||||
}
|
}
|
||||||
|
trace.end(lane, Default::default());
|
||||||
}
|
}
|
||||||
if !results.is_empty() {
|
if !results.is_empty() {
|
||||||
|
let source_span = trace.begin("lexical.source_locations", parent);
|
||||||
// Ownership has a B-tree index. The rowid IN constraint intersects
|
// Ownership has a B-tree index. The rowid IN constraint intersects
|
||||||
// the FTS postings with only the actual finalists' passage IDs.
|
// the FTS postings with only the actual finalists' passage IDs.
|
||||||
let placeholders = (0..results.len())
|
let placeholders = (0..results.len())
|
||||||
@@ -762,6 +914,9 @@ impl SearchIndex {
|
|||||||
result.snippet = snippet.clone();
|
result.snippet = snippet.clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
trace.counter(source_span, "files", results.len() as u64);
|
||||||
|
trace.counter(source_span, "matched_sources", sources.len() as u64);
|
||||||
|
trace.end(source_span, Default::default());
|
||||||
}
|
}
|
||||||
Ok(results)
|
Ok(results)
|
||||||
}
|
}
|
||||||
@@ -856,6 +1011,100 @@ fn substring_anchor(characters: &[char], frequencies: &[(i64, String)]) -> Strin
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
#[test]
|
||||||
|
fn measured_search_preserves_results_and_contains_private_free_nested_spans(
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let directory = tempfile::tempdir()?;
|
||||||
|
let root = directory.path().join("sensitive-private-location");
|
||||||
|
std::fs::create_dir(&root)?;
|
||||||
|
std::fs::write(
|
||||||
|
root.join("agreement.txt"),
|
||||||
|
"confidentialneedle cancellation policy",
|
||||||
|
)?;
|
||||||
|
let db = directory.path().join("index.db");
|
||||||
|
let mut index = super::SearchIndex::open(&db)?;
|
||||||
|
index.index_folder(&root)?;
|
||||||
|
let basic = index.search_measured("confidentialneedle", 10, false)?;
|
||||||
|
let debug = index.search_measured("confidentialneedle", 10, true)?;
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&basic.results)?,
|
||||||
|
serde_json::to_value(&debug.results)?
|
||||||
|
);
|
||||||
|
assert_eq!(basic.metrics.result_count, 1);
|
||||||
|
assert!(basic.metrics.trace.is_none());
|
||||||
|
let spans = debug.metrics.trace.as_ref().unwrap();
|
||||||
|
assert!(spans.iter().any(|s| s.name == "lexical.content"));
|
||||||
|
assert!(spans.iter().any(|s| s.name == "lexical.source_locations"));
|
||||||
|
for span in spans {
|
||||||
|
assert!(span.start_ms >= 0.0 && span.duration_ms >= 0.0);
|
||||||
|
assert!(span.start_ms + span.duration_ms <= debug.metrics.total_ms + 1e-6);
|
||||||
|
if let Some(parent) = span.parent {
|
||||||
|
let parent = &spans[parent];
|
||||||
|
assert!(span.start_ms >= parent.start_ms);
|
||||||
|
assert!(
|
||||||
|
span.start_ms + span.duration_ms <= parent.start_ms + parent.duration_ms + 1e-6
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let trace = serde_json::to_string(spans)?;
|
||||||
|
assert!(!trace.contains("confidentialneedle"));
|
||||||
|
assert!(!trace.contains("sensitive-private-location"));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn genuine_media_tags_search_duration_filter_and_lifecycle() -> anyhow::Result<()> {
|
||||||
|
let directory = tempfile::tempdir()?;
|
||||||
|
let root = directory.path().join("music");
|
||||||
|
std::fs::create_dir(&root)?;
|
||||||
|
let file = root.join("track.ogg");
|
||||||
|
std::fs::write(&file, b"storage fixture")?;
|
||||||
|
let db = directory.path().join("index.db");
|
||||||
|
let mut index = super::SearchIndex::open(&db)?;
|
||||||
|
index.index_folder(&root)?;
|
||||||
|
let path = file.canonicalize()?.to_string_lossy().into_owned();
|
||||||
|
let (size, mtime): (u64, i64) = index.connection.query_row(
|
||||||
|
"SELECT size,mtime FROM files WHERE path=?1",
|
||||||
|
[&path],
|
||||||
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||||
|
)?;
|
||||||
|
index.set_media_metadata(
|
||||||
|
&path,
|
||||||
|
"artist: Kimiko Ishizaka\ntitle: Variatio 13 a 2 Clav.",
|
||||||
|
180000,
|
||||||
|
size,
|
||||||
|
mtime,
|
||||||
|
)?;
|
||||||
|
let results = index.search("Kimiko Ishizaka", 10)?;
|
||||||
|
assert_eq!(results.len(), 1);
|
||||||
|
assert_eq!(results[0].match_kind, "media_metadata");
|
||||||
|
assert!(results[0].snippet.contains("Kimiko"));
|
||||||
|
let too_short = super::SearchOptions {
|
||||||
|
max_duration_ms: Some(100000),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(index
|
||||||
|
.search_with_options("track.ogg", 10, &too_short)?
|
||||||
|
.is_empty());
|
||||||
|
let correct = super::SearchOptions {
|
||||||
|
min_duration_ms: Some(170000),
|
||||||
|
max_duration_ms: Some(190000),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(index.search_with_options("Kimiko", 10, &correct)?.len(), 1);
|
||||||
|
drop(index);
|
||||||
|
let mut index = super::SearchIndex::open(&db)?;
|
||||||
|
assert_eq!(index.search("Variatio", 10)?.len(), 1);
|
||||||
|
std::fs::write(&file, b"changed file bytes")?;
|
||||||
|
index.index_folder(&root)?;
|
||||||
|
assert!(index.search("Kimiko", 10)?.is_empty());
|
||||||
|
assert!(index
|
||||||
|
.set_media_metadata(&path, "stale", 180000, size, mtime)
|
||||||
|
.is_err());
|
||||||
|
std::fs::remove_file(&file)?;
|
||||||
|
index.index_folder(&root)?;
|
||||||
|
assert!(index.search("Variatio", 10)?.is_empty());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
use super::*;
|
use super::*;
|
||||||
#[test]
|
#[test]
|
||||||
fn persistence_updates_deletes_and_exact_priority() -> Result<()> {
|
fn persistence_updates_deletes_and_exact_priority() -> Result<()> {
|
||||||
|
|||||||
@@ -0,0 +1,292 @@
|
|||||||
|
//! Monotonic request metrics. Stage names are static labels, never queries/paths.
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::{
|
||||||
|
collections::BTreeMap,
|
||||||
|
sync::atomic::{AtomicU64, Ordering},
|
||||||
|
time::Instant,
|
||||||
|
};
|
||||||
|
const MAX_SPANS: usize = 512;
|
||||||
|
static COLLECTOR_ID: AtomicU64 = AtomicU64::new(1);
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct SearchResponse<T> {
|
||||||
|
pub results: Vec<T>,
|
||||||
|
pub metrics: SearchMetrics,
|
||||||
|
}
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct SearchMetrics {
|
||||||
|
pub total_ms: f64,
|
||||||
|
pub result_count: usize,
|
||||||
|
pub trace: Option<Vec<SearchSpan>>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub dropped_spans: u64,
|
||||||
|
}
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct SearchSpan {
|
||||||
|
pub name: String,
|
||||||
|
pub start_ms: f64,
|
||||||
|
pub duration_ms: f64,
|
||||||
|
pub parent: Option<usize>,
|
||||||
|
pub counters: BTreeMap<String, u64>,
|
||||||
|
}
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum MetricsMode {
|
||||||
|
Basic,
|
||||||
|
Debug,
|
||||||
|
}
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub struct SpanToken {
|
||||||
|
collector: u64,
|
||||||
|
index: usize,
|
||||||
|
}
|
||||||
|
impl SpanToken {
|
||||||
|
pub fn index(self) -> usize {
|
||||||
|
self.index
|
||||||
|
}
|
||||||
|
}
|
||||||
|
struct OpenSpan {
|
||||||
|
value: SearchSpan,
|
||||||
|
started: Instant,
|
||||||
|
ended: bool,
|
||||||
|
}
|
||||||
|
struct DebugTrace {
|
||||||
|
id: u64,
|
||||||
|
spans: Vec<OpenSpan>,
|
||||||
|
dropped: u64,
|
||||||
|
}
|
||||||
|
/// Basic mode stores only one Instant: no span Vec or stage/counter allocation.
|
||||||
|
pub struct TraceCollector {
|
||||||
|
started: Instant,
|
||||||
|
debug: Option<DebugTrace>,
|
||||||
|
}
|
||||||
|
impl TraceCollector {
|
||||||
|
pub fn new(mode: MetricsMode) -> Self {
|
||||||
|
Self {
|
||||||
|
started: Instant::now(),
|
||||||
|
debug: match mode {
|
||||||
|
MetricsMode::Basic => None,
|
||||||
|
MetricsMode::Debug => Some(DebugTrace {
|
||||||
|
id: COLLECTOR_ID.fetch_add(1, Ordering::Relaxed),
|
||||||
|
spans: Vec::new(),
|
||||||
|
dropped: 0,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn elapsed_ms(&self) -> f64 {
|
||||||
|
self.started.elapsed().as_secs_f64() * 1000.0
|
||||||
|
}
|
||||||
|
pub fn is_debug(&self) -> bool {
|
||||||
|
self.debug.is_some()
|
||||||
|
}
|
||||||
|
pub fn begin(&mut self, name: &'static str, parent: Option<SpanToken>) -> Option<SpanToken> {
|
||||||
|
let debug = self.debug.as_mut()?;
|
||||||
|
let parent_index = match parent {
|
||||||
|
Some(token)
|
||||||
|
if token.collector == debug.id
|
||||||
|
&& debug.spans.get(token.index).is_some_and(|span| !span.ended) =>
|
||||||
|
{
|
||||||
|
Some(token.index)
|
||||||
|
}
|
||||||
|
Some(_) => {
|
||||||
|
debug.dropped = debug.dropped.saturating_add(1);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
if debug.spans.len() >= MAX_SPANS {
|
||||||
|
debug.dropped = debug.dropped.saturating_add(1);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let started = Instant::now();
|
||||||
|
let index = debug.spans.len();
|
||||||
|
debug.spans.push(OpenSpan {
|
||||||
|
value: SearchSpan {
|
||||||
|
name: name.chars().take(96).collect(),
|
||||||
|
start_ms: started.duration_since(self.started).as_secs_f64() * 1000.0,
|
||||||
|
duration_ms: 0.0,
|
||||||
|
parent: parent_index,
|
||||||
|
counters: BTreeMap::new(),
|
||||||
|
},
|
||||||
|
started,
|
||||||
|
ended: false,
|
||||||
|
});
|
||||||
|
Some(SpanToken {
|
||||||
|
collector: debug.id,
|
||||||
|
index,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
/// Invalid/previously ended tokens are harmless. Closing a parent also
|
||||||
|
/// closes open descendants at the same instant, preserving containment.
|
||||||
|
pub fn end(&mut self, token: Option<SpanToken>, counters: BTreeMap<String, u64>) {
|
||||||
|
let Some(token) = token else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Some(debug) = self.debug.as_mut() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if token.collector != debug.id
|
||||||
|
|| token.index >= debug.spans.len()
|
||||||
|
|| debug.spans[token.index].ended
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let now = Instant::now();
|
||||||
|
for index in token.index..debug.spans.len() {
|
||||||
|
let mut ancestor = Some(index);
|
||||||
|
let mut contained = false;
|
||||||
|
while let Some(current) = ancestor {
|
||||||
|
if current == token.index {
|
||||||
|
contained = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
ancestor = debug.spans[current].value.parent;
|
||||||
|
}
|
||||||
|
if contained && !debug.spans[index].ended {
|
||||||
|
let span = &mut debug.spans[index];
|
||||||
|
span.value.duration_ms = now.duration_since(span.started).as_secs_f64() * 1000.0;
|
||||||
|
span.ended = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// At most32 distinct bounded keys; callers should use static diagnostic
|
||||||
|
// names and counts, never source text or paths.
|
||||||
|
let stored = &mut debug.spans[token.index].value.counters;
|
||||||
|
for (key, value) in counters.into_iter().take(32) {
|
||||||
|
let key: String = key.chars().take(96).collect();
|
||||||
|
if stored.len() < 32 || stored.contains_key(&key) {
|
||||||
|
stored.insert(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn counter(&mut self, token: Option<SpanToken>, name: &'static str, value: u64) {
|
||||||
|
let Some(token) = token else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Some(debug) = self.debug.as_mut() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if token.collector != debug.id {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Some(span) = debug.spans.get_mut(token.index) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if span.value.counters.len() < 32 || span.value.counters.contains_key(name) {
|
||||||
|
span.value
|
||||||
|
.counters
|
||||||
|
.insert(name.chars().take(96).collect(), value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/// Return the closure's value unchanged, including any Result error.
|
||||||
|
/// The closure receives its parent token for safe nested instrumentation.
|
||||||
|
pub fn timed<R>(
|
||||||
|
&mut self,
|
||||||
|
name: &'static str,
|
||||||
|
parent: Option<SpanToken>,
|
||||||
|
run: impl FnOnce(&mut Self, Option<SpanToken>) -> R,
|
||||||
|
) -> R {
|
||||||
|
let token = self.begin(name, parent);
|
||||||
|
let value = run(self, token);
|
||||||
|
self.end(token, BTreeMap::new());
|
||||||
|
value
|
||||||
|
}
|
||||||
|
pub fn finish(mut self, result_count: usize) -> SearchMetrics {
|
||||||
|
let now = Instant::now();
|
||||||
|
if let Some(debug) = self.debug.as_mut() {
|
||||||
|
for span in &mut debug.spans {
|
||||||
|
if !span.ended {
|
||||||
|
span.value.duration_ms =
|
||||||
|
now.duration_since(span.started).as_secs_f64() * 1000.0;
|
||||||
|
span.ended = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let (trace, dropped_spans) = match self.debug {
|
||||||
|
Some(debug) => (
|
||||||
|
Some(debug.spans.into_iter().map(|span| span.value).collect()),
|
||||||
|
debug.dropped,
|
||||||
|
),
|
||||||
|
None => (None, 0),
|
||||||
|
};
|
||||||
|
SearchMetrics {
|
||||||
|
total_ms: now.duration_since(self.started).as_secs_f64() * 1000.0,
|
||||||
|
result_count,
|
||||||
|
trace,
|
||||||
|
dropped_spans,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
#[test]
|
||||||
|
fn nesting_contains_children_and_preserves_error() {
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
let result: Result<(), &str> = trace.timed("request", None, |trace, parent| {
|
||||||
|
trace.timed("index", parent, |trace, child| {
|
||||||
|
trace.counter(child, "rows", 7);
|
||||||
|
Err("failure")
|
||||||
|
})
|
||||||
|
});
|
||||||
|
assert_eq!(result, Err("failure"));
|
||||||
|
let metrics = trace.finish(0);
|
||||||
|
let spans = metrics.trace.unwrap();
|
||||||
|
assert_eq!(spans[1].parent, Some(0));
|
||||||
|
assert_eq!(spans[1].counters["rows"], 7);
|
||||||
|
assert!(spans[1].start_ms >= spans[0].start_ms);
|
||||||
|
assert!(
|
||||||
|
spans[1].start_ms + spans[1].duration_ms
|
||||||
|
<= spans[0].start_ms + spans[0].duration_ms + 1e-6
|
||||||
|
);
|
||||||
|
assert!(metrics.total_ms >= spans[0].start_ms + spans[0].duration_ms - 1e-6);
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn basic_disables_stage_storage_and_preserves_counts() {
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Basic);
|
||||||
|
assert!(!trace.is_debug());
|
||||||
|
assert!(trace.begin("unused", None).is_none());
|
||||||
|
assert_eq!(trace.timed("unused", None, |_, _| 42), 42);
|
||||||
|
let metrics = trace.finish(3);
|
||||||
|
assert_eq!(metrics.result_count, 3);
|
||||||
|
assert!(metrics.trace.is_none());
|
||||||
|
assert_eq!(metrics.dropped_spans, 0);
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn trace_is_capped_and_reports_dropped_spans() {
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
for index in 0..520 {
|
||||||
|
let token = trace.begin("stage", None);
|
||||||
|
if index == 0 {
|
||||||
|
trace.counter(token, "existing", 99);
|
||||||
|
}
|
||||||
|
let counters = if index == 0 {
|
||||||
|
(0..40).map(|key| (format!("counter_{key}"), key)).collect()
|
||||||
|
} else {
|
||||||
|
BTreeMap::new()
|
||||||
|
};
|
||||||
|
trace.end(token, counters);
|
||||||
|
}
|
||||||
|
let metrics = trace.finish(2);
|
||||||
|
let spans = metrics.trace.unwrap();
|
||||||
|
assert_eq!(spans.len(), 512);
|
||||||
|
assert_eq!(spans[0].counters.len(), 32);
|
||||||
|
assert_eq!(spans[0].counters["existing"], 99);
|
||||||
|
assert_eq!(metrics.dropped_spans, 8);
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn foreign_tokens_and_early_parent_close_are_safe() {
|
||||||
|
let mut one = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
let mut two = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
let root = one.begin("root", None);
|
||||||
|
let child = one.begin("child", root);
|
||||||
|
assert!(two.begin("foreign", root).is_none());
|
||||||
|
one.end(root, BTreeMap::new());
|
||||||
|
one.end(child, BTreeMap::new());
|
||||||
|
let spans = one.finish(0).trace.unwrap();
|
||||||
|
assert!(
|
||||||
|
spans[1].start_ms + spans[1].duration_ms
|
||||||
|
<= spans[0].start_ms + spans[0].duration_ms + 1e-6
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
[package]
|
||||||
|
name = "searchi-media"
|
||||||
|
version.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
description = "Bounded local CPU media probing and timestamp-preserving sampling"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
anyhow.workspace = true
|
||||||
|
serde.workspace = true
|
||||||
|
serde_json.workspace = true
|
||||||
|
sha2 = "0.10"
|
||||||
|
tempfile = "3"
|
||||||
|
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "bmp"] }
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# Local media preparation
|
||||||
|
|
||||||
|
`searchi-media` prepares actual files for the Google multimodal embedder. It
|
||||||
|
does not download programs or models and has no UI dependency. Callers supply
|
||||||
|
verified packaged FFmpeg/ffprobe paths through `FfmpegTools::new`, or explicitly
|
||||||
|
set both `SEARCHI_FFMPEG_PATH` and `SEARCHI_FFPROBE_PATH`.
|
||||||
|
|
||||||
|
FFmpeg handles maintained codecs and container timestamps; serde parses its
|
||||||
|
bounded JSON, tempfile owns frame cleanup, and sha2 validates file identity.
|
||||||
|
The embedding crate uses the Rust image library for bounded raster decoding.
|
||||||
|
USearch supplies the service's persisted HNSW index; no codec or vector index
|
||||||
|
is reimplemented here.
|
||||||
|
|
||||||
|
Every FFmpeg/ffprobe media input restricts protocols to `file,pipe`. A selected
|
||||||
|
local playlist cannot fetch HTTP/HTTPS/RTSP references. A real loopback-listener
|
||||||
|
regression verifies probing and both decoding paths never connect. This is a
|
||||||
|
protocol restriction, not a filesystem sandbox: explicitly selected files on
|
||||||
|
network-mounted drives still use their operating system's filesystem access.
|
||||||
|
|
||||||
|
Windows VM tests use CPU only. Decoder commands specify one thread and
|
||||||
|
`-hwaccel none`; subprocess windows are hidden. Portable argument arrays avoid
|
||||||
|
shell interpretation. Production hardware decode and reliable CPU fallback
|
||||||
|
remain unimplemented; this crate does not silently enable accelerators.
|
||||||
|
|
||||||
|
Supported images are PNG, JPEG, WebP and BMP (actual bounded embedding-library
|
||||||
|
decode, including EXIF orientation). AVIF, HEIC, TIFF, animated GIF and embedded
|
||||||
|
album artwork are not supported yet. Movie containers include MP4, MKV, MOV,
|
||||||
|
AVI, WebM, M4V, MPEG/MPG; audio includes MP3, FLAC, WAV, M4A, Ogg, Opus, AAC,
|
||||||
|
AIFF. Actual codec support comes from the packaged FFmpeg build. Unknown or
|
||||||
|
corrupt files remain filename-searchable with explicit indexing errors.
|
||||||
|
|
||||||
|
Default limits: 16 GiB media input; four-hour known duration; 16 million source
|
||||||
|
video pixels; 45 seconds per decoder command; 64 KiB ffprobe JSON; 256 KiB
|
||||||
|
frame diagnostic output; 8 MiB encoded PNG frame. Images are separately limited
|
||||||
|
to 64 MiB input. Captured pipe output is bounded and timed processes are killed.
|
||||||
|
Process address-space sandboxing remains a release gate; these limits do not
|
||||||
|
guarantee a codec's internal allocations fit a fixed RAM ceiling.
|
||||||
|
|
||||||
|
Video samples span the whole timeline at two-second intervals, becoming coarser
|
||||||
|
when the default 240 distributed-frame budget is reached. Up to 120 additional
|
||||||
|
scene-cut frames are captured within 1.5-second windows around sample anchors.
|
||||||
|
This is **partial visual coverage**, not exhaustive scene detection. Actual
|
||||||
|
decoded presentation timestamps are retained relative to container start. The
|
||||||
|
primary non-cover-art video stream is selected. Full images use model budget
|
||||||
|
280; video frames use 70. No fabricated captions are stored.
|
||||||
|
|
||||||
|
The primary audio stream is decoded to mono 16 kHz PCM, embedded in 30-second
|
||||||
|
windows with two seconds of overlap. The complete stream is covered or the
|
||||||
|
600-window budget returns an explicit failure. Standalone music uses the same
|
||||||
|
real audio embedding; artist/title/album/genre tags are merged from genuine
|
||||||
|
stream and format metadata and stored separately for lexical search. Audio
|
||||||
|
embeddings are not transcripts; exact spoken-word retrieval is not provided.
|
||||||
|
Alternative language tracks and isolated channels remain a release gate.
|
||||||
|
|
||||||
|
The service publishes text, image, video and audio vectors as one generation.
|
||||||
|
It persists origin and page/timestamp locations, policy/model fingerprints,
|
||||||
|
full file SHA at indexing, and size/mtime guards. Corrupt-file failures discard
|
||||||
|
that file's partial vectors and persist reasons. Model inference failure aborts
|
||||||
|
the generation. At query time only actual media candidate hits are stat-checked;
|
||||||
|
no corpus-wide hash/stat scan occurs. Changes preserving both size and mtime
|
||||||
|
cannot be detected by this fast guard; periodic content-hash reconciliation is
|
||||||
|
a release gate. Builds currently rebuild a complete generation, with a
|
||||||
|
100,000-vector / 10,000-media-file bound; unchanged-vector reuse is future work.
|
||||||
|
|
||||||
|
Offline tests cover complete audio windows, frame budgets and identity changes.
|
||||||
|
The ignored `real_decoder_preserves_frame_timestamps_and_audio` test has been
|
||||||
|
executed successfully in the CPU Windows VM with provisioned FFmpeg 9.0.2.
|
||||||
|
It creates genuine FFV1/PCM media, validates decoded two-second PTS and actual
|
||||||
|
mono sine samples. Linux/macOS execution and installer codec/license delivery
|
||||||
|
have not been validated.
|
||||||
@@ -0,0 +1,789 @@
|
|||||||
|
//! Local bounded media preparation. No network, model, executable installer, or
|
||||||
|
//! fabricated caption/transcript lives here. FFmpeg paths come from the caller.
|
||||||
|
//!
|
||||||
|
//! Video coverage is distributed sampling, not a guarantee that every scene is
|
||||||
|
//! represented. Audio windows cover the complete primary stream or return an
|
||||||
|
//! explicit budget error. Decoder process memory is not sandbox-limited yet.
|
||||||
|
use anyhow::{bail, ensure, Context, Result};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::{
|
||||||
|
collections::BTreeMap,
|
||||||
|
fs::{self, File},
|
||||||
|
io::Read,
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
process::{Command, Stdio},
|
||||||
|
thread,
|
||||||
|
time::{Duration, Instant, UNIX_EPOCH},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||||
|
pub enum MediaKind {
|
||||||
|
Image,
|
||||||
|
Video,
|
||||||
|
Audio,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn fixture(duration_ms: u32) -> MediaProbe {
|
||||||
|
MediaProbe {
|
||||||
|
duration_ms,
|
||||||
|
start_ms: 0,
|
||||||
|
video_stream_index: 0,
|
||||||
|
width: 64,
|
||||||
|
height: 64,
|
||||||
|
has_video: true,
|
||||||
|
has_audio: true,
|
||||||
|
audio_start_ms: 0,
|
||||||
|
audio_end_ms: duration_ms,
|
||||||
|
tags: BTreeMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn audio_covers_tail_with_overlap_and_rejects_budget() -> Result<()> {
|
||||||
|
let probe = fixture(65_000);
|
||||||
|
let limits = MediaLimits::default();
|
||||||
|
let windows = audio_windows(&probe, &limits)?;
|
||||||
|
assert_eq!(
|
||||||
|
windows.iter().map(|w| w.start_ms).collect::<Vec<_>>(),
|
||||||
|
vec![0, 28_000, 56_000]
|
||||||
|
);
|
||||||
|
assert_eq!(windows.last().unwrap().duration_ms, 9_000);
|
||||||
|
let tight = MediaLimits {
|
||||||
|
max_audio_windows: 2,
|
||||||
|
..limits
|
||||||
|
};
|
||||||
|
assert!(audio_windows(&probe, &tight).is_err());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn finer_audio_windows_cover_long_movie_or_explicitly_reject() -> Result<()> {
|
||||||
|
let probe = fixture(3 * 60 * 60 * 1000);
|
||||||
|
let limits = MediaLimits {
|
||||||
|
audio_window_ms: 10_000,
|
||||||
|
audio_overlap_ms: 2000,
|
||||||
|
max_audio_windows: 1800,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let windows = audio_windows(&probe, &limits)?;
|
||||||
|
assert_eq!(windows.first().unwrap().start_ms, 0);
|
||||||
|
let last = windows.last().unwrap();
|
||||||
|
assert_eq!(last.start_ms + last.duration_ms, probe.duration_ms);
|
||||||
|
assert!(windows.len() >= 1300 && windows.len() <= 1440);
|
||||||
|
let too_small = MediaLimits {
|
||||||
|
max_audio_windows: 1000,
|
||||||
|
..limits
|
||||||
|
};
|
||||||
|
assert!(audio_windows(&probe, &too_small).is_err());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn frame_budget_spans_entire_movie() -> Result<()> {
|
||||||
|
let probe = fixture(3_600_000);
|
||||||
|
let times = frame_times(&probe, &MediaLimits::default())?;
|
||||||
|
assert!(times.len() <= 240);
|
||||||
|
assert_eq!(times.first(), Some(&0));
|
||||||
|
assert_eq!(times.last(), Some(&3_599_500));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn identity_rejects_changes_and_oversized_input() -> Result<()> {
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
let file = temp.path().join("audio.wav");
|
||||||
|
fs::write(&file, b"first")?;
|
||||||
|
assert!(MediaIdentity::read(&file, 4).is_err());
|
||||||
|
let identity = MediaIdentity::read(&file, 10)?;
|
||||||
|
fs::write(&file, b"changed size")?;
|
||||||
|
assert!(identity.ensure_unchanged().is_err());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "Requires explicitly provisioned CPU FFmpeg/ffprobe executables"]
|
||||||
|
fn decoder_rejects_network_references_inside_local_media() -> Result<()> {
|
||||||
|
let tools = FfmpegTools::from_environment()?.context("Set CPU decoder paths")?;
|
||||||
|
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
|
||||||
|
listener.set_nonblocking(true)?;
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
let file = temp.path().join("local.m3u8");
|
||||||
|
fs::write(&file,format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:2\n#EXTINF:2.0,\nhttp://{}/segment.ts\n#EXT-X-ENDLIST\n",listener.local_addr()?))?;
|
||||||
|
let limits = MediaLimits {
|
||||||
|
command_timeout: Duration::from_secs(3),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let error = super::probe(&tools, &file, &limits).unwrap_err();
|
||||||
|
assert!(format!("{error:#}").contains("whitelist"), "{error:#}");
|
||||||
|
assert!(decode_frames(&tools, &file, &fixture(2000), 0, false, &limits).is_err());
|
||||||
|
assert!(decode_audio(
|
||||||
|
&tools,
|
||||||
|
&file,
|
||||||
|
AudioWindow {
|
||||||
|
start_ms: 0,
|
||||||
|
duration_ms: 2000
|
||||||
|
},
|
||||||
|
&limits
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
match listener.accept() {
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {}
|
||||||
|
other => panic!("Decoder attempted network connection: {other:?}"),
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "Requires explicitly provisioned CPU FFmpeg/ffprobe executables"]
|
||||||
|
fn real_decoder_preserves_frame_timestamps_and_audio() -> Result<()> {
|
||||||
|
let tools = FfmpegTools::from_environment()?.context("Set CPU decoder paths")?;
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
let video = temp.path().join("fixture.mkv");
|
||||||
|
let mut command = Command::new(&tools.ffmpeg);
|
||||||
|
command
|
||||||
|
.args([
|
||||||
|
"-nostdin",
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-threads",
|
||||||
|
"1",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
"color=c=red:s=64x64:r=10:d=4",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
"sine=frequency=440:sample_rate=16000:duration=4",
|
||||||
|
"-c:v",
|
||||||
|
"ffv1",
|
||||||
|
"-c:a",
|
||||||
|
"pcm_s16le",
|
||||||
|
"-threads",
|
||||||
|
"1",
|
||||||
|
"-y",
|
||||||
|
])
|
||||||
|
.arg(&video);
|
||||||
|
run(command, Duration::from_secs(30), 4096, 65536)?;
|
||||||
|
let limits = MediaLimits::default();
|
||||||
|
let probe = probe(&tools, &video, &limits)?;
|
||||||
|
assert!(probe.has_video && probe.has_audio);
|
||||||
|
let frames = decode_frames(&tools, &video, &probe, 2000, false, &limits)?;
|
||||||
|
assert_eq!(frames.frames.len(), 1);
|
||||||
|
assert!((1900..=2100).contains(&frames.frames[0].timestamp_ms));
|
||||||
|
let audio = decode_audio(
|
||||||
|
&tools,
|
||||||
|
&video,
|
||||||
|
AudioWindow {
|
||||||
|
start_ms: 1000,
|
||||||
|
duration_ms: 2000,
|
||||||
|
},
|
||||||
|
&limits,
|
||||||
|
)?;
|
||||||
|
assert!((31_000..=32_000).contains(&audio.len()));
|
||||||
|
assert!(audio.iter().any(|x| x.abs() > 0.01));
|
||||||
|
assert!(audio.iter().all(|x| x.is_finite() && x.abs() <= 1.0));
|
||||||
|
let ogg = temp.path().join("overshoot.ogg");
|
||||||
|
let mut command = Command::new(&tools.ffmpeg);
|
||||||
|
command
|
||||||
|
.args([
|
||||||
|
"-nostdin",
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-threads",
|
||||||
|
"1",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
"sine=frequency=440:sample_rate=44100:duration=31.123",
|
||||||
|
"-c:a",
|
||||||
|
"libvorbis",
|
||||||
|
"-metadata",
|
||||||
|
"artist=CPU decoder fixture",
|
||||||
|
"-threads",
|
||||||
|
"1",
|
||||||
|
"-y",
|
||||||
|
])
|
||||||
|
.arg(&ogg);
|
||||||
|
run(command, Duration::from_secs(30), 4096, 65536)?;
|
||||||
|
let music_probe = super::probe(&tools, &ogg, &limits)?;
|
||||||
|
assert_eq!(
|
||||||
|
music_probe.tags.get("artist").map(String::as_str),
|
||||||
|
Some("CPU decoder fixture")
|
||||||
|
);
|
||||||
|
let pcm = decode_audio(
|
||||||
|
&tools,
|
||||||
|
&ogg,
|
||||||
|
AudioWindow {
|
||||||
|
start_ms: 0,
|
||||||
|
duration_ms: 30000,
|
||||||
|
},
|
||||||
|
&limits,
|
||||||
|
)?;
|
||||||
|
assert_eq!(
|
||||||
|
pcm.len(),
|
||||||
|
480000,
|
||||||
|
"Vorbis packets must not overshoot model limit"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn classify(path: &Path) -> Option<MediaKind> {
|
||||||
|
match path.extension()?.to_str()?.to_ascii_lowercase().as_str() {
|
||||||
|
"png" | "jpg" | "jpeg" | "webp" | "bmp" => Some(MediaKind::Image),
|
||||||
|
"mp4" | "mkv" | "mov" | "avi" | "webm" | "m4v" | "mpeg" | "mpg" => Some(MediaKind::Video),
|
||||||
|
"mp3" | "flac" | "wav" | "m4a" | "ogg" | "opus" | "aac" | "aiff" | "aif" => {
|
||||||
|
Some(MediaKind::Audio)
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validate bounded raster decoding before treating model errors as fatal.
|
||||||
|
/// The provider still performs its own bounds and EXIF-aware preprocessing.
|
||||||
|
pub fn validate_image(path: &Path) -> Result<()> {
|
||||||
|
ensure!(
|
||||||
|
fs::metadata(path)?.len() <= 64 * 1024 * 1024,
|
||||||
|
"Image exceeds 64 MiB"
|
||||||
|
);
|
||||||
|
let reader = image::ImageReader::open(path)?.with_guessed_format()?;
|
||||||
|
let (width, height) = reader.into_dimensions()?;
|
||||||
|
ensure!(
|
||||||
|
width <= 16384 && height <= 16384 && u64::from(width) * u64::from(height) <= 16_000_000,
|
||||||
|
"Image exceeds 16M pixels"
|
||||||
|
);
|
||||||
|
let mut reader = image::ImageReader::open(path)?.with_guessed_format()?;
|
||||||
|
let mut limits = image::Limits::default();
|
||||||
|
limits.max_alloc = Some(256 * 1024 * 1024);
|
||||||
|
limits.max_image_width = Some(16384);
|
||||||
|
limits.max_image_height = Some(16384);
|
||||||
|
reader.limits(limits);
|
||||||
|
use image::ImageDecoder;
|
||||||
|
let mut decoder = reader.into_decoder()?;
|
||||||
|
decoder
|
||||||
|
.orientation()
|
||||||
|
.context("Invalid image EXIF orientation")?;
|
||||||
|
image::DynamicImage::from_decoder(decoder).context("Image decoder rejected corrupt content")?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct FfmpegTools {
|
||||||
|
pub ffmpeg: PathBuf,
|
||||||
|
pub ffprobe: PathBuf,
|
||||||
|
}
|
||||||
|
impl FfmpegTools {
|
||||||
|
pub fn new(ffmpeg: impl AsRef<Path>, ffprobe: impl AsRef<Path>) -> Result<Self> {
|
||||||
|
ensure!(
|
||||||
|
ffmpeg.as_ref().is_file() && ffprobe.as_ref().is_file(),
|
||||||
|
"FFmpeg/ffprobe executable is missing"
|
||||||
|
);
|
||||||
|
Ok(Self {
|
||||||
|
ffmpeg: fs::canonicalize(ffmpeg)?,
|
||||||
|
ffprobe: fs::canonicalize(ffprobe)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
pub fn from_environment() -> Result<Option<Self>> {
|
||||||
|
match (
|
||||||
|
std::env::var_os("SEARCHI_FFMPEG_PATH"),
|
||||||
|
std::env::var_os("SEARCHI_FFPROBE_PATH"),
|
||||||
|
) {
|
||||||
|
(None, None) => Ok(None),
|
||||||
|
(Some(ffmpeg), Some(ffprobe)) => Self::new(ffmpeg, ffprobe).map(Some),
|
||||||
|
_ => bail!("Set both SEARCHI_FFMPEG_PATH and SEARCHI_FFPROBE_PATH"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct MediaLimits {
|
||||||
|
pub max_file_bytes: u64,
|
||||||
|
pub max_duration_ms: u32,
|
||||||
|
pub max_frames: usize,
|
||||||
|
pub frame_interval_ms: u32,
|
||||||
|
pub max_scene_frames: usize,
|
||||||
|
pub audio_window_ms: u32,
|
||||||
|
pub audio_overlap_ms: u32,
|
||||||
|
pub max_audio_windows: usize,
|
||||||
|
pub command_timeout: Duration,
|
||||||
|
}
|
||||||
|
impl Default for MediaLimits {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
max_file_bytes: 16 * 1024 * 1024 * 1024,
|
||||||
|
max_duration_ms: 4 * 60 * 60 * 1000,
|
||||||
|
max_frames: 240,
|
||||||
|
frame_interval_ms: 2000,
|
||||||
|
max_scene_frames: 120,
|
||||||
|
audio_window_ms: 30_000,
|
||||||
|
audio_overlap_ms: 2000,
|
||||||
|
max_audio_windows: 600,
|
||||||
|
command_timeout: Duration::from_secs(45),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct MediaIdentity {
|
||||||
|
pub path: PathBuf,
|
||||||
|
pub bytes: u64,
|
||||||
|
pub modified_ns: i64,
|
||||||
|
pub sha256: String,
|
||||||
|
}
|
||||||
|
impl MediaIdentity {
|
||||||
|
pub fn read(path: &Path, maximum: u64) -> Result<Self> {
|
||||||
|
let file = File::open(path)?;
|
||||||
|
let before = file.metadata()?;
|
||||||
|
ensure!(
|
||||||
|
before.len() <= maximum,
|
||||||
|
"Media file exceeds {maximum} bytes"
|
||||||
|
);
|
||||||
|
let modified_ns = modified_ns(&before)?;
|
||||||
|
let mut reader = file.take(maximum.saturating_add(1));
|
||||||
|
let mut buffer = [0u8; 65536];
|
||||||
|
let mut hash = Sha256::new();
|
||||||
|
let mut count = 0u64;
|
||||||
|
loop {
|
||||||
|
let read = reader.read(&mut buffer)?;
|
||||||
|
if read == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
count += read as u64;
|
||||||
|
ensure!(count <= maximum, "Media grew past file budget");
|
||||||
|
hash.update(&buffer[..read]);
|
||||||
|
}
|
||||||
|
let identity = Self {
|
||||||
|
path: fs::canonicalize(path)?,
|
||||||
|
bytes: count,
|
||||||
|
modified_ns,
|
||||||
|
sha256: format!("{:x}", hash.finalize()),
|
||||||
|
};
|
||||||
|
ensure!(count == before.len(), "Media changed while fingerprinting");
|
||||||
|
identity.ensure_unchanged()?;
|
||||||
|
Ok(identity)
|
||||||
|
}
|
||||||
|
pub fn ensure_unchanged(&self) -> Result<()> {
|
||||||
|
let metadata = fs::metadata(&self.path)?;
|
||||||
|
ensure!(
|
||||||
|
metadata.len() == self.bytes && modified_ns(&metadata)? == self.modified_ns,
|
||||||
|
"Media changed during indexing"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn modified_ns(metadata: &fs::Metadata) -> Result<i64> {
|
||||||
|
i64::try_from(metadata.modified()?.duration_since(UNIX_EPOCH)?.as_nanos())
|
||||||
|
.context("Unsupported media timestamp")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct MediaProbe {
|
||||||
|
pub duration_ms: u32,
|
||||||
|
pub start_ms: i64,
|
||||||
|
pub video_stream_index: u32,
|
||||||
|
pub width: u32,
|
||||||
|
pub height: u32,
|
||||||
|
pub has_video: bool,
|
||||||
|
pub has_audio: bool,
|
||||||
|
pub audio_start_ms: u32,
|
||||||
|
pub audio_end_ms: u32,
|
||||||
|
pub tags: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct ProbeJson {
|
||||||
|
#[serde(default)]
|
||||||
|
streams: Vec<ProbeStream>,
|
||||||
|
format: ProbeFormat,
|
||||||
|
}
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct ProbeFormat {
|
||||||
|
duration: String,
|
||||||
|
#[serde(default)]
|
||||||
|
start_time: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
tags: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct ProbeStream {
|
||||||
|
index: u32,
|
||||||
|
codec_type: String,
|
||||||
|
#[serde(default)]
|
||||||
|
width: u32,
|
||||||
|
#[serde(default)]
|
||||||
|
height: u32,
|
||||||
|
#[serde(default)]
|
||||||
|
duration: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
start_time: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
tags: BTreeMap<String, String>,
|
||||||
|
#[serde(default)]
|
||||||
|
disposition: BTreeMap<String, i64>,
|
||||||
|
}
|
||||||
|
fn milliseconds(value: &str) -> Result<i64> {
|
||||||
|
let value: f64 = value.parse()?;
|
||||||
|
ensure!(
|
||||||
|
value.is_finite() && value.abs() < 100_000_000.0,
|
||||||
|
"Invalid media duration/timestamp"
|
||||||
|
);
|
||||||
|
Ok((value * 1000.0).round() as i64)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn probe(tools: &FfmpegTools, path: &Path, limits: &MediaLimits) -> Result<MediaProbe> {
|
||||||
|
let mut command = Command::new(&tools.ffprobe);
|
||||||
|
command.args(["-protocol_whitelist", "file,pipe"]);
|
||||||
|
command
|
||||||
|
.args([
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-threads",
|
||||||
|
"1",
|
||||||
|
"-probesize",
|
||||||
|
"5000000",
|
||||||
|
"-analyzeduration",
|
||||||
|
"5000000",
|
||||||
|
"-show_streams",
|
||||||
|
"-show_format",
|
||||||
|
"-of",
|
||||||
|
"json",
|
||||||
|
])
|
||||||
|
.arg(path);
|
||||||
|
let output = run(command, limits.command_timeout, 65536, 65536)?;
|
||||||
|
let parsed: ProbeJson =
|
||||||
|
serde_json::from_slice(&output.stdout).context("Invalid ffprobe JSON")?;
|
||||||
|
let duration = milliseconds(&parsed.format.duration)?;
|
||||||
|
ensure!(
|
||||||
|
duration > 0 && duration <= i64::from(limits.max_duration_ms),
|
||||||
|
"Media duration exceeds policy or is unknown"
|
||||||
|
);
|
||||||
|
let start = parsed
|
||||||
|
.format
|
||||||
|
.start_time
|
||||||
|
.as_deref()
|
||||||
|
.map(milliseconds)
|
||||||
|
.transpose()?
|
||||||
|
.unwrap_or(0);
|
||||||
|
let video = parsed.streams.iter().find(|stream| {
|
||||||
|
stream.codec_type == "video" && stream.disposition.get("attached_pic") != Some(&1)
|
||||||
|
});
|
||||||
|
if let Some(video) = video {
|
||||||
|
ensure!(
|
||||||
|
u64::from(video.width) * u64::from(video.height) <= 16_000_000,
|
||||||
|
"Video dimensions exceed 16M pixels"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let audio = parsed
|
||||||
|
.streams
|
||||||
|
.iter()
|
||||||
|
.find(|stream| stream.codec_type == "audio");
|
||||||
|
let audio_start = audio
|
||||||
|
.and_then(|stream| stream.start_time.as_deref())
|
||||||
|
.map(milliseconds)
|
||||||
|
.transpose()?
|
||||||
|
.unwrap_or(start)
|
||||||
|
- start;
|
||||||
|
let audio_length = audio
|
||||||
|
.and_then(|stream| stream.duration.as_deref())
|
||||||
|
.map(milliseconds)
|
||||||
|
.transpose()?
|
||||||
|
.unwrap_or(duration);
|
||||||
|
let mut tags = parsed.format.tags;
|
||||||
|
for stream in &parsed.streams {
|
||||||
|
for (key, value) in &stream.tags {
|
||||||
|
tags.entry(key.clone()).or_insert_with(|| value.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tags = tags
|
||||||
|
.into_iter()
|
||||||
|
.map(|(key, value)| {
|
||||||
|
(
|
||||||
|
key.to_ascii_lowercase(),
|
||||||
|
value.chars().take(2048).collect::<String>(),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.filter(|(key, _)| {
|
||||||
|
matches!(
|
||||||
|
key.as_str(),
|
||||||
|
"title"
|
||||||
|
| "artist"
|
||||||
|
| "album"
|
||||||
|
| "album_artist"
|
||||||
|
| "genre"
|
||||||
|
| "comment"
|
||||||
|
| "date"
|
||||||
|
| "composer"
|
||||||
|
| "description"
|
||||||
|
| "track"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.take(16)
|
||||||
|
.collect();
|
||||||
|
Ok(MediaProbe {
|
||||||
|
duration_ms: duration as u32,
|
||||||
|
start_ms: start,
|
||||||
|
video_stream_index: video.map_or(0, |stream| stream.index),
|
||||||
|
width: video.map_or(0, |stream| stream.width),
|
||||||
|
height: video.map_or(0, |stream| stream.height),
|
||||||
|
has_video: video.is_some(),
|
||||||
|
has_audio: audio.is_some(),
|
||||||
|
audio_start_ms: audio_start.max(0) as u32,
|
||||||
|
audio_end_ms: (audio_start + audio_length).clamp(0, duration) as u32,
|
||||||
|
tags,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Uniform coverage is distributed over the entire movie; this is not a prefix
|
||||||
|
/// truncation. Scene probes may add extra local cuts around these anchors.
|
||||||
|
pub fn frame_times(probe: &MediaProbe, limits: &MediaLimits) -> Result<Vec<u32>> {
|
||||||
|
if !probe.has_video {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
ensure!(
|
||||||
|
limits.max_frames >= 2 && limits.max_frames <= 2000,
|
||||||
|
"Frame budget must be 2..2000"
|
||||||
|
);
|
||||||
|
let last = probe.duration_ms.saturating_sub(500);
|
||||||
|
let interval = limits
|
||||||
|
.frame_interval_ms
|
||||||
|
.max(last.div_ceil((limits.max_frames - 1) as u32))
|
||||||
|
.max(1);
|
||||||
|
let mut times = Vec::new();
|
||||||
|
let mut time = 0;
|
||||||
|
while time < last {
|
||||||
|
times.push(time);
|
||||||
|
time = time.saturating_add(interval);
|
||||||
|
}
|
||||||
|
times.push(last);
|
||||||
|
ensure!(
|
||||||
|
times.len() <= limits.max_frames,
|
||||||
|
"Frame sampling budget exceeded"
|
||||||
|
);
|
||||||
|
Ok(times)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Serialize)]
|
||||||
|
pub struct AudioWindow {
|
||||||
|
pub start_ms: u32,
|
||||||
|
pub duration_ms: u32,
|
||||||
|
}
|
||||||
|
pub fn audio_windows(probe: &MediaProbe, limits: &MediaLimits) -> Result<Vec<AudioWindow>> {
|
||||||
|
if !probe.has_audio {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
ensure!(
|
||||||
|
limits.audio_window_ms >= 5_000
|
||||||
|
&& limits.audio_window_ms <= 30_000
|
||||||
|
&& limits.audio_overlap_ms < limits.audio_window_ms,
|
||||||
|
"Invalid audio window policy"
|
||||||
|
);
|
||||||
|
let step = limits.audio_window_ms - limits.audio_overlap_ms;
|
||||||
|
let mut windows = Vec::new();
|
||||||
|
let mut start = probe.audio_start_ms;
|
||||||
|
while start < probe.audio_end_ms {
|
||||||
|
ensure!(
|
||||||
|
windows.len() < limits.max_audio_windows,
|
||||||
|
"Whole audio coverage exceeds window budget; no partial index produced"
|
||||||
|
);
|
||||||
|
windows.push(AudioWindow {
|
||||||
|
start_ms: start,
|
||||||
|
duration_ms: limits.audio_window_ms.min(probe.audio_end_ms - start),
|
||||||
|
});
|
||||||
|
if start + limits.audio_window_ms >= probe.audio_end_ms {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
start = start.saturating_add(step);
|
||||||
|
}
|
||||||
|
Ok(windows)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct DecodedFrame {
|
||||||
|
pub path: PathBuf,
|
||||||
|
pub timestamp_ms: u32,
|
||||||
|
}
|
||||||
|
pub struct FrameBatch {
|
||||||
|
directory: tempfile::TempDir,
|
||||||
|
pub frames: Vec<DecodedFrame>,
|
||||||
|
}
|
||||||
|
impl FrameBatch {
|
||||||
|
pub fn directory(&self) -> &Path {
|
||||||
|
self.directory.path()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn decode_frames(
|
||||||
|
tools: &FfmpegTools,
|
||||||
|
path: &Path,
|
||||||
|
probe: &MediaProbe,
|
||||||
|
start_ms: u32,
|
||||||
|
include_scene: bool,
|
||||||
|
limits: &MediaLimits,
|
||||||
|
) -> Result<FrameBatch> {
|
||||||
|
let directory = tempfile::tempdir()?;
|
||||||
|
let mut command = Command::new(&tools.ffmpeg);
|
||||||
|
command.args(["-protocol_whitelist", "file,pipe"]);
|
||||||
|
command.args(["-nostdin","-hide_banner","-loglevel","info","-threads","1","-hwaccel","none","-copyts","-ss"])
|
||||||
|
.arg(format!("{:.3}",f64::from(start_ms)/1000.0)).args(["-t",if include_scene {"1.5"} else {"0.5"},"-i"]).arg(path)
|
||||||
|
.arg("-map").arg(format!("0:{}",probe.video_stream_index)).args(["-an","-filter_threads","1","-vf"])
|
||||||
|
.arg(if include_scene {"select=eq(n\\,0)+gt(scene\\,0.32),showinfo,scale=960:960:force_original_aspect_ratio=decrease"}
|
||||||
|
else {"showinfo,scale=960:960:force_original_aspect_ratio=decrease"})
|
||||||
|
.args(["-frames:v",if include_scene {"2"} else {"1"},"-fps_mode","vfr","-threads","1","-y"])
|
||||||
|
.arg(directory.path().join("frame-%02d.png"));
|
||||||
|
let output = run(command, limits.command_timeout, 4096, 256 * 1024)?;
|
||||||
|
let mut timestamps = Vec::new();
|
||||||
|
for line in String::from_utf8_lossy(&output.stderr).lines() {
|
||||||
|
if line.contains("showinfo") {
|
||||||
|
if let Some((_, tail)) = line.split_once("pts_time:") {
|
||||||
|
let value = tail
|
||||||
|
.split_whitespace()
|
||||||
|
.next()
|
||||||
|
.context("Missing frame PTS")?;
|
||||||
|
let actual = (milliseconds(value)? - probe.start_ms).max(0);
|
||||||
|
ensure!(
|
||||||
|
actual <= i64::from(probe.duration_ms) + 1000,
|
||||||
|
"Frame PTS lies beyond media timeline"
|
||||||
|
);
|
||||||
|
timestamps.push(actual as u32);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut frames = Vec::new();
|
||||||
|
for ordinal in 1..=if include_scene { 2 } else { 1 } {
|
||||||
|
let file = directory.path().join(format!("frame-{ordinal:02}.png"));
|
||||||
|
if file.is_file() {
|
||||||
|
ensure!(
|
||||||
|
fs::metadata(&file)?.len() <= 8 * 1024 * 1024,
|
||||||
|
"Decoded frame exceeds 8MiB"
|
||||||
|
);
|
||||||
|
let timestamp_ms = *timestamps
|
||||||
|
.get(ordinal - 1)
|
||||||
|
.context("Decoder did not report genuine frame PTS")?;
|
||||||
|
frames.push(DecodedFrame {
|
||||||
|
path: file,
|
||||||
|
timestamp_ms,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ensure!(
|
||||||
|
!frames.is_empty(),
|
||||||
|
"Decoder produced no frame at requested timeline position"
|
||||||
|
);
|
||||||
|
Ok(FrameBatch { directory, frames })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decode_audio(
|
||||||
|
tools: &FfmpegTools,
|
||||||
|
path: &Path,
|
||||||
|
window: AudioWindow,
|
||||||
|
limits: &MediaLimits,
|
||||||
|
) -> Result<Vec<f32>> {
|
||||||
|
ensure!(
|
||||||
|
window.duration_ms > 0 && window.duration_ms <= 30_000,
|
||||||
|
"Audio decode window exceeds 30 seconds"
|
||||||
|
);
|
||||||
|
let mut command = Command::new(&tools.ffmpeg);
|
||||||
|
command.args(["-protocol_whitelist", "file,pipe"]);
|
||||||
|
command
|
||||||
|
.args([
|
||||||
|
"-nostdin",
|
||||||
|
"-hide_banner",
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-threads",
|
||||||
|
"1",
|
||||||
|
"-ss",
|
||||||
|
])
|
||||||
|
.arg(format!("{:.3}", f64::from(window.start_ms) / 1000.0))
|
||||||
|
.args(["-t"])
|
||||||
|
.arg(format!("{:.3}", f64::from(window.duration_ms) / 1000.0))
|
||||||
|
.arg("-i")
|
||||||
|
.arg(path)
|
||||||
|
.arg("-af")
|
||||||
|
.arg(format!(
|
||||||
|
"aresample=16000,atrim=end_sample={},asetpts=PTS-STARTPTS",
|
||||||
|
u64::from(window.duration_ms) * 16
|
||||||
|
))
|
||||||
|
.args([
|
||||||
|
"-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000", "-threads", "1", "-f", "s16le",
|
||||||
|
"pipe:1",
|
||||||
|
]);
|
||||||
|
let output = run(command, limits.command_timeout, 960_000, 65536)?;
|
||||||
|
ensure!(
|
||||||
|
!output.stdout.is_empty() && output.stdout.len() % 2 == 0,
|
||||||
|
"Empty or invalid decoded PCM audio"
|
||||||
|
);
|
||||||
|
Ok(output
|
||||||
|
.stdout
|
||||||
|
.as_chunks::<2>()
|
||||||
|
.0
|
||||||
|
.iter()
|
||||||
|
.map(|bytes| f32::from(i16::from_le_bytes([bytes[0], bytes[1]])) / 32768.0)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Output {
|
||||||
|
stdout: Vec<u8>,
|
||||||
|
stderr: Vec<u8>,
|
||||||
|
}
|
||||||
|
fn run(
|
||||||
|
mut command: Command,
|
||||||
|
timeout: Duration,
|
||||||
|
stdout_limit: usize,
|
||||||
|
stderr_limit: usize,
|
||||||
|
) -> Result<Output> {
|
||||||
|
#[cfg(windows)]
|
||||||
|
{
|
||||||
|
use std::os::windows::process::CommandExt;
|
||||||
|
command.creation_flags(0x08000000);
|
||||||
|
}
|
||||||
|
let mut child = command
|
||||||
|
.stdin(Stdio::null())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.context("Launch configured FFmpeg tool")?;
|
||||||
|
let stdout = child.stdout.take().context("Missing decoder stdout")?;
|
||||||
|
let stderr = child.stderr.take().context("Missing decoder stderr")?;
|
||||||
|
let read = |reader: Box<dyn Read + Send>, limit: usize| {
|
||||||
|
thread::spawn(move || -> Result<Vec<u8>> {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
reader.take(limit as u64 + 1).read_to_end(&mut bytes)?;
|
||||||
|
ensure!(bytes.len() <= limit, "Decoder output exceeded byte budget");
|
||||||
|
Ok(bytes)
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let out = read(Box::new(stdout), stdout_limit);
|
||||||
|
let err = read(Box::new(stderr), stderr_limit);
|
||||||
|
let start = Instant::now();
|
||||||
|
let status = loop {
|
||||||
|
if let Some(status) = child.try_wait()? {
|
||||||
|
break Some(status);
|
||||||
|
}
|
||||||
|
if start.elapsed() > timeout {
|
||||||
|
child.kill()?;
|
||||||
|
child.wait()?;
|
||||||
|
break None;
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(10));
|
||||||
|
};
|
||||||
|
let stdout = out
|
||||||
|
.join()
|
||||||
|
.map_err(|_| anyhow::anyhow!("Decoder stdout reader panicked"))??;
|
||||||
|
let stderr = err
|
||||||
|
.join()
|
||||||
|
.map_err(|_| anyhow::anyhow!("Decoder stderr reader panicked"))??;
|
||||||
|
let status = status.context("Decoder exceeded time budget")?;
|
||||||
|
ensure!(
|
||||||
|
status.success(),
|
||||||
|
"Decoder failed: {}",
|
||||||
|
String::from_utf8_lossy(&stderr)
|
||||||
|
.chars()
|
||||||
|
.take(2048)
|
||||||
|
.collect::<String>()
|
||||||
|
);
|
||||||
|
Ok(Output { stdout, stderr })
|
||||||
|
}
|
||||||
@@ -11,6 +11,9 @@ minilm-proof = ["dep:ort", "dep:tokenizers"]
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
anyhow.workspace = true
|
anyhow.workspace = true
|
||||||
|
base64 = "0.22"
|
||||||
|
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "bmp"] }
|
||||||
|
jpeg-decoder = { version = "0.3.2", default-features = false }
|
||||||
ort = { version = "=2.0.0-rc.13", optional = true, default-features = false, features = ["std", "load-dynamic", "api-27"] }
|
ort = { version = "=2.0.0-rc.13", optional = true, default-features = false, features = ["std", "load-dynamic", "api-27"] }
|
||||||
sha2 = "0.10"
|
sha2 = "0.10"
|
||||||
tokenizers = { version = "=0.22.2", optional = true, default-features = false, features = ["fancy-regex"] }
|
tokenizers = { version = "=0.22.2", optional = true, default-features = false, features = ["fancy-regex"] }
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# Google worker image decoding
|
||||||
|
|
||||||
|
The production adapter verifies model/projector SHA256 and performs genuine
|
||||||
|
Google MTMD image/audio inference. Images are decoded with established Rust
|
||||||
|
libraries, not filename captions. `WorkerInfo.media_supported` comes from the
|
||||||
|
native handshake; absent fields on older text workers mean false. Media APIs
|
||||||
|
return an explicit runtime repair error for such workers while text remains
|
||||||
|
compatible.
|
||||||
|
|
||||||
|
JPEG sources up to 64 Mi pixels (including 48 MP camera photos), 16,384 pixels
|
||||||
|
per side, and 64 MiB compressed size are supported. For JPEG above 16 Mi pixels,
|
||||||
|
`image-rs/jpeg-decoder` 0.3.2 applies scaled IDCT before final model resizing.
|
||||||
|
Its default features are disabled, so this path adds no Rayon worker pool.
|
||||||
|
The source dimensions are checked before pixels are decoded. The secondary
|
||||||
|
header must match the first to detect file changes. RGB24 and grayscale camera
|
||||||
|
JPEGs are supported on this scaled path; unusual large CMYK/16-bit JPEGs fail
|
||||||
|
explicitly rather than silently taking an unbounded full-resolution fallback.
|
||||||
|
|
||||||
|
The published decoder offers 1/8, 1/4, 1/2, and full-scale decoding. It is in
|
||||||
|
maintenance mode; the current image 0.25 zune JPEG adapter exposes no scaled
|
||||||
|
IDCT interface. See the [decoder API](https://docs.rs/jpeg-decoder/0.3.2/jpeg_decoder/struct.Decoder.html)
|
||||||
|
and [upstream maintenance note](https://github.com/image-rs/jpeg-decoder).
|
||||||
|
Small JPEGs retain the existing image/zune path and pinned parity behavior.
|
||||||
|
|
||||||
|
PNG, WebP, and BMP remain capped at 16 Mi pixels with explicit diagnostics.
|
||||||
|
Both paths enforce a 256 MiB advertised decoding-buffer ceiling and apply
|
||||||
|
EXIF orientation. That ceiling is a buffer/output bound, not an absolute
|
||||||
|
process RSS guarantee: compressed data, codec working state, and resizing
|
||||||
|
have additional allocations. Extreme/progressive files need broader resource
|
||||||
|
profiling. The media fingerprint includes `largejpeg-scaled-v2`; generations
|
||||||
|
must additionally include visual token budget. Text fingerprints are unchanged.
|
||||||
|
|
||||||
|
An executed Windows CPU release test encodes and decodes an actual 8000x6000
|
||||||
|
JPEG with EXIF rotation, verifies orientation and region colors, and resizes it
|
||||||
|
to a 280-token grid. Decode plus resize took 176–220 ms and produced 672x912
|
||||||
|
RGB (1,838,592 bytes). A sampled whole-test peak was 146.86 MiB, including
|
||||||
|
144 MB of source fixture construction; it is not a decoder-only measurement
|
||||||
|
or a semantic quality benchmark. An 81 MP JPEG header is rejected before
|
||||||
|
pixel decode in the default tests.
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cargo test --release -p searchi-semantic large_phone_jpeg -- --ignored --nocapture
|
||||||
|
cargo test -p searchi-semantic --lib
|
||||||
|
```
|
||||||
@@ -41,11 +41,12 @@ set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "No runtime tools" FORCE)
|
|||||||
set(LLAMA_BUILD_SERVER OFF CACHE BOOL "No HTTP server" FORCE)
|
set(LLAMA_BUILD_SERVER OFF CACHE BOOL "No HTTP server" FORCE)
|
||||||
set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "No examples" FORCE)
|
set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "No examples" FORCE)
|
||||||
set(LLAMA_BUILD_TESTS OFF CACHE BOOL "No upstream tests in worker build" FORCE)
|
set(LLAMA_BUILD_TESTS OFF CACHE BOOL "No upstream tests in worker build" FORCE)
|
||||||
set(LLAMA_BUILD_MTMD OFF CACHE BOOL "Text worker; media encoder follows separately" FORCE)
|
set(LLAMA_BUILD_MTMD ON CACHE BOOL "Google image/audio projector" FORCE)
|
||||||
|
set(MTMD_VIDEO OFF CACHE BOOL "Frames decoded by bounded media adapter" FORCE)
|
||||||
set(LLAMA_OPENSSL OFF CACHE BOOL "No network runtime dependencies" FORCE)
|
set(LLAMA_OPENSSL OFF CACHE BOOL "No network runtime dependencies" FORCE)
|
||||||
add_subdirectory("${LLAMA_CPP_DIR}" llama-build)
|
add_subdirectory("${LLAMA_CPP_DIR}" llama-build)
|
||||||
add_executable(searchi-gemma-worker worker.cpp)
|
add_executable(searchi-gemma-worker worker.cpp)
|
||||||
target_include_directories(searchi-gemma-worker PRIVATE "${LLAMA_CPP_DIR}/vendor")
|
target_include_directories(searchi-gemma-worker PRIVATE "${LLAMA_CPP_DIR}/vendor")
|
||||||
target_link_libraries(searchi-gemma-worker PRIVATE llama)
|
target_link_libraries(searchi-gemma-worker PRIVATE llama mtmd)
|
||||||
target_compile_definitions(searchi-gemma-worker PRIVATE SEARCHI_LLAMA_TAG="b11461-no-logits-v1")
|
target_compile_definitions(searchi-gemma-worker PRIVATE SEARCHI_LLAMA_TAG="b11461-no-logits-v1")
|
||||||
install(TARGETS searchi-gemma-worker RUNTIME DESTINATION bin)
|
install(TARGETS searchi-gemma-worker RUNTIME DESTINATION bin)
|
||||||
@@ -1,11 +1,14 @@
|
|||||||
// Local NDJSON helper around the pinned llama.cpp encoder. stdout is exclusively
|
// Local NDJSON helper around the pinned llama.cpp encoder. stdout is exclusively
|
||||||
// protocol output; llama diagnostics go to stderr. No model download or HTTP.
|
// protocol output; llama diagnostics go to stderr. No model download or HTTP.
|
||||||
#include "llama.h"
|
#include "llama.h"
|
||||||
|
#include "mtmd.h"
|
||||||
#include "nlohmann/json.hpp"
|
#include "nlohmann/json.hpp"
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
|
#include <chrono>
|
||||||
#include <cstdlib>
|
#include <cstdlib>
|
||||||
#include <cstdio>
|
#include <cstdio>
|
||||||
|
#include <cstring>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
@@ -13,7 +16,7 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
using json = nlohmann::json;
|
using json = nlohmann::json;
|
||||||
constexpr size_t max_request_bytes = 2 * 1024 * 1024;
|
constexpr size_t max_request_bytes = 16 * 1024 * 1024;
|
||||||
constexpr size_t max_passage_bytes = 1024 * 1024;
|
constexpr size_t max_passage_bytes = 1024 * 1024;
|
||||||
constexpr size_t max_text_bytes = 65536;
|
constexpr size_t max_text_bytes = 65536;
|
||||||
constexpr size_t max_windows = 256;
|
constexpr size_t max_windows = 256;
|
||||||
@@ -34,12 +37,14 @@ class Encoder {
|
|||||||
llama_context * context = nullptr;
|
llama_context * context = nullptr;
|
||||||
llama_batch_ext * batch = nullptr;
|
llama_batch_ext * batch = nullptr;
|
||||||
const llama_vocab * vocab = nullptr;
|
const llama_vocab * vocab = nullptr;
|
||||||
|
mtmd_context * media = nullptr;
|
||||||
std::string model_path;
|
std::string model_path;
|
||||||
void release() {
|
void release() {
|
||||||
|
if (media) mtmd_free(media);
|
||||||
if (batch) llama_batch_ext_free(batch);
|
if (batch) llama_batch_ext_free(batch);
|
||||||
if (context) llama_free(context);
|
if (context) llama_free(context);
|
||||||
if (model) llama_model_free(model);
|
if (model) llama_model_free(model);
|
||||||
batch = nullptr; context = nullptr; model = nullptr; vocab = nullptr;
|
batch = nullptr; context = nullptr; model = nullptr; vocab = nullptr; media = nullptr;
|
||||||
}
|
}
|
||||||
bool open(bool gpu) {
|
bool open(bool gpu) {
|
||||||
auto mp = llama_model_default_params();
|
auto mp = llama_model_default_params();
|
||||||
@@ -68,6 +73,9 @@ public:
|
|||||||
llama_log_set([](ggml_log_level level, const char * text, void *) {
|
llama_log_set([](ggml_log_level level, const char * text, void *) {
|
||||||
if (level >= GGML_LOG_LEVEL_WARN) std::fputs(text,stderr);
|
if (level >= GGML_LOG_LEVEL_WARN) std::fputs(text,stderr);
|
||||||
},nullptr);
|
},nullptr);
|
||||||
|
mtmd_log_set([](ggml_log_level level, const char * text, void *) {
|
||||||
|
if (level >= GGML_LOG_LEVEL_WARN) std::fputs(text,stderr);
|
||||||
|
},nullptr);
|
||||||
llama_backend_init();
|
llama_backend_init();
|
||||||
bool use_gpu = want_gpu && llama_supports_gpu_offload();
|
bool use_gpu = want_gpu && llama_supports_gpu_offload();
|
||||||
require(open(use_gpu) || (use_gpu && open(false)),"Cannot initialize local EmbeddingGemma 2 model/context");
|
require(open(use_gpu) || (use_gpu && open(false)),"Cannot initialize local EmbeddingGemma 2 model/context");
|
||||||
@@ -77,6 +85,52 @@ public:
|
|||||||
llama_backend_free();
|
llama_backend_free();
|
||||||
}
|
}
|
||||||
bool healthy() const {return context != nullptr && batch != nullptr;}
|
bool healthy() const {return context != nullptr && batch != nullptr;}
|
||||||
|
void ensure_media(const std::string & projector) {
|
||||||
|
if (media) return;
|
||||||
|
auto params = mtmd_context_params_default();
|
||||||
|
params.use_gpu = device == "gpu"; params.n_threads = threads;
|
||||||
|
params.print_timings = false; params.warmup = false;
|
||||||
|
params.image_min_tokens = 1; params.image_max_tokens = 1120;
|
||||||
|
media = mtmd_init_from_file(projector.c_str(),model,params);
|
||||||
|
require_inference(media != nullptr,"Cannot initialize Google media projector");
|
||||||
|
}
|
||||||
|
std::vector<float> embed_bitmap(mtmd_bitmap * bitmap) {
|
||||||
|
std::unique_ptr<mtmd_input_chunks,decltype(&mtmd_input_chunks_free)> chunks(mtmd_input_chunks_init(),mtmd_input_chunks_free);
|
||||||
|
require_inference(chunks != nullptr,"Cannot allocate media chunks");
|
||||||
|
const char * marker = mtmd_default_marker();
|
||||||
|
mtmd_input_text text{marker,std::strlen(marker),true,true};
|
||||||
|
const mtmd_bitmap * bitmaps[] = {bitmap};
|
||||||
|
require_inference(mtmd_tokenize(media,chunks.get(),&text,bitmaps,1) == 0,"Media tokenization failed");
|
||||||
|
llama_batch_ext_clear(batch); size_t position = 0;
|
||||||
|
auto place = [&](int index) {
|
||||||
|
require_inference(index >= 0 && position < context_tokens,"Media batch exceeds context");
|
||||||
|
llama_pos pos[4] = {llama_pos(position++),0,0,0};
|
||||||
|
require_inference(llama_batch_ext_set_pos(batch,index,pos) && llama_batch_ext_set_output_embd(batch,index,true),"Cannot place media row");
|
||||||
|
};
|
||||||
|
for (size_t i=0;i<mtmd_input_chunks_size(chunks.get());++i) {
|
||||||
|
const auto * chunk = mtmd_input_chunks_get(chunks.get(),i);
|
||||||
|
if (mtmd_input_chunk_get_type(chunk) == MTMD_INPUT_CHUNK_TYPE_TEXT) {
|
||||||
|
size_t count=0; const auto * tokens=mtmd_input_chunk_get_tokens_text(chunk,&count);
|
||||||
|
for(size_t j=0;j<count;++j) place(llama_batch_ext_add_token(batch,0,tokens[j]));
|
||||||
|
} else {
|
||||||
|
require_inference(mtmd_encode_chunk(media,chunk)==0,"Google media encoder failed");
|
||||||
|
const float * rows=mtmd_get_output_embd(media);
|
||||||
|
require_inference(rows != nullptr,"No projected media rows");
|
||||||
|
size_t width=llama_model_n_embd_inp(model);
|
||||||
|
for(size_t j=0;j<mtmd_input_chunk_get_n_tokens(chunk);++j) {
|
||||||
|
llama_embd embd{rows+j*width,1,width};
|
||||||
|
place(llama_batch_ext_add_embd(batch,0,embd));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require_inference(llama_process(context,LLAMA_PROCESS_TYPE_ENCODE,batch)==0,"Media text encoder failed");
|
||||||
|
const float * pooled=llama_get_embeddings_seq(context,0);
|
||||||
|
require_inference(pooled != nullptr,"No pooled media vector");
|
||||||
|
double norm=0; for(int i=0;i<dimensions;++i){require_inference(std::isfinite(pooled[i]),"Nonfinite media vector");norm+=double(pooled[i])*pooled[i];}
|
||||||
|
require_inference(norm>0 && std::isfinite(norm),"Invalid media vector norm");
|
||||||
|
std::vector<float> vector(dimensions);for(int i=0;i<dimensions;++i)vector[i]=float(pooled[i]/std::sqrt(norm));
|
||||||
|
return vector;
|
||||||
|
}
|
||||||
void retry_cpu() {
|
void retry_cpu() {
|
||||||
require(device == "gpu","Only a GPU inference failure can request a CPU retry");
|
require(device == "gpu","Only a GPU inference failure can request a CPU retry");
|
||||||
release();
|
release();
|
||||||
@@ -103,10 +157,21 @@ public:
|
|||||||
require(title.size() <= 512, "Document title exceeds 512 UTF-8 bytes");
|
require(title.size() <= 512, "Document title exceeds 512 UTF-8 bytes");
|
||||||
return "title: " + (title.empty() ? "none" : title) + " | text: ";
|
return "title: " + (title.empty() ? "none" : title) + " | text: ";
|
||||||
}
|
}
|
||||||
std::vector<float> embed(const std::string & text, int cap) {
|
std::vector<float> embed(const std::string & text, int cap, json * timings = nullptr) {
|
||||||
|
using clock = std::chrono::steady_clock;
|
||||||
|
auto started = timings ? clock::now() : clock::time_point{};
|
||||||
|
auto mark = [&](const char * name) {
|
||||||
|
if (timings) {
|
||||||
|
auto now = clock::now();
|
||||||
|
(*timings)[name] = std::chrono::duration_cast<std::chrono::microseconds>(now-started).count();
|
||||||
|
started = now;
|
||||||
|
}
|
||||||
|
};
|
||||||
require(!text.empty() && text.size() <= max_text_bytes, "Embedding text must be nonempty and at most 64 KiB");
|
require(!text.empty() && text.size() <= max_text_bytes, "Embedding text must be nonempty and at most 64 KiB");
|
||||||
auto tokens = tokenize(text,true);
|
auto tokens = tokenize(text,true);
|
||||||
require(!tokens.empty() && tokens.size() <= size_t(cap), "Prompt exceeds model token budget; split document windows first");
|
require(!tokens.empty() && tokens.size() <= size_t(cap), "Prompt exceeds model token budget; split document windows first");
|
||||||
|
if (timings) (*timings)["token_count"] = tokens.size();
|
||||||
|
mark("tokenization_us");
|
||||||
llama_batch_ext_clear(batch);
|
llama_batch_ext_clear(batch);
|
||||||
for (size_t i = 0; i < tokens.size(); ++i) {
|
for (size_t i = 0; i < tokens.size(); ++i) {
|
||||||
int index = llama_batch_ext_add_token(batch,0,tokens[i]);
|
int index = llama_batch_ext_add_token(batch,0,tokens[i]);
|
||||||
@@ -115,7 +180,9 @@ public:
|
|||||||
require(llama_batch_ext_set_pos(batch,index,position), "Cannot set encoder position");
|
require(llama_batch_ext_set_pos(batch,index,position), "Cannot set encoder position");
|
||||||
require(llama_batch_ext_set_output_embd(batch,index,true), "Cannot select embedding output");
|
require(llama_batch_ext_set_output_embd(batch,index,true), "Cannot select embedding output");
|
||||||
}
|
}
|
||||||
|
mark("batch_prepare_us");
|
||||||
require_inference(llama_process(context,LLAMA_PROCESS_TYPE_ENCODE,batch) == 0, "llama_process encoder failed");
|
require_inference(llama_process(context,LLAMA_PROCESS_TYPE_ENCODE,batch) == 0, "llama_process encoder failed");
|
||||||
|
mark("inference_us");
|
||||||
const float * pooled = llama_get_embeddings_seq(context,0);
|
const float * pooled = llama_get_embeddings_seq(context,0);
|
||||||
require_inference(pooled != nullptr, "Encoder returned no pooled vector");
|
require_inference(pooled != nullptr, "Encoder returned no pooled vector");
|
||||||
double norm = 0;
|
double norm = 0;
|
||||||
@@ -123,6 +190,7 @@ public:
|
|||||||
require_inference(norm > 0 && std::isfinite(norm), "Zero or nonfinite embedding norm");
|
require_inference(norm > 0 && std::isfinite(norm), "Zero or nonfinite embedding norm");
|
||||||
std::vector<float> vector(dimensions);
|
std::vector<float> vector(dimensions);
|
||||||
for (int i = 0; i < dimensions; ++i) vector[i] = float(pooled[i]/std::sqrt(norm));
|
for (int i = 0; i < dimensions; ++i) vector[i] = float(pooled[i]/std::sqrt(norm));
|
||||||
|
mark("pool_normalize_us");
|
||||||
return vector;
|
return vector;
|
||||||
}
|
}
|
||||||
json windows(const std::string & text, const std::string & title) const {
|
json windows(const std::string & text, const std::string & title) const {
|
||||||
@@ -157,8 +225,42 @@ public:
|
|||||||
}
|
}
|
||||||
json handle(const json & request) {
|
json handle(const json & request) {
|
||||||
auto op = request.at("op").get<std::string>();
|
auto op = request.at("op").get<std::string>();
|
||||||
|
if(op=="image" || op=="audio") {
|
||||||
|
auto encoded=request.at("data").get<std::string>();
|
||||||
|
require(encoded.size()<=12000000 && encoded.size()%4==0,"Invalid media payload length");
|
||||||
|
static const std::string alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||||
|
std::vector<unsigned char> bytes; bytes.reserve(encoded.size()/4*3);
|
||||||
|
unsigned accumulator=0;int bits=0;bool padding=false;
|
||||||
|
for(char c:encoded){if(c=='='){padding=true;continue;} require(!padding,"Invalid base64 padding");auto n=alphabet.find(c);require(n!=std::string::npos,"Invalid base64");accumulator=(accumulator<<6)|unsigned(n);bits+=6;if(bits>=8){bits-=8;bytes.push_back(static_cast<unsigned char>(accumulator>>bits));}}
|
||||||
|
auto projector=request.at("projector").get<std::string>();
|
||||||
|
require(!projector.empty() && projector.size()<32768,"Invalid projector path");
|
||||||
|
ensure_media(projector);
|
||||||
|
mtmd_bitmap * raw=nullptr;
|
||||||
|
if(op=="image") {
|
||||||
|
int w=request.at("width"),h=request.at("height");
|
||||||
|
require(w>=48 && h>=48 && w%48==0 && h%48==0 && uint64_t(w)*h<=1120*48*48,"Invalid bounded image dimensions");
|
||||||
|
require(bytes.size()==size_t(w)*h*3,"RGB image length mismatch");
|
||||||
|
raw=mtmd_bitmap_init(w,h,bytes.data());
|
||||||
|
} else {
|
||||||
|
require(bytes.size()%4==0 && !bytes.empty() && bytes.size()/4<=30*16000,"Audio must be at most30seconds mono16kHz PCM");
|
||||||
|
std::vector<float> pcm(bytes.size()/4);
|
||||||
|
std::memcpy(pcm.data(),bytes.data(),bytes.size());
|
||||||
|
for(float f:pcm)require(std::isfinite(f) && std::abs(f)<=1,"Audio PCM outside normalized range");
|
||||||
|
raw=mtmd_bitmap_init_from_audio(pcm.size(),pcm.data());
|
||||||
|
}
|
||||||
|
std::unique_ptr<mtmd_bitmap,decltype(&mtmd_bitmap_free)> bitmap(raw,mtmd_bitmap_free);
|
||||||
|
require_inference(bitmap != nullptr,"Cannot allocate media bitmap");
|
||||||
|
return {{"vectors",json::array({embed_bitmap(bitmap.get())})}};
|
||||||
|
}
|
||||||
if (op == "windows") return {{"windows",windows(request.at("text").get<std::string>(),request.value("title",std::string()))}};
|
if (op == "windows") return {{"windows",windows(request.at("text").get<std::string>(),request.value("title",std::string()))}};
|
||||||
if (op == "query") return {{"vectors",json::array({embed("task: search result | query: "+request.at("text").get<std::string>(),512)})}};
|
if (op == "query") {
|
||||||
|
bool debug = request.value("debug_performance",false);
|
||||||
|
json timings;
|
||||||
|
auto vector = embed("task: search result | query: "+request.at("text").get<std::string>(),512,debug ? &timings : nullptr);
|
||||||
|
json response = {{"vectors",json::array({vector})}};
|
||||||
|
if (debug) response["performance"] = timings;
|
||||||
|
return response;
|
||||||
|
}
|
||||||
if (op == "documents") {
|
if (op == "documents") {
|
||||||
const auto & texts = request.at("texts");
|
const auto & texts = request.at("texts");
|
||||||
require(texts.is_array() && !texts.empty() && texts.size() <= 8,"Batch size must be between 1 and 8");
|
require(texts.is_array() && !texts.empty() && texts.size() <= 8,"Batch size must be between 1 and 8");
|
||||||
@@ -187,11 +289,11 @@ int main(int argc, char ** argv) {
|
|||||||
bool force_cpu = std::getenv("SEARCHI_FORCE_CPU") && std::string(std::getenv("SEARCHI_FORCE_CPU")) == "1";
|
bool force_cpu = std::getenv("SEARCHI_FORCE_CPU") && std::string(std::getenv("SEARCHI_FORCE_CPU")) == "1";
|
||||||
Encoder encoder(path,threads,device == "auto" && !force_cpu);
|
Encoder encoder(path,threads,device == "auto" && !force_cpu);
|
||||||
std::cout << json({{"ready",true},{"model","google/embeddinggemma-2"},{"dimensions",dimensions},
|
std::cout << json({{"ready",true},{"model","google/embeddinggemma-2"},{"dimensions",dimensions},
|
||||||
{"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG}}).dump() << '\n' << std::flush;
|
{"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true}}).dump() << '\n' << std::flush;
|
||||||
for (;;) {
|
for (;;) {
|
||||||
std::string line; char c;
|
std::string line; char c;
|
||||||
while (std::cin.get(c) && c != '\n') {
|
while (std::cin.get(c) && c != '\n') {
|
||||||
require(line.size() < max_request_bytes,"Request exceeds 2 MiB"); line.push_back(c);
|
require(line.size() < max_request_bytes,"Request exceeds 16 MiB"); line.push_back(c);
|
||||||
}
|
}
|
||||||
if (!std::cin && line.empty()) break;
|
if (!std::cin && line.empty()) break;
|
||||||
json response;
|
json response;
|
||||||
|
|||||||
@@ -1,24 +1,35 @@
|
|||||||
use crate::TextWindow;
|
use crate::TextWindow;
|
||||||
use anyhow::{ensure, Context, Result};
|
use anyhow::{ensure, Context, Result};
|
||||||
|
use base64::Engine;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::{
|
use std::{
|
||||||
|
collections::BTreeMap,
|
||||||
fs::File,
|
fs::File,
|
||||||
io::{BufRead, BufReader, Read, Write},
|
io::{BufRead, BufReader, Read, Write},
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
process::{Child, ChildStdin, Command, Stdio},
|
process::{Child, ChildStdin, Command, Stdio},
|
||||||
sync::mpsc::{self, Receiver},
|
sync::mpsc::{self, Receiver},
|
||||||
thread,
|
thread,
|
||||||
time::Duration,
|
time::{Duration, Instant},
|
||||||
};
|
};
|
||||||
|
|
||||||
pub const GEMMA_DIMENSIONS: usize = 768;
|
pub const GEMMA_DIMENSIONS: usize = 768;
|
||||||
pub const GEMMA_FINGERPRINT: &str = "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1";
|
pub const GEMMA_FINGERPRINT: &str = "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1";
|
||||||
|
pub const GEMMA_MEDIA_FINGERPRINT: &str = "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mtmd:mean:l2:rgb-catmullrom48-exif-largejpeg-scaled-v2:pcm16k30s-v1";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct ImageEmbedding {
|
||||||
|
pub vector: Vec<f32>,
|
||||||
|
pub encoded_width: u32,
|
||||||
|
pub encoded_height: u32,
|
||||||
|
pub token_budget: u32,
|
||||||
|
}
|
||||||
const MODEL_FILE: &str = "embeddinggemma-2-Q8_0.gguf";
|
const MODEL_FILE: &str = "embeddinggemma-2-Q8_0.gguf";
|
||||||
const MODEL_BYTES: u64 = 309855456;
|
const MODEL_BYTES: u64 = 309855456;
|
||||||
const MODEL_SHA256: &str = "2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135";
|
const MODEL_SHA256: &str = "2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135";
|
||||||
const MAX_REQUEST: usize = 2 * 1024 * 1024;
|
const MAX_REQUEST: usize = 16 * 1024 * 1024;
|
||||||
const MAX_RESPONSE: usize = 4 * 1024 * 1024;
|
const MAX_RESPONSE: usize = 4 * 1024 * 1024;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
@@ -35,6 +46,8 @@ pub struct WorkerInfo {
|
|||||||
pub device: String,
|
pub device: String,
|
||||||
pub threads: usize,
|
pub threads: usize,
|
||||||
pub runtime: String,
|
pub runtime: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub media_supported: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Persistent isolated native worker. No HTTP or remote model fallback.
|
/// Persistent isolated native worker. No HTTP or remote model fallback.
|
||||||
@@ -47,6 +60,8 @@ pub struct GemmaEmbedder {
|
|||||||
force_cpu: bool,
|
force_cpu: bool,
|
||||||
model_dir: PathBuf,
|
model_dir: PathBuf,
|
||||||
worker_path: PathBuf,
|
worker_path: PathBuf,
|
||||||
|
projector_verified: bool,
|
||||||
|
query_profile: Option<BTreeMap<String, u64>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for GemmaEmbedder {
|
impl Drop for GemmaEmbedder {
|
||||||
@@ -161,6 +176,8 @@ impl GemmaEmbedder {
|
|||||||
force_cpu: cpu,
|
force_cpu: cpu,
|
||||||
model_dir,
|
model_dir,
|
||||||
worker_path: worker,
|
worker_path: worker,
|
||||||
|
projector_verified: false,
|
||||||
|
query_profile: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,10 +186,12 @@ impl GemmaEmbedder {
|
|||||||
self.healthy,
|
self.healthy,
|
||||||
"Gemma worker session failed; reopen it explicitly"
|
"Gemma worker session failed; reopen it explicitly"
|
||||||
);
|
);
|
||||||
|
let encode_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||||
let encoded = serde_json::to_vec(&request)?;
|
let encoded = serde_json::to_vec(&request)?;
|
||||||
|
self.record_query_duration("request_encode_us", encode_started);
|
||||||
ensure!(
|
ensure!(
|
||||||
encoded.len() < MAX_REQUEST,
|
encoded.len() < MAX_REQUEST,
|
||||||
"Gemma request exceeds 2 MiB protocol budget"
|
"Gemma request exceeds 16 MiB protocol budget"
|
||||||
);
|
);
|
||||||
let response = match self.exchange(&encoded) {
|
let response = match self.exchange(&encoded) {
|
||||||
Ok(response) => response,
|
Ok(response) => response,
|
||||||
@@ -180,6 +199,7 @@ impl GemmaEmbedder {
|
|||||||
// GPU runtimes can abort rather than return an encoder error.
|
// GPU runtimes can abort rather than return an encoder error.
|
||||||
// Restart once as CPU and retry this same bounded request.
|
// Restart once as CPU and retry this same bounded request.
|
||||||
let _ = self.child.wait();
|
let _ = self.child.wait();
|
||||||
|
let restart_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||||
let replacement = Self::load_with_device(
|
let replacement = Self::load_with_device(
|
||||||
&self.model_dir,
|
&self.model_dir,
|
||||||
&self.worker_path,
|
&self.worker_path,
|
||||||
@@ -189,7 +209,13 @@ impl GemmaEmbedder {
|
|||||||
.with_context(|| {
|
.with_context(|| {
|
||||||
format!("CPU worker restart failed after GPU transport error: {error}")
|
format!("CPU worker restart failed after GPU transport error: {error}")
|
||||||
})?;
|
})?;
|
||||||
|
let diagnostics = self.query_profile.take();
|
||||||
*self = replacement;
|
*self = replacement;
|
||||||
|
self.query_profile = diagnostics;
|
||||||
|
self.record_query_duration("cpu_restart_us", restart_started);
|
||||||
|
if let Some(profile) = &mut self.query_profile {
|
||||||
|
profile.insert("cpu_restart_count".into(), 1);
|
||||||
|
}
|
||||||
self.exchange(&encoded)?
|
self.exchange(&encoded)?
|
||||||
}
|
}
|
||||||
Err(error) => return Err(error),
|
Err(error) => return Err(error),
|
||||||
@@ -201,6 +227,11 @@ impl GemmaEmbedder {
|
|||||||
);
|
);
|
||||||
self.info.device = device.into();
|
self.info.device = device.into();
|
||||||
}
|
}
|
||||||
|
if response["fallback"].as_str() == Some("gpu_inference_failure") {
|
||||||
|
if let Some(profile) = &mut self.query_profile {
|
||||||
|
profile.insert("worker_cpu_retry_count".into(), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
if response["fatal"] == true {
|
if response["fatal"] == true {
|
||||||
self.healthy = false;
|
self.healthy = false;
|
||||||
self.info.ready = false;
|
self.info.ready = false;
|
||||||
@@ -215,18 +246,21 @@ impl GemmaEmbedder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn exchange(&mut self, encoded: &[u8]) -> Result<Value> {
|
fn exchange(&mut self, encoded: &[u8]) -> Result<Value> {
|
||||||
|
let write_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||||
let sent = (|| {
|
let sent = (|| {
|
||||||
self.stdin.write_all(encoded)?;
|
self.stdin.write_all(encoded)?;
|
||||||
self.stdin.write_all(b"\n")?;
|
self.stdin.write_all(b"\n")?;
|
||||||
self.stdin.flush()
|
self.stdin.flush()
|
||||||
})()
|
})()
|
||||||
.context("Cannot write local Gemma worker request");
|
.context("Cannot write local Gemma worker request");
|
||||||
|
self.record_query_duration("request_write_us", write_started);
|
||||||
if let Err(error) = sent {
|
if let Err(error) = sent {
|
||||||
self.healthy = false;
|
self.healthy = false;
|
||||||
self.info.ready = false;
|
self.info.ready = false;
|
||||||
let _ = self.child.kill();
|
let _ = self.child.kill();
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
|
let wait_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||||
let response = match self.responses.recv_timeout(Duration::from_secs(180)) {
|
let response = match self.responses.recv_timeout(Duration::from_secs(180)) {
|
||||||
Ok(Ok(value)) => value,
|
Ok(Ok(value)) => value,
|
||||||
other => {
|
other => {
|
||||||
@@ -236,6 +270,7 @@ impl GemmaEmbedder {
|
|||||||
anyhow::bail!("Gemma worker request failed: {other:?}");
|
anyhow::bail!("Gemma worker request failed: {other:?}");
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
self.record_query_duration("worker_wait_and_response_decode_us", wait_started);
|
||||||
Ok(response)
|
Ok(response)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -281,15 +316,265 @@ impl GemmaEmbedder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn embed_query(&mut self, query: &str) -> Result<Vec<f32>> {
|
pub fn embed_query(&mut self, query: &str) -> Result<Vec<f32>> {
|
||||||
|
self.query_profile = None;
|
||||||
|
self.embed_query_inner(query, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opt-in diagnostics contain only durations and counts, never query text.
|
||||||
|
pub fn embed_query_debug(&mut self, query: &str) -> Result<Vec<f32>> {
|
||||||
|
self.query_profile = Some(BTreeMap::new());
|
||||||
|
self.embed_query_inner(query, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn query_diagnostics(&self) -> BTreeMap<String, u64> {
|
||||||
|
self.query_profile.clone().unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_query_duration(&mut self, name: &str, started: Option<Instant>) {
|
||||||
|
if let (Some(profile), Some(started)) = (&mut self.query_profile, started) {
|
||||||
|
let micros = started.elapsed().as_micros().min(u64::MAX as u128) as u64;
|
||||||
|
*profile.entry(name.into()).or_default() += micros;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn embed_query_inner(&mut self, query: &str, debug: bool) -> Result<Vec<f32>> {
|
||||||
ensure!(
|
ensure!(
|
||||||
!query.trim().is_empty() && query.len() <= 4096,
|
!query.trim().is_empty() && query.len() <= 4096,
|
||||||
"Query must be nonempty and at most 4096 UTF-8 bytes"
|
"Query must be nonempty and at most 4096 UTF-8 bytes"
|
||||||
);
|
);
|
||||||
let response = self.request(json!({"op":"query","text":query}))?;
|
let response =
|
||||||
|
self.request(json!({"op":"query","text":query,"debug_performance":debug}))?;
|
||||||
|
if debug {
|
||||||
|
if let Some(profile) = &mut self.query_profile {
|
||||||
|
for name in [
|
||||||
|
"tokenization_us",
|
||||||
|
"batch_prepare_us",
|
||||||
|
"inference_us",
|
||||||
|
"pool_normalize_us",
|
||||||
|
"token_count",
|
||||||
|
] {
|
||||||
|
if let Some(value) = response["performance"][name].as_u64() {
|
||||||
|
profile.insert(format!("worker_{name}"), value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let validation_started = self.query_profile.as_ref().map(|_| Instant::now());
|
||||||
|
let result = extract_vectors(response, 1)?.remove(0);
|
||||||
|
self.record_query_duration("response_validate_us", validation_started);
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn projector(&mut self) -> Result<PathBuf> {
|
||||||
|
ensure!(
|
||||||
|
self.info.media_supported,
|
||||||
|
"Installed embedding worker does not support media; repair/rebuild the native runtime"
|
||||||
|
);
|
||||||
|
let path = self.model_dir.join("mmproj-embeddinggemma-2-Q8_0.gguf");
|
||||||
|
if !self.projector_verified {
|
||||||
|
verify_artifact(
|
||||||
|
&path,
|
||||||
|
554821024,
|
||||||
|
"c4a8a52691ecef40618438928bdf9e68379b854e24166f292592353db0aab64f",
|
||||||
|
)?;
|
||||||
|
self.projector_verified = true;
|
||||||
|
}
|
||||||
|
Ok(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Genuine Google pixel embeddings, serial bounded image operations.
|
||||||
|
pub fn embed_images(&mut self, paths: &[&Path]) -> Result<Vec<Vec<f32>>> {
|
||||||
|
self.embed_images_with_budget(paths, 280)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 70 tokens for video frames, 140 for photos, 280 or 1120 for screenshots.
|
||||||
|
pub fn embed_images_with_budget(
|
||||||
|
&mut self,
|
||||||
|
paths: &[&Path],
|
||||||
|
budget: u32,
|
||||||
|
) -> Result<Vec<Vec<f32>>> {
|
||||||
|
Ok(self
|
||||||
|
.embed_images_with_metadata(paths, budget)?
|
||||||
|
.into_iter()
|
||||||
|
.map(|image| image.vector)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn embed_images_with_metadata(
|
||||||
|
&mut self,
|
||||||
|
paths: &[&Path],
|
||||||
|
budget: u32,
|
||||||
|
) -> Result<Vec<ImageEmbedding>> {
|
||||||
|
ensure!(
|
||||||
|
!paths.is_empty() && paths.len() <= 8,
|
||||||
|
"Image batch must contain 1–8 paths"
|
||||||
|
);
|
||||||
|
let projector = self.projector()?;
|
||||||
|
let mut vectors = Vec::with_capacity(paths.len());
|
||||||
|
for path in paths {
|
||||||
|
let pixels = decode_image(path, budget)?;
|
||||||
|
let response=self.request(json!({"op":"image","projector":projector,"width":pixels.width(),"height":pixels.height(),"data":base64::engine::general_purpose::STANDARD.encode(pixels.as_raw())}))?;
|
||||||
|
vectors.push(ImageEmbedding {
|
||||||
|
vector: extract_vectors(response, 1)?.remove(0),
|
||||||
|
encoded_width: pixels.width(),
|
||||||
|
encoded_height: pixels.height(),
|
||||||
|
token_budget: budget,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(vectors)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One mono 16 kHz normalized PCM window, at most 30 seconds. Callers
|
||||||
|
/// segment entire recordings and preserve real timestamps; no truncation.
|
||||||
|
pub fn embed_audio_pcm(&mut self, samples: &[f32]) -> Result<Vec<f32>> {
|
||||||
|
ensure!(
|
||||||
|
!samples.is_empty() && samples.len() <= 480000,
|
||||||
|
"Audio window must contain 1–480000 mono 16 kHz samples"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
samples.iter().all(|v| v.is_finite() && v.abs() <= 1.0),
|
||||||
|
"Invalid normalized audio PCM"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
cfg!(target_endian = "little"),
|
||||||
|
"Native PCM worker requires little-endian host"
|
||||||
|
);
|
||||||
|
let projector = self.projector()?;
|
||||||
|
let bytes: Vec<u8> = samples.iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||||
|
let response=self.request(json!({"op":"audio","projector":projector,"data":base64::engine::general_purpose::STANDARD.encode(bytes)}))?;
|
||||||
Ok(extract_vectors(response, 1)?.remove(0))
|
Ok(extract_vectors(response, 1)?.remove(0))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn decode_image(path: &Path, budget: u32) -> Result<image::RgbImage> {
|
||||||
|
ensure!(
|
||||||
|
(70..=1120).contains(&budget),
|
||||||
|
"Image token budget must be 70–1120"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
std::fs::metadata(path)?.len() <= 64 * 1024 * 1024,
|
||||||
|
"Image file exceeds 64 MiB"
|
||||||
|
);
|
||||||
|
let mut reader = image::ImageReader::open(path)?.with_guessed_format()?;
|
||||||
|
let is_jpeg = reader.format() == Some(image::ImageFormat::Jpeg);
|
||||||
|
let mut limits = image::Limits::default();
|
||||||
|
limits.max_alloc = Some(256 * 1024 * 1024);
|
||||||
|
limits.max_image_width = Some(16384);
|
||||||
|
limits.max_image_height = Some(16384);
|
||||||
|
reader.limits(limits);
|
||||||
|
use image::ImageDecoder;
|
||||||
|
let mut decoder = reader
|
||||||
|
.into_decoder()
|
||||||
|
.context("Cannot initialize bounded image decoder")?;
|
||||||
|
let (source_width, source_height) = decoder.dimensions();
|
||||||
|
ensure!(
|
||||||
|
u64::from(source_width) * u64::from(source_height) <= 64 * 1024 * 1024
|
||||||
|
&& decoder.total_bytes() <= 256 * 1024 * 1024,
|
||||||
|
"Decoded image exceeds memory/pixel budget"
|
||||||
|
);
|
||||||
|
let orientation = decoder
|
||||||
|
.orientation()
|
||||||
|
.context("Cannot read image EXIF orientation")?;
|
||||||
|
if is_jpeg && u64::from(source_width) * u64::from(source_height) > 16 * 1024 * 1024 {
|
||||||
|
drop(decoder);
|
||||||
|
return decode_large_jpeg(path, budget, source_width, source_height, orientation);
|
||||||
|
}
|
||||||
|
ensure!(u64::from(source_width)*u64::from(source_height)<=16*1024*1024,"Non-JPEG image exceeds16megapixels; bounded scaled decoding is currently available for JPEG");
|
||||||
|
let mut decoded = image::DynamicImage::from_decoder(decoder)
|
||||||
|
.context("Cannot decode supported image pixels")?;
|
||||||
|
decoded.apply_orientation(orientation);
|
||||||
|
ensure!(
|
||||||
|
u64::from(decoded.width()) * u64::from(decoded.height()) <= 16 * 1024 * 1024,
|
||||||
|
"Image exceeds 16 megapixels"
|
||||||
|
);
|
||||||
|
let (w, h) = image_target(decoded.width(), decoded.height(), budget)?;
|
||||||
|
Ok(image::imageops::resize(
|
||||||
|
&decoded.to_rgb8(),
|
||||||
|
w,
|
||||||
|
h,
|
||||||
|
image::imageops::FilterType::CatmullRom,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_large_jpeg(
|
||||||
|
path: &Path,
|
||||||
|
budget: u32,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
orientation: image::metadata::Orientation,
|
||||||
|
) -> Result<image::RgbImage> {
|
||||||
|
let (target_width, target_height) = image_target(width, height, budget)?;
|
||||||
|
let mut decoder = jpeg_decoder::Decoder::new(BufReader::new(File::open(path)?));
|
||||||
|
decoder.set_max_decoding_buffer_size(256 * 1024 * 1024);
|
||||||
|
decoder
|
||||||
|
.read_info()
|
||||||
|
.context("Cannot read scaled JPEG header")?;
|
||||||
|
let source = decoder.info().context("Missing scaled JPEG header")?;
|
||||||
|
ensure!(
|
||||||
|
u32::from(source.width) == width && u32::from(source.height) == height,
|
||||||
|
"JPEG changed during decoding"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
matches!(
|
||||||
|
source.pixel_format,
|
||||||
|
jpeg_decoder::PixelFormat::RGB24 | jpeg_decoder::PixelFormat::L8
|
||||||
|
),
|
||||||
|
"Large JPEG requires supported RGB24 or grayscale scaled decoding"
|
||||||
|
);
|
||||||
|
let (w, h) = decoder.scale(u16::try_from(target_width)?, u16::try_from(target_height)?)?;
|
||||||
|
ensure!(
|
||||||
|
u64::from(w) * u64::from(h) * 4 <= 256 * 1024 * 1024,
|
||||||
|
"Scaled JPEG output exceeds memory budget"
|
||||||
|
);
|
||||||
|
let pixels = decoder
|
||||||
|
.decode()
|
||||||
|
.context("Cannot decode large JPEG with scaled IDCT")?;
|
||||||
|
let mut decoded = match source.pixel_format {
|
||||||
|
jpeg_decoder::PixelFormat::RGB24 => image::DynamicImage::ImageRgb8(
|
||||||
|
image::RgbImage::from_raw(u32::from(w), u32::from(h), pixels)
|
||||||
|
.context("Invalid scaled JPEG RGB length")?,
|
||||||
|
),
|
||||||
|
jpeg_decoder::PixelFormat::L8 => image::DynamicImage::ImageLuma8(
|
||||||
|
image::GrayImage::from_raw(u32::from(w), u32::from(h), pixels)
|
||||||
|
.context("Invalid scaled JPEG grayscale length")?,
|
||||||
|
),
|
||||||
|
other => anyhow::bail!(
|
||||||
|
"Large JPEG pixel format {other:?} is not supported by bounded scaled decoder"
|
||||||
|
),
|
||||||
|
};
|
||||||
|
decoded.apply_orientation(orientation);
|
||||||
|
let (w, h) = image_target(decoded.width(), decoded.height(), budget)?;
|
||||||
|
Ok(image::imageops::resize(
|
||||||
|
&decoded.into_rgb8(),
|
||||||
|
w,
|
||||||
|
h,
|
||||||
|
image::imageops::FilterType::CatmullRom,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn image_target(width: u32, height: u32, budget: u32) -> Result<(u32, u32)> {
|
||||||
|
ensure!(
|
||||||
|
width > 0 && height > 0 && (70..=1120).contains(&budget),
|
||||||
|
"Invalid image dimensions/budget"
|
||||||
|
);
|
||||||
|
let scale = (f64::from(budget) * 2304.0 / (f64::from(width) * f64::from(height))).sqrt();
|
||||||
|
let mut w = ((f64::from(width) * scale / 48.0).floor() as u32) * 48;
|
||||||
|
let mut h = ((f64::from(height) * scale / 48.0).floor() as u32) * 48;
|
||||||
|
if w == 0 {
|
||||||
|
w = 48;
|
||||||
|
h = (height / width * 48).min(budget * 48);
|
||||||
|
}
|
||||||
|
if h == 0 {
|
||||||
|
h = 48;
|
||||||
|
w = (width / height * 48).min(budget * 48);
|
||||||
|
}
|
||||||
|
ensure!(
|
||||||
|
w >= 48 && h >= 48 && u64::from(w) * u64::from(h) <= u64::from(budget) * 2304,
|
||||||
|
"Image target outside token budget"
|
||||||
|
);
|
||||||
|
Ok((w, h))
|
||||||
|
}
|
||||||
|
|
||||||
fn can_retry_cpu(actual_device: &str, force_cpu: bool) -> bool {
|
fn can_retry_cpu(actual_device: &str, force_cpu: bool) -> bool {
|
||||||
actual_device == "gpu" && !force_cpu
|
actual_device == "gpu" && !force_cpu
|
||||||
}
|
}
|
||||||
@@ -331,6 +616,10 @@ fn extract_vectors(response: Value, count: usize) -> Result<Vec<Vec<f32>>> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn verify_model(path: &Path) -> Result<()> {
|
fn verify_model(path: &Path) -> Result<()> {
|
||||||
|
verify_artifact(path, MODEL_BYTES, MODEL_SHA256)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_artifact(path: &Path, bytes: u64, sha: &str) -> Result<()> {
|
||||||
let file = File::open(path).with_context(|| {
|
let file = File::open(path).with_context(|| {
|
||||||
format!(
|
format!(
|
||||||
"Missing Google EmbeddingGemma 2 model {}; run explicit model setup first",
|
"Missing Google EmbeddingGemma 2 model {}; run explicit model setup first",
|
||||||
@@ -338,7 +627,7 @@ fn verify_model(path: &Path) -> Result<()> {
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
ensure!(
|
ensure!(
|
||||||
file.metadata()?.len() == MODEL_BYTES,
|
file.metadata()?.len() == bytes,
|
||||||
"EmbeddingGemma 2 model size mismatch"
|
"EmbeddingGemma 2 model size mismatch"
|
||||||
);
|
);
|
||||||
let mut reader = BufReader::new(file);
|
let mut reader = BufReader::new(file);
|
||||||
@@ -352,7 +641,7 @@ fn verify_model(path: &Path) -> Result<()> {
|
|||||||
hash.update(&buffer[..count]);
|
hash.update(&buffer[..count]);
|
||||||
}
|
}
|
||||||
ensure!(
|
ensure!(
|
||||||
format!("{:x}", hash.finalize()) == MODEL_SHA256,
|
format!("{:x}", hash.finalize()) == sha,
|
||||||
"EmbeddingGemma 2 model SHA256 mismatch"
|
"EmbeddingGemma 2 model SHA256 mismatch"
|
||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -362,6 +651,101 @@ fn verify_model(path: &Path) -> Result<()> {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
#[test]
|
#[test]
|
||||||
|
fn old_text_worker_handshake_does_not_claim_media_support() {
|
||||||
|
let mut handshake = json!({"ready":true,"model":"google/embeddinggemma-2","dimensions":768,"device":"cpu","threads":1,"runtime":"b11461-no-logits-v1"});
|
||||||
|
let old: WorkerInfo = serde_json::from_value(handshake.clone()).unwrap();
|
||||||
|
assert!(!old.media_supported);
|
||||||
|
handshake["media_supported"] = json!(true);
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<WorkerInfo>(handshake)
|
||||||
|
.unwrap()
|
||||||
|
.media_supported
|
||||||
|
);
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn media_image_grid_is_bounded_for_extreme_aspects_and_full_budget() {
|
||||||
|
for budget in [70, 140, 280, 1120] {
|
||||||
|
for (width, height) in [(1, 16384), (16384, 1), (1920, 1080), (500, 500)] {
|
||||||
|
let (w, h) = image_target(width, height, budget).unwrap();
|
||||||
|
assert_eq!(w % 48, 0);
|
||||||
|
assert_eq!(h % 48, 0);
|
||||||
|
assert!(u64::from(w) * u64::from(h) <= u64::from(budget) * 2304);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(image_target(0, 100, 280).is_err());
|
||||||
|
assert!(image_target(100, 100, 1121).is_err());
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn artifact_verification_rejects_projector_size_and_checksum_mismatch() {
|
||||||
|
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/LICENSE-DigUp.txt");
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
let sha = format!("{:x}", Sha256::digest(&bytes));
|
||||||
|
assert!(verify_artifact(&path, bytes.len() as u64, &sha).is_ok());
|
||||||
|
assert!(verify_artifact(&path, bytes.len() as u64 + 1, &sha).is_err());
|
||||||
|
assert!(verify_artifact(&path, bytes.len() as u64, &"0".repeat(64)).is_err());
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
#[ignore = "Explicit48MP JPEG resource test; no model needed, run release CPU"]
|
||||||
|
fn large_phone_jpeg_scaled_decode_preserves_exif_and_bounds() {
|
||||||
|
use image::ImageEncoder;
|
||||||
|
let image = image::RgbImage::from_fn(8000, 6000, |x, _| {
|
||||||
|
if x < 4000 {
|
||||||
|
image::Rgb([240, 10, 10])
|
||||||
|
} else {
|
||||||
|
image::Rgb([10, 10, 240])
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let mut encoded = Vec::new();
|
||||||
|
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut encoded, 80);
|
||||||
|
// Little-endian TIFF orientation tag6: rotate90clockwise.
|
||||||
|
encoder
|
||||||
|
.set_exif_metadata(vec![
|
||||||
|
b'I', b'I', 42, 0, 8, 0, 0, 0, 1, 0, 0x12, 1, 3, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0,
|
||||||
|
0, 0,
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
encoder.encode_image(&image).unwrap();
|
||||||
|
drop(image);
|
||||||
|
let path =
|
||||||
|
std::env::temp_dir().join(format!("searchi-phone48mp-{}.jpg", std::process::id()));
|
||||||
|
std::fs::write(&path, &encoded).unwrap();
|
||||||
|
drop(encoded);
|
||||||
|
let started = std::time::Instant::now();
|
||||||
|
let result = decode_image(&path, 280);
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
let pixels = result.unwrap();
|
||||||
|
println!(
|
||||||
|
"48MPactualJPEG scaleddecode+EXIF+resize {:?}; output{}x{} RGB{}bytes",
|
||||||
|
started.elapsed(),
|
||||||
|
pixels.width(),
|
||||||
|
pixels.height(),
|
||||||
|
pixels.as_raw().len()
|
||||||
|
);
|
||||||
|
assert!(pixels.height() > pixels.width());
|
||||||
|
assert!(pixels.get_pixel(pixels.width() / 2, pixels.height() / 4)[0] > 200);
|
||||||
|
assert!(pixels.get_pixel(pixels.width() / 2, pixels.height() * 3 / 4)[2] > 200);
|
||||||
|
assert!(u64::from(pixels.width()) * u64::from(pixels.height()) <= 280 * 2304);
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn oversized_jpeg_header_rejected_before_pixel_decode() {
|
||||||
|
let mut encoded = Vec::new();
|
||||||
|
image::codecs::jpeg::JpegEncoder::new(&mut encoded)
|
||||||
|
.encode_image(&image::RgbImage::new(16, 16))
|
||||||
|
.unwrap();
|
||||||
|
let sof = encoded.windows(2).position(|p| p == [0xff, 0xc0]).unwrap();
|
||||||
|
encoded[sof + 5..sof + 7].copy_from_slice(&9000u16.to_be_bytes());
|
||||||
|
encoded[sof + 7..sof + 9].copy_from_slice(&9000u16.to_be_bytes());
|
||||||
|
let path =
|
||||||
|
std::env::temp_dir().join(format!("searchi-too-large-{}.jpg", std::process::id()));
|
||||||
|
std::fs::write(&path, encoded).unwrap();
|
||||||
|
let result = decode_image(&path, 280);
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
assert!(result
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string()
|
||||||
|
.contains("memory/pixel budget"));
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
fn missing_google_model_fails_without_starting_worker() {
|
fn missing_google_model_fails_without_starting_worker() {
|
||||||
assert!(GemmaEmbedder::load("missing-google-model", "missing-worker", 1).is_err());
|
assert!(GemmaEmbedder::load("missing-google-model", "missing-worker", 1).is_err());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
//! Local Google EmbeddingGemma 2 inference through an isolated native worker.
|
//! Local Google EmbeddingGemma 2 inference through an isolated native worker.
|
||||||
//! The rejected MiniLM experiment is available only under `minilm-proof`.
|
//! The rejected MiniLM experiment is available only under `minilm-proof`.
|
||||||
mod gemma;
|
mod gemma;
|
||||||
pub use gemma::{Device, GemmaEmbedder, WorkerInfo, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT};
|
pub use gemma::{
|
||||||
|
Device, GemmaEmbedder, ImageEmbedding, WorkerInfo, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT,
|
||||||
|
GEMMA_MEDIA_FINGERPRINT,
|
||||||
|
};
|
||||||
|
|
||||||
/// A verified source byte range covered by a model token window.
|
/// A verified source byte range covered by a model token window.
|
||||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
# Pinned Google media reference fixtures
|
||||||
|
|
||||||
|
The `digup-photo.jpg`, `digup-screen.png`, `digup-speech.wav`, and
|
||||||
|
`digup-media-reference.json` files are copied unchanged from
|
||||||
|
`ARahim3/DigUp`, commit `6faa29e4c315a3a591de1b41bd78751bdf39588d`,
|
||||||
|
`Tests/LlamaRuntimeTests/Reference/`. They and the existing text reference
|
||||||
|
fixture are retained under the adjacent `LICENSE-DigUp.txt` MIT notice.
|
||||||
|
|
||||||
|
The source test uses image budgets 280 (screen), 140 (photo), 70 (frame),
|
||||||
|
and a mono 16 kHz speech recording. The native CPU test compares actual
|
||||||
|
768-dimensional Google vectors with the pinned llama.cpp b11461 reference
|
||||||
|
at cosine >=0.999. This is runtime parity evidence, not a production media
|
||||||
|
relevance corpus. Independent images, audio, and video require separate
|
||||||
|
quality evaluation.
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"made_with":"llama.cpp b11461","vector_version":1,"vectors":{"document":[0.0023304052,-0.021797102,-0.025986522,-0.025894934,0.0046086097,-0.029744277,-0.04186634,0.03234369,0.015701862,-0.13444579,-0.049679857,-0.067124754,-0.008888296,0.049783457,0.04075608,-0.0027419934,0.0494676,0.0092225745,-0.003042357,0.021539804,-0.066572726,-0.04184304,-0.008146685,-0.015800655,0.0028940293,0.026850378,0.023585942,0.012522315,-0.018817414,0.19809185,-0.0541878,-0.033899978,0.06363426,-0.012184333,0.026218394,-0.030783277,-0.016514625,0.0010143976,0.016249185,0.00673781,0.07569905,-0.032087695,0.01618515,0.008316216,0.009405525,0.035788074,0.011760172,-0.0018986556,0.03811712,0.04686964,-0.0019338331,-0.06858261,-0.020950682,0.0061196685,-0.012488069,0.055386145,0.00828158,-0.028973866,0.029866058,0.019298207,0.0064102146,0.03210796,0.031232368,-0.024290783,-0.009193791,-0.00788917,0.07007842,-0.036232397,0.048318822,0.03329114,-0.021271994,-0.01810213,0.056723867,0.026635557,-0.0066668643,-0.03339305,-0.09275154,-0.005457068,-0.037602138,0.06852603,-0.086277,0.0108406255,-0.022301262,0.054799795,0.048695795,-0.0094250385,0.04326083,-0.05584021,0.009427591,0.018164966,0.0077456376,0.012890579,0.011140378,0.009403773,-0.013322473,0.014014791,0.0045265807,0.0026267597,-0.02777846,0.03337617,-0.014311906,-0.0027436074,0.0016559366,0.04267254,-0.0074306116,0.010555651,0.00801582,0.019190526,0.026427642,-0.02305468,0.017346293,-0.254231,-0.037654083,0.004254816,-0.029559055,0.30139065,-0.0017701226,0.012945749,0.045082387,-0.005745192,0.023981802,0.11611545,0.02849787,0.0050043385,0.019032571,0.01543062,0.013486391,-0.0073420904,-0.0120351305,0.027507395,0.017718071,0.0048503615,0.0070377868,-0.00946547,0.015142535,0.013139711,0.019906517,-0.038481727,0.060138095,0.0027306078,-0.010124241,0.020947324,-0.005082965,0.009129657,-0.06960446,0.04814667,0.012469531,-0.017168682,-0.007835558,-0.018339474,0.04499317,-0.048769984,-0.021295255,0.0008460758,0.042394932,0.02629723,0.015862137,-0.027827857,0.02877754,0.044078708,0.02864664,0.015502866,-0.011931367,0.006051371,-0.0676744,-0.032051444,-0.0051915697,0.022663914,0.010766299,-0.045957707,-0.071434505,-0.0071970616,-0.00080499344,-0.019176258,0.039329063,0.07615527,-0.02934876,0.03178097,0.030727895,-0.0014563488,0.033275653,-0.016663045,0.016650142,0.03149862,-0.042355653,-0.01447139,-0.031222051,-0.0371221,-0.071084455,0.02895739,0.012906462,-0.034588896,0.005292062,0.04567936,0.057948757,0.016161263,0.026154758,0.037441123,0.020264363,0.0021852585,0.010503882,0.02564951,-0.0034963347,-0.002975977,-0.05896342,0.090469755,-0.02141068,-0.018266847,0.03979448,0.006404804,0.048994806,-0.0013431136,0.003767527,0.009192836,0.03172842,0.0018301627,-0.032883424,0.070755646,-0.008929331,0.007731602,0.039488316,-0.0026821592,0.047416102,0.017939921,-0.046493225,-0.003571527,0.017453436,0.027679116,-0.045083456,-0.0018627502,-0.0058596916,-0.016741525,-0.041559335,-0.07398639,0.035852198,-0.01250213,-0.027620463,-0.022970477,-0.007722595,0.006369175,-0.027909646,0.05087572,0.04319605,0.0249135,-0.045503743,-0.050388955,-0.07399011,-0.031614304,-0.009977491,-0.011919013,0.017587284,0.046735547,0.051120915,-0.01913088,-0.0040965537,-0.030126736,0.022246089,-0.024694057,-0.0011967963,-0.0076620826,-0.0001045683,0.025630128,0.0023922755,-0.016251149,0.029261485,0.026775412,-0.0011348778,-0.023079926,0.0046075485,0.013615946,-0.022271913,0.0063756416,-0.011097959,-0.057059642,0.017879428,0.039326653,-0.010385263,-0.024296982,-0.0056849266,-0.021441974,0.023893535,-0.004312103,-0.0029023795,0.011346968,-0.05022057,-0.00962676,-0.030292084,0.0022458949,-0.0012382208,0.014016518,0.0130033605,-0.0075721424,0.022480633,0.0378293,-0.0060748337,0.020296352,-0.038013346,0.02737668,0.051704004,0.029559156,-0.008555367,-0.040540576,0.0028047708,-0.01888827,-0.017475538,0.022049233,-0.018655647,0.00035712068,0.0007242559,-0.031585597,0.002809768,-0.010847336,0.019464653,-0.038227003,0.004094894,-0.0045381063,-0.032818098,-0.025449943,-0.0048597767,-0.052715924,0.024247607,-0.025617195,-0.028776655,-0.01491966,0.027141426,0.022653097,0.01347185,0.0010195217,0.0077305734,0.006165836,-0.009121481,-0.011312435,0.0010023145,0.033321623,0.0020216978,-0.04601626,-0.020557044,-0.023531808,0.013099387,0.049464565,-0.0083905,-0.12218178,-0.0090951715,-0.037435923,-0.043180246,0.008593723,0.003133877,0.03239143,-0.02740052,0.028175933,0.00713765,0.010822759,-0.011402951,-0.042760924,-0.03611006,0.011457286,0.060568936,0.0027335465,-0.024305783,-0.004420147,0.0068441983,0.002837347,-0.02218109,-0.022486687,-0.01941801,0.01667096,-0.03958557,-0.0077348636,-0.0012779543,-0.009598079,-0.008962751,0.033793483,0.018625712,-0.006633453,0.026242023,0.021685809,0.013799533,-0.0060389466,0.0020937775,0.02297482,-0.02592566,0.0011351658,0.00793861,-0.01809629,0.050504446,-0.026299631,-0.028017156,0.006029957,-0.02648004,0.03030571,0.016809817,0.29351303,-0.028621448,0.01694766,0.021401705,0.012320341,0.0015229117,-0.Line truncated
|
||||||
|
After Width: | Height: | Size: 165 KiB |
|
After Width: | Height: | Size: 6.3 KiB |
@@ -1,6 +1,49 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use searchi_semantic::{Device, GemmaEmbedder, GEMMA_DIMENSIONS};
|
use searchi_semantic::{Device, GemmaEmbedder, GEMMA_DIMENSIONS};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "Requires explicitly provisioned Google model and CPU worker"]
|
||||||
|
fn google_cpu_debug_query_reports_actual_worker_stages_without_changing_vectors() -> Result<()> {
|
||||||
|
let model = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set SEARCHI_GEMMA_MODEL_DIR");
|
||||||
|
let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set SEARCHI_GEMMA_WORKER");
|
||||||
|
let mut embedder = GemmaEmbedder::load_with_device(model, worker, 1, Device::Cpu)?;
|
||||||
|
let query = "supplier contract cancellation policy";
|
||||||
|
let debug = embedder.embed_query_debug(query)?;
|
||||||
|
let diagnostics = embedder.query_diagnostics();
|
||||||
|
for name in [
|
||||||
|
"request_encode_us",
|
||||||
|
"request_write_us",
|
||||||
|
"worker_wait_and_response_decode_us",
|
||||||
|
"response_validate_us",
|
||||||
|
"worker_tokenization_us",
|
||||||
|
"worker_batch_prepare_us",
|
||||||
|
"worker_inference_us",
|
||||||
|
"worker_pool_normalize_us",
|
||||||
|
"worker_token_count",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
diagnostics.contains_key(name),
|
||||||
|
"Missing actual stage {name}: {diagnostics:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(diagnostics["worker_inference_us"] > 0);
|
||||||
|
assert!(diagnostics["worker_token_count"] > 0);
|
||||||
|
assert!(!serde_json::to_string(&diagnostics)?.contains(query));
|
||||||
|
let normal = embedder.embed_query(query)?;
|
||||||
|
assert!(embedder.query_diagnostics().is_empty());
|
||||||
|
let cosine: f64 = debug
|
||||||
|
.iter()
|
||||||
|
.zip(&normal)
|
||||||
|
.map(|(a, b)| f64::from(*a) * f64::from(*b))
|
||||||
|
.sum();
|
||||||
|
assert!(
|
||||||
|
cosine >= 0.99999,
|
||||||
|
"Debug tracing changed embeddings: {cosine}"
|
||||||
|
);
|
||||||
|
println!("CPU query diagnostics: {diagnostics:?}; debug/normal cosine {cosine:.8}");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "Requires explicitly provisioned Google model and CPU worker"]
|
#[ignore = "Requires explicitly provisioned Google model and CPU worker"]
|
||||||
fn google_cpu_real_embeddings_and_verified_tail_windows() -> Result<()> {
|
fn google_cpu_real_embeddings_and_verified_tail_windows() -> Result<()> {
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use searchi_semantic::{Device, GemmaEmbedder};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "Requires explicitly provisioned Google media model and CPU worker"]
|
||||||
|
fn google_cpu_pixels_match_digup_reference() -> Result<()> {
|
||||||
|
let model = std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").expect("Set model path");
|
||||||
|
let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").expect("Set worker path");
|
||||||
|
let mut embedder = GemmaEmbedder::load_with_device(model, worker, 1, Device::Cpu)?;
|
||||||
|
let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
|
||||||
|
let reference: serde_json::Value =
|
||||||
|
serde_json::from_str(include_str!("fixtures/digup-media-reference.json"))?;
|
||||||
|
for (name, file, budget) in [
|
||||||
|
("screen-280", "digup-screen.png", 280),
|
||||||
|
("photo-140", "digup-photo.jpg", 140),
|
||||||
|
("frame-70", "digup-photo.jpg", 70),
|
||||||
|
] {
|
||||||
|
let path = fixtures.join(file);
|
||||||
|
let vector = embedder
|
||||||
|
.embed_images_with_budget(&[path.as_path()], budget)?
|
||||||
|
.remove(0);
|
||||||
|
let cosine: f64 = vector
|
||||||
|
.iter()
|
||||||
|
.zip(reference["vectors"][name].as_array().unwrap())
|
||||||
|
.map(|(a, b)| f64::from(*a) * b.as_f64().unwrap())
|
||||||
|
.sum();
|
||||||
|
println!("Google image {name} reference cosine={cosine:.8}");
|
||||||
|
assert!(cosine >= 0.999, "Image parity below0.999: {name} {cosine}");
|
||||||
|
}
|
||||||
|
assert_eq!(embedder.info.device, "cpu");
|
||||||
|
let wav = include_bytes!("fixtures/digup-speech.wav");
|
||||||
|
let mut offset = 12;
|
||||||
|
let mut samples = Vec::new();
|
||||||
|
while offset + 8 <= wav.len() {
|
||||||
|
let length = u32::from_le_bytes(wav[offset + 4..offset + 8].try_into()?) as usize;
|
||||||
|
if &wav[offset..offset + 4] == b"data" {
|
||||||
|
for pair in wav[offset + 8..offset + 8 + length].as_chunks::<2>().0 {
|
||||||
|
samples.push(f32::from(i16::from_le_bytes(*pair)) / 32768.0);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
offset += 8 + length + (length % 2);
|
||||||
|
}
|
||||||
|
let vector = embedder.embed_audio_pcm(&samples)?;
|
||||||
|
let cosine: f64 = vector
|
||||||
|
.iter()
|
||||||
|
.zip(reference["vectors"]["speech"].as_array().unwrap())
|
||||||
|
.map(|(a, b)| f64::from(*a) * b.as_f64().unwrap())
|
||||||
|
.sum();
|
||||||
|
println!("Google audio reference cosine={cosine:.8}");
|
||||||
|
assert!(
|
||||||
|
cosine >= 0.999,
|
||||||
|
"Audio reference parity below0.999: {cosine}"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ anyhow.workspace = true
|
|||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
searchi-core = { path = "../searchi-core" }
|
searchi-core = { path = "../searchi-core" }
|
||||||
searchi-semantic = { path = "../searchi-semantic" }
|
searchi-semantic = { path = "../searchi-semantic" }
|
||||||
|
searchi-media = { path = "../searchi-media" }
|
||||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||||
sha2 = "0.10"
|
sha2 = "0.10"
|
||||||
fs2 = "0.4"
|
fs2 = "0.4"
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# Shared hybrid retrieval
|
||||||
|
|
||||||
|
`HybridSearch` combines genuine lexical and Google multimodal retrieval. The
|
||||||
|
CLI and desktop use the same service; it has no desktop-shell dependency.
|
||||||
|
|
||||||
|
`search`, `search_semantic` and `search_semantic_with_origin` preserve grouped
|
||||||
|
parent files. ANN candidates expand until enough unique eligible parents exist
|
||||||
|
or the complete bounded eligible chunk set is requested. Filename/type/folder,
|
||||||
|
size/date/duration and explicit source filters apply before ANN top-k. Media
|
||||||
|
candidate filesystem metadata is checked once per parent per query. Ninety-six
|
||||||
|
near-identical video vectors cannot hide a second eligible parent (regression
|
||||||
|
tested). HNSW remains approximate retrieval, not an exact recall guarantee.
|
||||||
|
|
||||||
|
Explicit spoken/audio wording prefers actual audio-source vectors, preserving
|
||||||
|
exact lexical filename priority. Conservative English/German token cues detect
|
||||||
|
requests such as “someone mentions”, “spoken”, “audio”, “jemand erwähnt” and
|
||||||
|
“Gespräch”. Visual/document cues veto automatic routing. This finite parser is
|
||||||
|
not language understanding; paraphrases, negation and mixed intents can remain
|
||||||
|
ambiguous. If the preferred lane has no eligible results, ordinary retrieval
|
||||||
|
is used. `SemanticOrigin::All` bypasses the hint; `Audio`, `Image`, `Video` and
|
||||||
|
`Text` select an explicit channel without fallback or reembedding.
|
||||||
|
|
||||||
|
Persisted custom sampling limits can be reopened without generating embeddings
|
||||||
|
again only when the complete model and preprocessing fingerprint matches.
|
||||||
|
The custom suffix must contain the eight valid numeric sampling limits. Model
|
||||||
|
or preprocessing changes never silently reuse incompatible vectors.
|
||||||
|
|
||||||
|
`search_measured(query,limit,options,debug)` returns `SearchResponse` with results
|
||||||
|
and monotonic total milliseconds/result count. Basic mode stores no trace.
|
||||||
|
`search_traced(...,&mut TraceCollector,parent)` shares the native request clock
|
||||||
|
so queue/lock time and actual engine stages remain comparable. Debug mode adds
|
||||||
|
bounded nested spans for lexical SQL, query embedding, worker diagnostics,
|
||||||
|
filtered eligibility and its cache, ANN read/checksum/deserialization/search
|
||||||
|
passes, parent materialization and validation, and rank fusion. Static stage
|
||||||
|
names and numerical counters exclude query/source text and paths. Worker-local
|
||||||
|
microsecond counters have a separate clock and are not aligned request spans.
|
||||||
|
|
||||||
|
Measured/source APIs retain existing result behavior except the explicit
|
||||||
|
spoken/audio preference. Debug instrumentation does not change vector policy.
|
||||||
|
Audio timestamps represent segment starts, not exact word locations. Smaller
|
||||||
|
windows alone have not demonstrated accurate spoken navigation. No-answer
|
||||||
|
calibration, music discrimination, exact spoken positions, process isolation,
|
||||||
|
unchanged-vector reuse and production platform checks remain release gates.
|
||||||
@@ -1,10 +1,15 @@
|
|||||||
//! Shared offline hybrid search using real Google embeddings and persisted,
|
//! Shared offline hybrid search using real Google embeddings and persisted,
|
||||||
//! filtered HNSW retrieval. Production-scale validation remains a release gate.
|
//! filtered HNSW retrieval. Production-scale validation remains a release gate.
|
||||||
|
mod media;
|
||||||
|
mod routing;
|
||||||
|
pub use routing::SemanticOrigin;
|
||||||
mod store;
|
mod store;
|
||||||
|
pub use searchi_media::{FfmpegTools, MediaLimits};
|
||||||
|
|
||||||
use anyhow::{ensure, Result};
|
use anyhow::{ensure, Result};
|
||||||
|
use searchi_core::metrics::{MetricsMode, SpanToken, TraceCollector};
|
||||||
use searchi_core::SearchIndex;
|
use searchi_core::SearchIndex;
|
||||||
pub use searchi_core::{SearchOptions, SearchResult};
|
pub use searchi_core::{SearchMetrics, SearchOptions, SearchResponse, SearchResult};
|
||||||
pub use searchi_semantic::WorkerInfo;
|
pub use searchi_semantic::WorkerInfo;
|
||||||
use searchi_semantic::{GemmaEmbedder, TextWindow, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT};
|
use searchi_semantic::{GemmaEmbedder, TextWindow, GEMMA_DIMENSIONS, GEMMA_FINGERPRINT};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
@@ -17,7 +22,24 @@ const BYTE_OVERLAP: usize = 512;
|
|||||||
const MAX_PASSAGE_WINDOWS: usize = 2048;
|
const MAX_PASSAGE_WINDOWS: usize = 2048;
|
||||||
const CHUNK_POLICY: &str = "byte64k-overlap512-title512-v1";
|
const CHUNK_POLICY: &str = "byte64k-overlap512-title512-v1";
|
||||||
fn generation_fingerprint(model: &str) -> String {
|
fn generation_fingerprint(model: &str) -> String {
|
||||||
format!("{model}:{CHUNK_POLICY}")
|
format!(
|
||||||
|
"{model}:{CHUNK_POLICY}:{}:image280-frame70-audio30s-overlap2-distributed240-scene120-v1",
|
||||||
|
searchi_semantic::GEMMA_MEDIA_FINGERPRINT
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reusable_policy(expected: &str, active: &str) -> bool {
|
||||||
|
if active == expected || active == format!("{expected}:text-only") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let Some(policy) = active.strip_prefix(&format!("{expected}:custom-")) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let values = policy
|
||||||
|
.split('-')
|
||||||
|
.map(str::parse::<u64>)
|
||||||
|
.collect::<std::result::Result<Vec<_>, _>>();
|
||||||
|
matches!(values,Ok(v) if v.len()==8 && v[0]>=2 && (5000..=30000).contains(&v[3]) && v[4]<v[3] && v[5]>0 && v[6]<=u32::MAX as u64)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Model-specific query/document prompts and actual tokenizer windows belong
|
/// Model-specific query/document prompts and actual tokenizer windows belong
|
||||||
@@ -28,6 +50,18 @@ pub trait EmbeddingProvider: Send {
|
|||||||
fn split_windows(&mut self, text: &str, title: &str) -> Result<Vec<TextWindow>>;
|
fn split_windows(&mut self, text: &str, title: &str) -> Result<Vec<TextWindow>>;
|
||||||
fn embed_documents(&mut self, texts: &[&str], title: &str) -> Result<Vec<Vec<f32>>>;
|
fn embed_documents(&mut self, texts: &[&str], title: &str) -> Result<Vec<Vec<f32>>>;
|
||||||
fn embed_query(&mut self, query: &str) -> Result<Vec<f32>>;
|
fn embed_query(&mut self, query: &str) -> Result<Vec<f32>>;
|
||||||
|
fn embed_query_debug(&mut self, query: &str) -> Result<Vec<f32>> {
|
||||||
|
self.embed_query(query)
|
||||||
|
}
|
||||||
|
fn query_diagnostics(&self) -> std::collections::BTreeMap<String, u64> {
|
||||||
|
Default::default()
|
||||||
|
}
|
||||||
|
fn embed_images(&mut self, _paths: &[&Path], _budget: u32) -> Result<Vec<Vec<f32>>> {
|
||||||
|
anyhow::bail!("Provider does not support real images")
|
||||||
|
}
|
||||||
|
fn embed_audio(&mut self, _pcm: &[f32]) -> Result<Vec<f32>> {
|
||||||
|
anyhow::bail!("Provider does not support real audio")
|
||||||
|
}
|
||||||
fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
|
fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
@@ -49,13 +83,34 @@ impl EmbeddingProvider for GemmaEmbedder {
|
|||||||
fn embed_query(&mut self, query: &str) -> Result<Vec<f32>> {
|
fn embed_query(&mut self, query: &str) -> Result<Vec<f32>> {
|
||||||
self.embed_query(query)
|
self.embed_query(query)
|
||||||
}
|
}
|
||||||
|
fn embed_query_debug(&mut self, query: &str) -> Result<Vec<f32>> {
|
||||||
|
GemmaEmbedder::embed_query_debug(self, query)
|
||||||
|
}
|
||||||
|
fn query_diagnostics(&self) -> std::collections::BTreeMap<String, u64> {
|
||||||
|
GemmaEmbedder::query_diagnostics(self)
|
||||||
|
}
|
||||||
|
fn embed_images(&mut self, paths: &[&Path], budget: u32) -> Result<Vec<Vec<f32>>> {
|
||||||
|
self.embed_images_with_budget(paths, budget)
|
||||||
|
}
|
||||||
|
fn embed_audio(&mut self, pcm: &[f32]) -> Result<Vec<f32>> {
|
||||||
|
self.embed_audio_pcm(pcm)
|
||||||
|
}
|
||||||
fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
|
fn worker_info(&self) -> Option<searchi_semantic::WorkerInfo> {
|
||||||
Some(self.info.clone())
|
Some(self.info.clone())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct MediaFailure {
|
||||||
|
pub path: String,
|
||||||
|
pub message: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
pub struct EmbeddingReport {
|
pub struct EmbeddingReport {
|
||||||
|
pub media_files: usize,
|
||||||
|
pub media_failures: usize,
|
||||||
|
pub failures: Vec<MediaFailure>,
|
||||||
pub passages: usize,
|
pub passages: usize,
|
||||||
pub chunks: usize,
|
pub chunks: usize,
|
||||||
pub generation: i64,
|
pub generation: i64,
|
||||||
@@ -64,6 +119,7 @@ pub struct EmbeddingReport {
|
|||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
pub struct EmbeddingStatus {
|
pub struct EmbeddingStatus {
|
||||||
|
pub failures: Vec<MediaFailure>,
|
||||||
pub generation: Option<i64>,
|
pub generation: Option<i64>,
|
||||||
pub chunks: usize,
|
pub chunks: usize,
|
||||||
pub model_fingerprint: Option<String>,
|
pub model_fingerprint: Option<String>,
|
||||||
@@ -98,9 +154,10 @@ impl HybridSearch {
|
|||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
let pending: usize =
|
let pending: usize =
|
||||||
connection.query_row("SELECT count(*) FROM passages", [], |row| row.get(0))?;
|
connection.query_row("SELECT (SELECT count(*) FROM passages)+(SELECT count(*) FROM files WHERE extension IN ('png','jpg','jpeg','webp','bmp','mp4','mkv','mov','avi','webm','m4v','mpeg','mpg','mp3','flac','wav','m4a','ogg','opus','aac','aiff','aif'))", [], |row| row.get(0))?;
|
||||||
if !exists {
|
if !exists {
|
||||||
return Ok(EmbeddingStatus {
|
return Ok(EmbeddingStatus {
|
||||||
|
failures: Vec::new(),
|
||||||
generation: None,
|
generation: None,
|
||||||
chunks: 0,
|
chunks: 0,
|
||||||
model_fingerprint: None,
|
model_fingerprint: None,
|
||||||
@@ -113,8 +170,9 @@ impl HybridSearch {
|
|||||||
[],
|
[],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
if version < 5 {
|
if version < 6 {
|
||||||
return Ok(EmbeddingStatus {
|
return Ok(EmbeddingStatus {
|
||||||
|
failures: Vec::new(),
|
||||||
generation: None,
|
generation: None,
|
||||||
chunks: 0,
|
chunks: 0,
|
||||||
model_fingerprint: None,
|
model_fingerprint: None,
|
||||||
@@ -123,7 +181,7 @@ impl HybridSearch {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
ensure!(
|
ensure!(
|
||||||
version == 5,
|
version == 6,
|
||||||
"Unsupported semantic schema version {version}"
|
"Unsupported semantic schema version {version}"
|
||||||
);
|
);
|
||||||
let (generation,fingerprint,revision,current):(Option<i64>,Option<String>,Option<i64>,i64)=connection.query_row(
|
let (generation,fingerprint,revision,current):(Option<i64>,Option<String>,Option<i64>,i64)=connection.query_row(
|
||||||
@@ -134,9 +192,19 @@ impl HybridSearch {
|
|||||||
[generation],
|
[generation],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
|
let mut failures_query=connection.prepare("SELECT path,message FROM semantic_media_errors WHERE generation=?1 ORDER BY path LIMIT 1000")?;
|
||||||
|
let failures = failures_query
|
||||||
|
.query_map([generation], |r| {
|
||||||
|
Ok(MediaFailure {
|
||||||
|
path: r.get(0)?,
|
||||||
|
message: r.get(1)?,
|
||||||
|
})
|
||||||
|
})?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
let stale = revision != Some(current)
|
let stale = revision != Some(current)
|
||||||
|| fingerprint.as_deref() != Some(generation_fingerprint(GEMMA_FINGERPRINT).as_str());
|
|| fingerprint.as_deref() != Some(generation_fingerprint(GEMMA_FINGERPRINT).as_str());
|
||||||
Ok(EmbeddingStatus {
|
Ok(EmbeddingStatus {
|
||||||
|
failures,
|
||||||
generation,
|
generation,
|
||||||
chunks,
|
chunks,
|
||||||
model_fingerprint: fingerprint,
|
model_fingerprint: fingerprint,
|
||||||
@@ -162,7 +230,12 @@ impl HybridSearch {
|
|||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let lexical = SearchIndex::open(&db)?;
|
let lexical = SearchIndex::open(&db)?;
|
||||||
let store = Store::open(db.as_ref(), embedder.dimensions())?;
|
let store = Store::open(db.as_ref(), embedder.dimensions())?;
|
||||||
let generation_fingerprint = generation_fingerprint(embedder.fingerprint());
|
let expected = generation_fingerprint(embedder.fingerprint());
|
||||||
|
let active = store.active_fingerprint()?;
|
||||||
|
let generation_fingerprint = match active {
|
||||||
|
Some(active) if reusable_policy(&expected, &active) => active,
|
||||||
|
_ => expected,
|
||||||
|
};
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
lexical,
|
lexical,
|
||||||
embedder,
|
embedder,
|
||||||
@@ -189,6 +262,11 @@ impl HybridSearch {
|
|||||||
/// storage errors leave the previous generation active. A crashed build is
|
/// storage errors leave the previous generation active. A crashed build is
|
||||||
/// reclaimed on the next explicit rebuild under an OS advisory lock.
|
/// reclaimed on the next explicit rebuild under an OS advisory lock.
|
||||||
pub fn index_embeddings(&mut self) -> Result<EmbeddingReport> {
|
pub fn index_embeddings(&mut self) -> Result<EmbeddingReport> {
|
||||||
|
let previous_fingerprint = self.generation_fingerprint.clone();
|
||||||
|
self.generation_fingerprint = format!(
|
||||||
|
"{}:text-only",
|
||||||
|
generation_fingerprint(self.embedder.fingerprint())
|
||||||
|
);
|
||||||
let _lock = self.store.build_lock()?;
|
let _lock = self.store.build_lock()?;
|
||||||
let generation = self.store.begin(&self.generation_fingerprint)?;
|
let generation = self.store.begin(&self.generation_fingerprint)?;
|
||||||
let built = self.build(generation);
|
let built = self.build(generation);
|
||||||
@@ -199,6 +277,7 @@ impl HybridSearch {
|
|||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
self.store.abandon(generation)?;
|
self.store.abandon(generation)?;
|
||||||
|
self.generation_fingerprint = previous_fingerprint;
|
||||||
Err(error)
|
Err(error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -206,6 +285,9 @@ impl HybridSearch {
|
|||||||
|
|
||||||
fn build(&mut self, generation: i64) -> Result<EmbeddingReport> {
|
fn build(&mut self, generation: i64) -> Result<EmbeddingReport> {
|
||||||
let mut report = EmbeddingReport {
|
let mut report = EmbeddingReport {
|
||||||
|
media_files: 0,
|
||||||
|
media_failures: 0,
|
||||||
|
failures: Vec::new(),
|
||||||
passages: 0,
|
passages: 0,
|
||||||
chunks: 0,
|
chunks: 0,
|
||||||
generation,
|
generation,
|
||||||
@@ -253,21 +335,81 @@ impl HybridSearch {
|
|||||||
query: &str,
|
query: &str,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
options: &SearchOptions,
|
options: &SearchOptions,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Basic);
|
||||||
|
self.search_traced(query, limit, options, &mut trace, None)
|
||||||
|
}
|
||||||
|
pub fn search_measured(
|
||||||
|
&mut self,
|
||||||
|
query: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
debug: bool,
|
||||||
|
) -> Result<SearchResponse<SearchResult>> {
|
||||||
|
let mut trace = TraceCollector::new(if debug {
|
||||||
|
MetricsMode::Debug
|
||||||
|
} else {
|
||||||
|
MetricsMode::Basic
|
||||||
|
});
|
||||||
|
let results = self.search_traced(query, limit, options, &mut trace, None)?;
|
||||||
|
let metrics = trace.finish(results.len());
|
||||||
|
Ok(SearchResponse { results, metrics })
|
||||||
|
}
|
||||||
|
pub fn search_traced(
|
||||||
|
&mut self,
|
||||||
|
query: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
trace: &mut TraceCollector,
|
||||||
|
parent: Option<SpanToken>,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
trace.timed("hybrid.search", parent, |trace, parent| {
|
||||||
|
self.search_inner(query, limit, options, trace, parent)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
fn search_inner(
|
||||||
|
&mut self,
|
||||||
|
query: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
trace: &mut TraceCollector,
|
||||||
|
parent: Option<SpanToken>,
|
||||||
) -> Result<Vec<SearchResult>> {
|
) -> Result<Vec<SearchResult>> {
|
||||||
let candidates = limit.saturating_mul(4).clamp(20, 1000);
|
let candidates = limit.saturating_mul(4).clamp(20, 1000);
|
||||||
// Core validates filters and query budgets even for empty requests.
|
// Core validates filters and query budgets even for empty requests.
|
||||||
let mut lexical = self
|
let mut lexical = self
|
||||||
.lexical
|
.lexical
|
||||||
.search_with_options(query, candidates, options)?;
|
.search_with_options_traced(query, candidates, options, trace, parent)?;
|
||||||
if query.trim().is_empty() || limit == 0 {
|
if query.trim().is_empty() || limit == 0 {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
promote_identifiers(query.trim(), &mut lexical);
|
trace.timed("hybrid.identifier_priority", parent, |_, _| {
|
||||||
let vector = self.embedder.embed_query(query.trim())?;
|
promote_identifiers(query.trim(), &mut lexical)
|
||||||
|
});
|
||||||
|
let embedding_span = trace.begin("semantic.query_embedding", parent);
|
||||||
|
let vector = if trace.is_debug() {
|
||||||
|
self.embedder.embed_query_debug(query.trim())
|
||||||
|
} else {
|
||||||
|
self.embedder.embed_query(query.trim())
|
||||||
|
};
|
||||||
|
trace.counter(embedding_span, "failed", u64::from(vector.is_err()));
|
||||||
|
trace.end(
|
||||||
|
embedding_span,
|
||||||
|
if trace.is_debug() {
|
||||||
|
self.embedder.query_diagnostics()
|
||||||
|
} else {
|
||||||
|
Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let vector = vector?;
|
||||||
let semantic =
|
let semantic =
|
||||||
self.store
|
self.routed_semantic_traced(query, &vector, candidates, options, trace, parent)?;
|
||||||
.search(&vector, &self.generation_fingerprint, candidates, options)?;
|
let results = trace.timed("hybrid.rank_fusion", parent, |trace, span| {
|
||||||
Ok(fuse(lexical, semantic, limit.min(1000)))
|
trace.counter(span, "lexical_files", lexical.len() as u64);
|
||||||
|
trace.counter(span, "semantic_files", semantic.len() as u64);
|
||||||
|
fuse(lexical, semantic, limit.min(1000))
|
||||||
|
});
|
||||||
|
Ok(results)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Semantic lane for reproducible relevance evaluation and diagnostics.
|
/// Semantic lane for reproducible relevance evaluation and diagnostics.
|
||||||
@@ -282,12 +424,7 @@ impl HybridSearch {
|
|||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
let vector = self.embedder.embed_query(query.trim())?;
|
let vector = self.embedder.embed_query(query.trim())?;
|
||||||
self.store.search(
|
self.routed_semantic(query, &vector, limit.min(1000), options)
|
||||||
&vector,
|
|
||||||
&self.generation_fingerprint,
|
|
||||||
limit.min(1000),
|
|
||||||
options,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -417,6 +554,58 @@ fn fuse(
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
#[test]
|
||||||
|
fn failed_query_inference_retains_diagnostics_in_shared_trace() -> Result<()> {
|
||||||
|
// Failure-only provider: no fabricated embeddings are generated.
|
||||||
|
struct FailingProvider;
|
||||||
|
impl EmbeddingProvider for FailingProvider {
|
||||||
|
fn fingerprint(&self) -> &str {
|
||||||
|
"failure-injection"
|
||||||
|
}
|
||||||
|
fn dimensions(&self) -> usize {
|
||||||
|
768
|
||||||
|
}
|
||||||
|
fn split_windows(&mut self, _: &str, _: &str) -> Result<Vec<TextWindow>> {
|
||||||
|
anyhow::bail!("unused")
|
||||||
|
}
|
||||||
|
fn embed_documents(&mut self, _: &[&str], _: &str) -> Result<Vec<Vec<f32>>> {
|
||||||
|
anyhow::bail!("unused")
|
||||||
|
}
|
||||||
|
fn embed_query(&mut self, _: &str) -> Result<Vec<f32>> {
|
||||||
|
anyhow::bail!("injected worker failure")
|
||||||
|
}
|
||||||
|
fn query_diagnostics(&self) -> std::collections::BTreeMap<String, u64> {
|
||||||
|
std::collections::BTreeMap::from([("request_write_us".into(), 7)])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let directory = tempfile::tempdir()?;
|
||||||
|
let db = directory.path().join("index.db");
|
||||||
|
let mut service = HybridSearch::with_provider(&db, Box::new(FailingProvider))?;
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
assert!(service
|
||||||
|
.search_traced(
|
||||||
|
"ordinary query",
|
||||||
|
5,
|
||||||
|
&SearchOptions::default(),
|
||||||
|
&mut trace,
|
||||||
|
None
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
let metrics = trace.finish(0);
|
||||||
|
let spans = metrics.trace.unwrap();
|
||||||
|
let embedding = spans
|
||||||
|
.iter()
|
||||||
|
.find(|s| s.name == "semantic.query_embedding")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(embedding.counters["failed"], 1);
|
||||||
|
assert_eq!(embedding.counters["request_write_us"], 7);
|
||||||
|
let parent = &spans[embedding.parent.unwrap()];
|
||||||
|
assert!(
|
||||||
|
embedding.start_ms + embedding.duration_ms
|
||||||
|
<= parent.start_ms + parent.duration_ms + 1e-6
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
fn result(path: &str, kind: &str) -> SearchResult {
|
fn result(path: &str, kind: &str) -> SearchResult {
|
||||||
SearchResult {
|
SearchResult {
|
||||||
path: path.into(),
|
path: path.into(),
|
||||||
|
|||||||
@@ -0,0 +1,232 @@
|
|||||||
|
use super::*;
|
||||||
|
use searchi_media::*;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct InferenceFailure(anyhow::Error);
|
||||||
|
impl std::fmt::Display for InferenceFailure {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(f, "Media inference failed: {}", self.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl std::error::Error for InferenceFailure {}
|
||||||
|
|
||||||
|
impl HybridSearch {
|
||||||
|
/// Publish text and actual image/frame/audio vectors in one atomic generation.
|
||||||
|
pub fn index_embeddings_with_media(
|
||||||
|
&mut self,
|
||||||
|
tools: &FfmpegTools,
|
||||||
|
limits: &MediaLimits,
|
||||||
|
) -> Result<EmbeddingReport> {
|
||||||
|
let previous_fingerprint = self.generation_fingerprint.clone();
|
||||||
|
self.generation_fingerprint = generation_fingerprint(self.embedder.fingerprint());
|
||||||
|
let defaults = MediaLimits::default();
|
||||||
|
if limits.max_frames != defaults.max_frames
|
||||||
|
|| limits.frame_interval_ms != defaults.frame_interval_ms
|
||||||
|
|| limits.max_scene_frames != defaults.max_scene_frames
|
||||||
|
|| limits.audio_window_ms != defaults.audio_window_ms
|
||||||
|
|| limits.audio_overlap_ms != defaults.audio_overlap_ms
|
||||||
|
|| limits.max_audio_windows != defaults.max_audio_windows
|
||||||
|
|| limits.max_duration_ms != defaults.max_duration_ms
|
||||||
|
|| limits.max_file_bytes != defaults.max_file_bytes
|
||||||
|
{
|
||||||
|
self.generation_fingerprint.push_str(&format!(
|
||||||
|
":custom-{}-{}-{}-{}-{}-{}-{}-{}",
|
||||||
|
limits.max_frames,
|
||||||
|
limits.frame_interval_ms,
|
||||||
|
limits.max_scene_frames,
|
||||||
|
limits.audio_window_ms,
|
||||||
|
limits.audio_overlap_ms,
|
||||||
|
limits.max_audio_windows,
|
||||||
|
limits.max_duration_ms,
|
||||||
|
limits.max_file_bytes
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let _lock = self.store.build_lock()?;
|
||||||
|
let generation = self.store.begin(&self.generation_fingerprint)?;
|
||||||
|
let built = (|| {
|
||||||
|
let mut report = self.build(generation)?;
|
||||||
|
let mut after = 0;
|
||||||
|
let mut media_count = 0usize;
|
||||||
|
loop {
|
||||||
|
let files = self.store.media_batch(after)?;
|
||||||
|
if files.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
for (id, path, _, _) in files {
|
||||||
|
after = id;
|
||||||
|
let path = Path::new(&path);
|
||||||
|
let Some(kind) = classify(path) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
media_count += 1;
|
||||||
|
ensure!(
|
||||||
|
media_count <= 10000,
|
||||||
|
"Media generation exceeds 10000-file resource budget"
|
||||||
|
);
|
||||||
|
let before_chunks = report.chunks;
|
||||||
|
match self.build_media_file(
|
||||||
|
generation,
|
||||||
|
id,
|
||||||
|
path,
|
||||||
|
kind,
|
||||||
|
tools,
|
||||||
|
limits,
|
||||||
|
&mut report,
|
||||||
|
) {
|
||||||
|
Ok(()) => report.media_files += 1,
|
||||||
|
Err(error) => {
|
||||||
|
if error.downcast_ref::<InferenceFailure>().is_some() {
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
self.store.media_failure(
|
||||||
|
generation,
|
||||||
|
&path.to_string_lossy(),
|
||||||
|
&format!("{error:#}"),
|
||||||
|
)?;
|
||||||
|
report.chunks = before_chunks;
|
||||||
|
if report.failures.len() < 1000 {
|
||||||
|
report.failures.push(MediaFailure {
|
||||||
|
path: path.to_string_lossy().into_owned(),
|
||||||
|
message: format!("{error:#}").chars().take(2048).collect(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
report.media_failures += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(report)
|
||||||
|
})();
|
||||||
|
match built {
|
||||||
|
Ok(report) => {
|
||||||
|
self.store.activate(generation)?;
|
||||||
|
Ok(report)
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
self.store.abandon(generation)?;
|
||||||
|
self.generation_fingerprint = previous_fingerprint;
|
||||||
|
Err(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn build_media_file(
|
||||||
|
&mut self,
|
||||||
|
generation: i64,
|
||||||
|
id: i64,
|
||||||
|
path: &Path,
|
||||||
|
kind: MediaKind,
|
||||||
|
tools: &FfmpegTools,
|
||||||
|
limits: &MediaLimits,
|
||||||
|
report: &mut EmbeddingReport,
|
||||||
|
) -> Result<()> {
|
||||||
|
let identity = MediaIdentity::read(
|
||||||
|
path,
|
||||||
|
if kind == MediaKind::Image {
|
||||||
|
64 * 1024 * 1024
|
||||||
|
} else {
|
||||||
|
limits.max_file_bytes
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
let mut ordinal = 0;
|
||||||
|
if kind == MediaKind::Image {
|
||||||
|
validate_image(path)?;
|
||||||
|
ensure!(
|
||||||
|
report.chunks < MAX_SCAN_CHUNKS,
|
||||||
|
"Media exceeds generation chunk budget"
|
||||||
|
);
|
||||||
|
let vectors = self
|
||||||
|
.embedder
|
||||||
|
.embed_images(&[path], 280)
|
||||||
|
.map_err(|e| anyhow::Error::new(InferenceFailure(e)))?;
|
||||||
|
ensure!(vectors.len() == 1, "Image embedding count mismatch");
|
||||||
|
self.store
|
||||||
|
.insert_media(generation, id, ordinal, &identity, "image", 0, &vectors[0])?;
|
||||||
|
report.chunks += 1;
|
||||||
|
} else {
|
||||||
|
let metadata = probe(tools, path, limits)?;
|
||||||
|
let tags = metadata
|
||||||
|
.tags
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| format!("{k}: {v}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
self.lexical.set_media_metadata(
|
||||||
|
&path.to_string_lossy(),
|
||||||
|
&tags,
|
||||||
|
metadata.duration_ms,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
)?;
|
||||||
|
if metadata.has_video && kind == MediaKind::Video {
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
let mut scenes = 0;
|
||||||
|
for time in frame_times(&metadata, limits)? {
|
||||||
|
let frames = decode_frames(
|
||||||
|
tools,
|
||||||
|
path,
|
||||||
|
&metadata,
|
||||||
|
time,
|
||||||
|
scenes < limits.max_scene_frames,
|
||||||
|
limits,
|
||||||
|
)?;
|
||||||
|
for (i, frame) in frames.frames.iter().enumerate() {
|
||||||
|
if !seen.insert(frame.timestamp_ms) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if i > 0 {
|
||||||
|
scenes += 1;
|
||||||
|
}
|
||||||
|
ensure!(
|
||||||
|
report.chunks < MAX_SCAN_CHUNKS,
|
||||||
|
"Media exceeds generation chunk budget"
|
||||||
|
);
|
||||||
|
let vectors = self
|
||||||
|
.embedder
|
||||||
|
.embed_images(&[frame.path.as_path()], 70)
|
||||||
|
.map_err(|e| anyhow::Error::new(InferenceFailure(e)))?;
|
||||||
|
ensure!(vectors.len() == 1, "Frame embedding count mismatch");
|
||||||
|
self.store.insert_media(
|
||||||
|
generation,
|
||||||
|
id,
|
||||||
|
ordinal,
|
||||||
|
&identity,
|
||||||
|
"video",
|
||||||
|
frame.timestamp_ms,
|
||||||
|
&vectors[0],
|
||||||
|
)?;
|
||||||
|
ordinal += 1;
|
||||||
|
report.chunks += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if metadata.has_audio {
|
||||||
|
for window in audio_windows(&metadata, limits)? {
|
||||||
|
ensure!(
|
||||||
|
report.chunks < MAX_SCAN_CHUNKS,
|
||||||
|
"Audio exceeds generation chunk budget"
|
||||||
|
);
|
||||||
|
let pcm = decode_audio(tools, path, window, limits)?;
|
||||||
|
let vector = self
|
||||||
|
.embedder
|
||||||
|
.embed_audio(&pcm)
|
||||||
|
.map_err(|e| anyhow::Error::new(InferenceFailure(e)))?;
|
||||||
|
self.store.insert_media(
|
||||||
|
generation,
|
||||||
|
id,
|
||||||
|
ordinal,
|
||||||
|
&identity,
|
||||||
|
"audio",
|
||||||
|
window.start_ms,
|
||||||
|
&vector,
|
||||||
|
)?;
|
||||||
|
ordinal += 1;
|
||||||
|
report.chunks += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
identity.ensure_unchanged()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Explicit semantic source channel, independent of filename/type filters.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum SemanticOrigin {
|
||||||
|
All,
|
||||||
|
Text,
|
||||||
|
Image,
|
||||||
|
Video,
|
||||||
|
Audio,
|
||||||
|
}
|
||||||
|
impl SemanticOrigin {
|
||||||
|
fn source_type(self) -> Option<&'static str> {
|
||||||
|
match self {
|
||||||
|
Self::All => None,
|
||||||
|
Self::Text => Some("text"),
|
||||||
|
Self::Image => Some("image"),
|
||||||
|
Self::Video => Some("video"),
|
||||||
|
Self::Audio => Some("audio"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A conservative intent hint, not a language understanding model. Explicit
|
||||||
|
/// document/visual cues veto it; ordinary topic and instrument queries do not
|
||||||
|
/// silently restrict modality. Word boundaries avoid filename substrings.
|
||||||
|
fn preferred_origin(query: &str) -> Option<SemanticOrigin> {
|
||||||
|
if query.split_whitespace().count() == 1 && Path::new(query).extension().is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let lower = query.to_lowercase();
|
||||||
|
let words: Vec<_> = lower
|
||||||
|
.split(|c: char| !c.is_alphanumeric())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.collect();
|
||||||
|
let has = |values: &[&str]| words.iter().any(|word| values.contains(word));
|
||||||
|
if has(&[
|
||||||
|
"image",
|
||||||
|
"photo",
|
||||||
|
"picture",
|
||||||
|
"screenshot",
|
||||||
|
"diagram",
|
||||||
|
"document",
|
||||||
|
"pdf",
|
||||||
|
"text",
|
||||||
|
"paper",
|
||||||
|
"bild",
|
||||||
|
"foto",
|
||||||
|
"screenshot",
|
||||||
|
"dokument",
|
||||||
|
"textdatei",
|
||||||
|
]) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if has(&[
|
||||||
|
"audio",
|
||||||
|
"spoken",
|
||||||
|
"speech",
|
||||||
|
"recording",
|
||||||
|
"conversation",
|
||||||
|
"interview",
|
||||||
|
"podcast",
|
||||||
|
"sound",
|
||||||
|
"music",
|
||||||
|
"gesprochen",
|
||||||
|
"gespräch",
|
||||||
|
"gespraech",
|
||||||
|
"geräusch",
|
||||||
|
"geraeusch",
|
||||||
|
"musik",
|
||||||
|
"hörbar",
|
||||||
|
"hörbarer",
|
||||||
|
"hörbare",
|
||||||
|
"hoerbar",
|
||||||
|
"hoerbarer",
|
||||||
|
"hoerbare",
|
||||||
|
"tonaufnahme",
|
||||||
|
]) {
|
||||||
|
return Some(SemanticOrigin::Audio);
|
||||||
|
}
|
||||||
|
if (has(&["someone", "speaker"])
|
||||||
|
&& has(&[
|
||||||
|
"mentions",
|
||||||
|
"mentioned",
|
||||||
|
"says",
|
||||||
|
"said",
|
||||||
|
"speaks",
|
||||||
|
"speaking",
|
||||||
|
]))
|
||||||
|
|| (has(&["jemand", "sprecher", "sprecherin"])
|
||||||
|
&& has(&["erwähnt", "erwaehnt", "sagt", "gesagt", "spricht"]))
|
||||||
|
{
|
||||||
|
return Some(SemanticOrigin::Audio);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HybridSearch {
|
||||||
|
pub(super) fn routed_semantic(
|
||||||
|
&mut self,
|
||||||
|
query: &str,
|
||||||
|
vector: &[f32],
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Basic);
|
||||||
|
self.routed_semantic_traced(query, vector, limit, options, &mut trace, None)
|
||||||
|
}
|
||||||
|
pub(super) fn routed_semantic_traced(
|
||||||
|
&mut self,
|
||||||
|
query: &str,
|
||||||
|
vector: &[f32],
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
trace: &mut TraceCollector,
|
||||||
|
parent: Option<SpanToken>,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
trace.timed("semantic.routing", parent, |trace, parent| {
|
||||||
|
if let Some(origin) = preferred_origin(query) {
|
||||||
|
trace.counter(parent, "audio_preference", 1);
|
||||||
|
let preferred = self.store.search_origin_traced(
|
||||||
|
vector,
|
||||||
|
&self.generation_fingerprint,
|
||||||
|
limit,
|
||||||
|
options,
|
||||||
|
origin.source_type(),
|
||||||
|
trace,
|
||||||
|
parent,
|
||||||
|
)?;
|
||||||
|
if !preferred.is_empty() {
|
||||||
|
return Ok(preferred);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
trace.counter(parent, "general_lane", 1);
|
||||||
|
self.store.search_origin_traced(
|
||||||
|
vector,
|
||||||
|
&self.generation_fingerprint,
|
||||||
|
limit,
|
||||||
|
options,
|
||||||
|
None,
|
||||||
|
trace,
|
||||||
|
parent,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Query one actual source channel, or All to bypass automatic spoken/audio
|
||||||
|
/// preference. This never embeds files again or changes an active policy.
|
||||||
|
pub fn search_semantic_with_origin(
|
||||||
|
&mut self,
|
||||||
|
query: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
origin: SemanticOrigin,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
let _ = self.lexical.search_with_options(query, 0, options)?;
|
||||||
|
if query.trim().is_empty() || limit == 0 {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
let vector = self.embedder.embed_query(query.trim())?;
|
||||||
|
self.store.search_origin(
|
||||||
|
&vector,
|
||||||
|
&self.generation_fingerprint,
|
||||||
|
limit.min(1000),
|
||||||
|
options,
|
||||||
|
origin.source_type(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
#[test]
|
||||||
|
fn persisted_custom_limits_require_identical_model_and_preprocessing() {
|
||||||
|
let expected = "pinned-model:pinned-media:preprocessing-v1";
|
||||||
|
assert!(reusable_policy(
|
||||||
|
expected,
|
||||||
|
&format!("{expected}:custom-240-2000-120-10000-2000-1800-14400000-17179869184")
|
||||||
|
));
|
||||||
|
for active in ["other-model:pinned-media:preprocessing-v1:custom-240-2000-120-10000-2000-1800-14400000-17179869184","pinned-model:pinned-media:preprocessing-v2:custom-240-2000-120-10000-2000-1800-14400000-17179869184","pinned-model:pinned-media:preprocessing-v1:custom-unknown"] {assert!(!reusable_policy(expected,active));}
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn explicit_spoken_intent_is_conservative_and_bilingual() {
|
||||||
|
for query in [
|
||||||
|
"where someone mentions a contract",
|
||||||
|
"a speaker says the date",
|
||||||
|
"spoken discussion of ecology",
|
||||||
|
"audio of a waterfall",
|
||||||
|
"jemand erwähnt das Budget",
|
||||||
|
"Wo spricht jemand über Kündigung?",
|
||||||
|
"Gespräch über die neue Wohnung",
|
||||||
|
"hörbarer Regen",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
preferred_origin(query),
|
||||||
|
Some(SemanticOrigin::Audio),
|
||||||
|
"{query}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for query in [
|
||||||
|
"contract deadline",
|
||||||
|
"violin solo",
|
||||||
|
"photo of someone speaking",
|
||||||
|
"Bild von einem Sprecher",
|
||||||
|
"a document that someone mentioned",
|
||||||
|
"speech.png",
|
||||||
|
"a screenshot of an audio waveform",
|
||||||
|
] {
|
||||||
|
assert_eq!(preferred_origin(query), None, "{query}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ use std::{
|
|||||||
use anyhow::{ensure, Context, Result};
|
use anyhow::{ensure, Context, Result};
|
||||||
use fs2::FileExt;
|
use fs2::FileExt;
|
||||||
use rusqlite::{params, Connection, OptionalExtension};
|
use rusqlite::{params, Connection, OptionalExtension};
|
||||||
|
use searchi_core::metrics::{MetricsMode, SpanToken, TraceCollector};
|
||||||
use searchi_core::{IndexedPassage, SearchOptions, SearchResult, SourceLocation};
|
use searchi_core::{IndexedPassage, SearchOptions, SearchResult, SourceLocation};
|
||||||
use searchi_semantic::TextWindow;
|
use searchi_semantic::TextWindow;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
@@ -24,6 +25,9 @@ struct FilterKey {
|
|||||||
max_size: Option<i64>,
|
max_size: Option<i64>,
|
||||||
modified_after: Option<i64>,
|
modified_after: Option<i64>,
|
||||||
modified_before: Option<i64>,
|
modified_before: Option<i64>,
|
||||||
|
min_duration: Option<u32>,
|
||||||
|
max_duration: Option<u32>,
|
||||||
|
source_type: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct EligibilityCache {
|
struct EligibilityCache {
|
||||||
@@ -57,8 +61,13 @@ pub(crate) fn content_hash(text: &str) -> String {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
thread_local! { static HASH_CALLS:std::cell::Cell<usize>=const {std::cell::Cell::new(0)}; }
|
thread_local! { static HASH_CALLS:std::cell::Cell<usize>=const {std::cell::Cell::new(0)}; }
|
||||||
|
#[cfg(test)]
|
||||||
|
thread_local! { static MEDIA_STAT_CALLS:std::cell::Cell<usize>=const {std::cell::Cell::new(0)}; }
|
||||||
|
|
||||||
impl Store {
|
impl Store {
|
||||||
|
pub(crate) fn active_fingerprint(&self) -> Result<Option<String>> {
|
||||||
|
Ok(self.connection.query_row("SELECT g.fingerprint FROM semantic_meta m JOIN semantic_generations g ON g.id=m.active_generation WHERE m.singleton=1",[],|r|r.get(0)).optional()?)
|
||||||
|
}
|
||||||
pub(crate) fn open(db: &Path, dimensions: usize) -> Result<Self> {
|
pub(crate) fn open(db: &Path, dimensions: usize) -> Result<Self> {
|
||||||
ensure!(
|
ensure!(
|
||||||
(1..=4096).contains(&dimensions),
|
(1..=4096).contains(&dimensions),
|
||||||
@@ -85,7 +94,7 @@ impl Store {
|
|||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
ensure!(
|
ensure!(
|
||||||
(1..=5).contains(&version),
|
(1..=6).contains(&version),
|
||||||
"Unsupported semantic schema version {version}"
|
"Unsupported semantic schema version {version}"
|
||||||
);
|
);
|
||||||
initialization.execute_batch("
|
initialization.execute_batch("
|
||||||
@@ -130,8 +139,25 @@ impl Store {
|
|||||||
UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
|
UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
|
||||||
INSERT INTO semantic_dirty_paths(path,revision) VALUES(old.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
|
INSERT INTO semantic_dirty_paths(path,revision) VALUES(old.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
|
||||||
ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;
|
ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;
|
||||||
UPDATE semantic_meta SET schema_version=5 WHERE singleton=1;",
|
UPDATE semantic_meta SET schema_version=6 WHERE singleton=1;",
|
||||||
)?;
|
)?;
|
||||||
|
for (column, declaration) in [
|
||||||
|
("source_type", "TEXT NOT NULL DEFAULT 'text'"),
|
||||||
|
("file_size", "INTEGER"),
|
||||||
|
("file_mtime", "INTEGER"),
|
||||||
|
] {
|
||||||
|
let exists: bool = initialization.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('semantic_chunks') WHERE name=?1)",
|
||||||
|
[column],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?;
|
||||||
|
if !exists {
|
||||||
|
initialization.execute_batch(&format!(
|
||||||
|
"ALTER TABLE semantic_chunks ADD COLUMN {column} {declaration}"
|
||||||
|
))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?;
|
||||||
initialization.commit()?;
|
initialization.commit()?;
|
||||||
let canonical = db.canonicalize()?;
|
let canonical = db.canonicalize()?;
|
||||||
let lock_path = PathBuf::from(format!("{}.semantic.lock", canonical.to_string_lossy()));
|
let lock_path = PathBuf::from(format!("{}.semantic.lock", canonical.to_string_lossy()));
|
||||||
@@ -170,6 +196,67 @@ impl Store {
|
|||||||
Ok(id)
|
Ok(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn media_batch(&self, after: i64) -> Result<Vec<(i64, String, i64, i64)>> {
|
||||||
|
let mut q = self.connection.prepare(
|
||||||
|
"SELECT rowid,path,size,mtime FROM files WHERE rowid>?1 ORDER BY rowid LIMIT 64",
|
||||||
|
)?;
|
||||||
|
let rows = q
|
||||||
|
.query_map([after], |r| {
|
||||||
|
Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))
|
||||||
|
})?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
Ok(rows)
|
||||||
|
}
|
||||||
|
pub(crate) fn media_failure(&self, generation: i64, path: &str, message: &str) -> Result<()> {
|
||||||
|
self.connection.execute(
|
||||||
|
"DELETE FROM semantic_chunks WHERE generation=?1 AND path=?2 AND source_type!='text'",
|
||||||
|
params![generation, path],
|
||||||
|
)?;
|
||||||
|
self.connection.execute("INSERT OR REPLACE INTO semantic_media_errors(generation,path,message) VALUES(?1,?2,?3)",params![generation,path,message.chars().take(2048).collect::<String>()])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub(crate) fn insert_media(
|
||||||
|
&self,
|
||||||
|
generation: i64,
|
||||||
|
file_id: i64,
|
||||||
|
ordinal: usize,
|
||||||
|
identity: &searchi_media::MediaIdentity,
|
||||||
|
source_type: &str,
|
||||||
|
timestamp: u32,
|
||||||
|
vector: &[f32],
|
||||||
|
) -> Result<()> {
|
||||||
|
ensure!(
|
||||||
|
vector.len() == self.dimensions && vector.iter().all(|v| v.is_finite()),
|
||||||
|
"Invalid media embedding"
|
||||||
|
);
|
||||||
|
let norm = vector.iter().map(|v| v * v).sum::<f32>().sqrt();
|
||||||
|
ensure!(
|
||||||
|
(norm - 1.).abs() < 1e-3,
|
||||||
|
"Media embeddings must be normalized"
|
||||||
|
);
|
||||||
|
let path = identity.path.to_string_lossy();
|
||||||
|
let valid:bool=self.connection.query_row("SELECT EXISTS(SELECT 1 FROM files WHERE rowid=?1 AND path=?2 AND size=?3 AND mtime=?4)",params![file_id,path,identity.bytes as i64,identity.modified_ns],|r|r.get(0))?;
|
||||||
|
ensure!(valid, "Media file changed since filesystem indexing");
|
||||||
|
let kind = if source_type == "image" {
|
||||||
|
"image"
|
||||||
|
} else {
|
||||||
|
"timestamp_ms"
|
||||||
|
};
|
||||||
|
let snippet = if source_type == "image" {
|
||||||
|
"Image content".to_string()
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"{} content at {:.3} seconds",
|
||||||
|
source_type,
|
||||||
|
f64::from(timestamp) / 1000.
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let bytes: Vec<u8> = vector.iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||||
|
self.connection.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime) VALUES(?1,?2,?3,?4,?5,?6,?7,0,0,?8,?9,?10,?11,?12)",params![generation,-file_id,ordinal as i64,path,kind,timestamp,identity.sha256,snippet,bytes,source_type,identity.bytes as i64,identity.modified_ns])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn insert(
|
pub(crate) fn insert(
|
||||||
&mut self,
|
&mut self,
|
||||||
generation: i64,
|
generation: i64,
|
||||||
@@ -312,6 +399,61 @@ impl Store {
|
|||||||
fingerprint: &str,
|
fingerprint: &str,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
options: &SearchOptions,
|
options: &SearchOptions,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
self.search_origin(query, fingerprint, limit, options, None)
|
||||||
|
}
|
||||||
|
pub(crate) fn search_origin(
|
||||||
|
&mut self,
|
||||||
|
query: &[f32],
|
||||||
|
fingerprint: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
source_type: Option<&str>,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Basic);
|
||||||
|
self.search_origin_traced(
|
||||||
|
query,
|
||||||
|
fingerprint,
|
||||||
|
limit,
|
||||||
|
options,
|
||||||
|
source_type,
|
||||||
|
&mut trace,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub(crate) fn search_origin_traced(
|
||||||
|
&mut self,
|
||||||
|
query: &[f32],
|
||||||
|
fingerprint: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
source_type: Option<&str>,
|
||||||
|
trace: &mut TraceCollector,
|
||||||
|
parent: Option<SpanToken>,
|
||||||
|
) -> Result<Vec<SearchResult>> {
|
||||||
|
trace.timed("semantic.retrieval", parent, |trace, parent| {
|
||||||
|
self.search_inner(
|
||||||
|
query,
|
||||||
|
fingerprint,
|
||||||
|
limit,
|
||||||
|
options,
|
||||||
|
source_type,
|
||||||
|
trace,
|
||||||
|
parent,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn search_inner(
|
||||||
|
&mut self,
|
||||||
|
query: &[f32],
|
||||||
|
fingerprint: &str,
|
||||||
|
limit: usize,
|
||||||
|
options: &SearchOptions,
|
||||||
|
source_type: Option<&str>,
|
||||||
|
trace: &mut TraceCollector,
|
||||||
|
parent: Option<SpanToken>,
|
||||||
) -> Result<Vec<SearchResult>> {
|
) -> Result<Vec<SearchResult>> {
|
||||||
ensure!(
|
ensure!(
|
||||||
query.len() == self.dimensions && query.iter().all(|v| v.is_finite()),
|
query.len() == self.dimensions && query.iter().all(|v| v.is_finite()),
|
||||||
@@ -321,17 +463,30 @@ impl Store {
|
|||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
// A consistent read snapshot covers eligibility and materialization.
|
// A consistent read snapshot covers eligibility and materialization.
|
||||||
|
let generation_span = trace.begin("semantic.generation_snapshot", parent);
|
||||||
let snapshot = self.connection.unchecked_transaction()?;
|
let snapshot = self.connection.unchecked_transaction()?;
|
||||||
let active:Option<i64>=snapshot.query_row("SELECT g.id FROM semantic_generations g JOIN semantic_meta m ON m.active_generation=g.id WHERE g.fingerprint=?1 AND g.dimensions=?2",
|
let active:Option<i64>=snapshot.query_row("SELECT g.id FROM semantic_generations g JOIN semantic_meta m ON m.active_generation=g.id WHERE g.fingerprint=?1 AND g.dimensions=?2",
|
||||||
params![fingerprint,self.dimensions as i64],|row|row.get(0)).optional()?;
|
params![fingerprint,self.dimensions as i64],|row|row.get(0)).optional()?;
|
||||||
let Some(active) = active else {
|
let Some(active) = active else {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
};
|
};
|
||||||
|
trace.end(generation_span, Default::default());
|
||||||
|
let ann_load = trace.begin("semantic.ann_load", parent);
|
||||||
|
trace.counter(
|
||||||
|
ann_load,
|
||||||
|
"cache_hit",
|
||||||
|
u64::from(
|
||||||
|
self.ann_cache
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|(generation, _)| *generation == active),
|
||||||
|
),
|
||||||
|
);
|
||||||
if !self
|
if !self
|
||||||
.ann_cache
|
.ann_cache
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|(generation, _)| *generation == active)
|
.is_some_and(|(generation, _)| *generation == active)
|
||||||
{
|
{
|
||||||
|
let blob_span = trace.begin("semantic.ann_blob_read", ann_load);
|
||||||
let length: Option<usize> = snapshot.query_row(
|
let length: Option<usize> = snapshot.query_row(
|
||||||
"SELECT length(ann_blob) FROM semantic_generations WHERE id=?1",
|
"SELECT length(ann_blob) FROM semantic_generations WHERE id=?1",
|
||||||
[active],
|
[active],
|
||||||
@@ -346,10 +501,15 @@ impl Store {
|
|||||||
[active],
|
[active],
|
||||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
)?;
|
)?;
|
||||||
|
trace.counter(blob_span, "bytes", serialized.len() as u64);
|
||||||
|
trace.end(blob_span, Default::default());
|
||||||
|
let checksum_span = trace.begin("semantic.ann_checksum", ann_load);
|
||||||
ensure!(
|
ensure!(
|
||||||
format!("{:x}", Sha256::digest(&serialized)) == checksum,
|
format!("{:x}", Sha256::digest(&serialized)) == checksum,
|
||||||
"ANN generation SHA256 mismatch; rebuild embeddings explicitly"
|
"ANN generation SHA256 mismatch; rebuild embeddings explicitly"
|
||||||
);
|
);
|
||||||
|
trace.end(checksum_span, Default::default());
|
||||||
|
let deserialize_span = trace.begin("semantic.ann_deserialize", ann_load);
|
||||||
let ann = self.new_ann()?;
|
let ann = self.new_ann()?;
|
||||||
ann.load_from_buffer(&serialized)?;
|
ann.load_from_buffer(&serialized)?;
|
||||||
ensure!(
|
ensure!(
|
||||||
@@ -357,8 +517,12 @@ impl Store {
|
|||||||
"ANN dimensions differ from embedding provider"
|
"ANN dimensions differ from embedding provider"
|
||||||
);
|
);
|
||||||
ann.reserve_capacity_and_threads(ann.size().max(1), 1)?;
|
ann.reserve_capacity_and_threads(ann.size().max(1), 1)?;
|
||||||
|
trace.counter(deserialize_span, "chunks", ann.size() as u64);
|
||||||
|
trace.end(deserialize_span, Default::default());
|
||||||
self.ann_cache = Some((active, ann));
|
self.ann_cache = Some((active, ann));
|
||||||
}
|
}
|
||||||
|
trace.end(ann_load, Default::default());
|
||||||
|
let filter_span = trace.begin("semantic.filter_normalization", parent);
|
||||||
let folder = options
|
let folder = options
|
||||||
.folder
|
.folder
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -384,13 +548,25 @@ impl Store {
|
|||||||
max_size,
|
max_size,
|
||||||
modified_after: options.modified_after_unix_ms,
|
modified_after: options.modified_after_unix_ms,
|
||||||
modified_before: options.modified_before_unix_ms,
|
modified_before: options.modified_before_unix_ms,
|
||||||
|
min_duration: options.min_duration_ms,
|
||||||
|
max_duration: options.max_duration_ms,
|
||||||
|
source_type: source_type.map(str::to_owned),
|
||||||
};
|
};
|
||||||
|
trace.end(filter_span, Default::default());
|
||||||
|
let eligibility_span = trace.begin("semantic.eligibility", parent);
|
||||||
let (revision, floor): (i64, i64) = snapshot.query_row(
|
let (revision, floor): (i64, i64) = snapshot.query_row(
|
||||||
"SELECT corpus_revision,dirty_floor FROM semantic_meta WHERE singleton=1",
|
"SELECT corpus_revision,dirty_floor FROM semantic_meta WHERE singleton=1",
|
||||||
[],
|
[],
|
||||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
)?;
|
)?;
|
||||||
let existing = self.eligibility_cache.take();
|
let existing = self.eligibility_cache.take();
|
||||||
|
trace.counter(
|
||||||
|
eligibility_span,
|
||||||
|
"cache_hit",
|
||||||
|
u64::from(existing.as_ref().is_some_and(|c| {
|
||||||
|
c.generation == active && c.filter == filter && c.revision == revision
|
||||||
|
})),
|
||||||
|
);
|
||||||
let mut cache = match existing {
|
let mut cache = match existing {
|
||||||
Some(mut cache)
|
Some(mut cache)
|
||||||
if cache.generation == active
|
if cache.generation == active
|
||||||
@@ -404,8 +580,17 @@ impl Store {
|
|||||||
row.get::<_, String>(0)
|
row.get::<_, String>(0)
|
||||||
})?
|
})?
|
||||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
trace.counter(eligibility_span, "dirty_paths", paths.len() as u64);
|
||||||
if paths.len() > 1000 {
|
if paths.len() > 1000 {
|
||||||
cache = collect_eligibility(&snapshot, active, revision, &filter, None)?;
|
cache = collect_eligibility_traced(
|
||||||
|
&snapshot,
|
||||||
|
active,
|
||||||
|
revision,
|
||||||
|
&filter,
|
||||||
|
None,
|
||||||
|
trace,
|
||||||
|
eligibility_span,
|
||||||
|
)?;
|
||||||
} else {
|
} else {
|
||||||
for path in paths {
|
for path in paths {
|
||||||
if let Some(old) = cache.by_path.remove(&path) {
|
if let Some(old) = cache.by_path.remove(&path) {
|
||||||
@@ -413,12 +598,14 @@ impl Store {
|
|||||||
Arc::make_mut(&mut cache.keys).remove(&key);
|
Arc::make_mut(&mut cache.keys).remove(&key);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let changed = collect_eligibility(
|
let changed = collect_eligibility_traced(
|
||||||
&snapshot,
|
&snapshot,
|
||||||
active,
|
active,
|
||||||
revision,
|
revision,
|
||||||
&filter,
|
&filter,
|
||||||
Some(&path),
|
Some(&path),
|
||||||
|
trace,
|
||||||
|
eligibility_span,
|
||||||
)?;
|
)?;
|
||||||
Arc::make_mut(&mut cache.keys).extend(changed.keys.iter().copied());
|
Arc::make_mut(&mut cache.keys).extend(changed.keys.iter().copied());
|
||||||
cache.by_path.extend(changed.by_path);
|
cache.by_path.extend(changed.by_path);
|
||||||
@@ -428,12 +615,22 @@ impl Store {
|
|||||||
}
|
}
|
||||||
cache
|
cache
|
||||||
}
|
}
|
||||||
_ => collect_eligibility(&snapshot, active, revision, &filter, None)?,
|
_ => collect_eligibility_traced(
|
||||||
|
&snapshot,
|
||||||
|
active,
|
||||||
|
revision,
|
||||||
|
&filter,
|
||||||
|
None,
|
||||||
|
trace,
|
||||||
|
eligibility_span,
|
||||||
|
)?,
|
||||||
};
|
};
|
||||||
ensure!(
|
ensure!(
|
||||||
cache.keys.len() <= MAX_SCAN_CHUNKS,
|
cache.keys.len() <= MAX_SCAN_CHUNKS,
|
||||||
"Eligibility cache exceeds chunk resource budget"
|
"Eligibility cache exceeds chunk resource budget"
|
||||||
);
|
);
|
||||||
|
trace.counter(eligibility_span, "eligible_chunks", cache.keys.len() as u64);
|
||||||
|
trace.end(eligibility_span, Default::default());
|
||||||
cache.revision = revision;
|
cache.revision = revision;
|
||||||
let eligible = Arc::clone(&cache.keys);
|
let eligible = Arc::clone(&cache.keys);
|
||||||
self.eligibility_cache = Some(cache);
|
self.eligibility_cache = Some(cache);
|
||||||
@@ -443,40 +640,97 @@ impl Store {
|
|||||||
let ann = &self.ann_cache.as_ref().expect("loaded ANN cache").1;
|
let ann = &self.ann_cache.as_ref().expect("loaded ANN cache").1;
|
||||||
let mut candidate_count = limit.saturating_mul(4).min(eligible.len()).max(1);
|
let mut candidate_count = limit.saturating_mul(4).min(eligible.len()).max(1);
|
||||||
let mut results;
|
let mut results;
|
||||||
|
let mut filesystem_validity = HashMap::<String, bool>::new();
|
||||||
|
let mut passes = 0u64;
|
||||||
loop {
|
loop {
|
||||||
|
passes += 1;
|
||||||
|
let pass_span = trace.begin("semantic.ann_pass", parent);
|
||||||
|
trace.counter(pass_span, "requested_chunks", candidate_count as u64);
|
||||||
|
let ann_span = trace.begin("semantic.ann_search", pass_span);
|
||||||
let matches =
|
let matches =
|
||||||
ann.filtered_search(query, candidate_count, |key| eligible.contains(&key))?;
|
ann.filtered_search(query, candidate_count, |key| eligible.contains(&key))?;
|
||||||
|
trace.counter(ann_span, "returned_chunks", matches.keys.len() as u64);
|
||||||
|
trace.end(ann_span, Default::default());
|
||||||
|
let materialization_span = trace.begin("semantic.parent_materialization", pass_span);
|
||||||
let mut ranked: Vec<_> = matches.keys.into_iter().zip(matches.distances).collect();
|
let mut ranked: Vec<_> = matches.keys.into_iter().zip(matches.distances).collect();
|
||||||
ranked.sort_by(|a, b| a.1.total_cmp(&b.1).then_with(|| a.0.cmp(&b.0)));
|
ranked.sort_by(|a, b| a.1.total_cmp(&b.1).then_with(|| a.0.cmp(&b.0)));
|
||||||
let mut best = HashMap::<String, SearchResult>::new();
|
let mut best = HashMap::<String, SearchResult>::new();
|
||||||
let mut materialize=snapshot.prepare("SELECT c.path,f.filename,c.snippet,c.kind,c.number FROM semantic_chunks c JOIN files f ON f.path=c.path WHERE c.rowid=?1 AND c.generation=?2")?;
|
let mut materialize=snapshot.prepare("SELECT c.path,f.filename,c.snippet,c.kind,c.number,c.source_type,c.file_size,c.file_mtime FROM semantic_chunks c JOIN files f ON f.path=c.path WHERE c.rowid=?1 AND c.generation=?2")?;
|
||||||
for (key, distance) in ranked {
|
for (key, distance) in ranked {
|
||||||
let cosine = 1.0 - distance;
|
let cosine = 1.0 - distance;
|
||||||
ensure!(cosine.is_finite(), "ANN produced nonfinite distance");
|
ensure!(cosine.is_finite(), "ANN produced nonfinite distance");
|
||||||
if cosine < MIN_COSINE {
|
if cosine < MIN_COSINE {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let result =
|
let (result, size, mtime) =
|
||||||
materialize.query_row(params![i64::try_from(key)?, active], |row| {
|
materialize.query_row(params![i64::try_from(key)?, active], |row| {
|
||||||
Ok(SearchResult {
|
Ok((
|
||||||
path: row.get(0)?,
|
SearchResult {
|
||||||
filename: row.get(1)?,
|
path: row.get(0)?,
|
||||||
snippet: row.get(2)?,
|
filename: row.get(1)?,
|
||||||
score: f64::from(cosine),
|
snippet: row.get(2)?,
|
||||||
match_kind: "semantic".into(),
|
score: f64::from(cosine),
|
||||||
source: Some(SourceLocation {
|
match_kind: match row.get::<_, String>(5)?.as_str() {
|
||||||
kind: row.get(3)?,
|
"text" => "semantic".into(),
|
||||||
number: row.get(4)?,
|
other => format!("semantic_{other}"),
|
||||||
}),
|
},
|
||||||
extraction_error: None,
|
source: Some(SourceLocation {
|
||||||
})
|
kind: row.get(3)?,
|
||||||
|
number: row.get(4)?,
|
||||||
|
}),
|
||||||
|
extraction_error: None,
|
||||||
|
},
|
||||||
|
row.get::<_, Option<i64>>(6)?,
|
||||||
|
row.get::<_, Option<i64>>(7)?,
|
||||||
|
))
|
||||||
})?;
|
})?;
|
||||||
|
if let (Some(size), Some(mtime)) = (size, mtime) {
|
||||||
|
let valid = *filesystem_validity
|
||||||
|
.entry(result.path.clone())
|
||||||
|
.or_insert_with(|| {
|
||||||
|
#[cfg(test)]
|
||||||
|
MEDIA_STAT_CALLS.with(|count| count.set(count.get() + 1));
|
||||||
|
let validation_span = trace.begin(
|
||||||
|
"semantic.parent_filesystem_validation",
|
||||||
|
materialization_span,
|
||||||
|
);
|
||||||
|
let valid =
|
||||||
|
std::fs::metadata(&result.path)
|
||||||
|
.ok()
|
||||||
|
.is_some_and(|metadata| {
|
||||||
|
metadata.is_file()
|
||||||
|
&& i64::try_from(metadata.len()).ok() == Some(size)
|
||||||
|
&& metadata
|
||||||
|
.modified()
|
||||||
|
.ok()
|
||||||
|
.and_then(|t| {
|
||||||
|
t.duration_since(std::time::UNIX_EPOCH).ok()
|
||||||
|
})
|
||||||
|
.and_then(|d| i64::try_from(d.as_nanos()).ok())
|
||||||
|
== Some(mtime)
|
||||||
|
});
|
||||||
|
trace.counter(validation_span, "valid", u64::from(valid));
|
||||||
|
trace.end(validation_span, Default::default());
|
||||||
|
valid
|
||||||
|
});
|
||||||
|
if !valid {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
if best.contains_key(&result.path) {
|
if best.contains_key(&result.path) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
best.insert(result.path.clone(), result);
|
best.insert(result.path.clone(), result);
|
||||||
}
|
}
|
||||||
results = best.into_values().collect::<Vec<_>>();
|
results = best.into_values().collect::<Vec<_>>();
|
||||||
|
trace.counter(materialization_span, "unique_parents", results.len() as u64);
|
||||||
|
trace.counter(
|
||||||
|
materialization_span,
|
||||||
|
"filesystem_checks",
|
||||||
|
filesystem_validity.len() as u64,
|
||||||
|
);
|
||||||
|
trace.end(materialization_span, Default::default());
|
||||||
|
trace.end(pass_span, Default::default());
|
||||||
if results.len() >= limit || candidate_count == eligible.len() {
|
if results.len() >= limit || candidate_count == eligible.len() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -488,6 +742,13 @@ impl Store {
|
|||||||
.then_with(|| a.path.cmp(&b.path))
|
.then_with(|| a.path.cmp(&b.path))
|
||||||
});
|
});
|
||||||
results.truncate(limit);
|
results.truncate(limit);
|
||||||
|
trace.counter(parent, "ann_passes", passes);
|
||||||
|
trace.counter(parent, "result_files", results.len() as u64);
|
||||||
|
trace.counter(
|
||||||
|
parent,
|
||||||
|
"filesystem_checks",
|
||||||
|
filesystem_validity.len() as u64,
|
||||||
|
);
|
||||||
snapshot.commit()?;
|
snapshot.commit()?;
|
||||||
Ok(results)
|
Ok(results)
|
||||||
}
|
}
|
||||||
@@ -614,6 +875,30 @@ impl Store {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn collect_eligibility_traced(
|
||||||
|
connection: &Connection,
|
||||||
|
generation: i64,
|
||||||
|
revision: i64,
|
||||||
|
filter: &FilterKey,
|
||||||
|
path: Option<&str>,
|
||||||
|
trace: &mut TraceCollector,
|
||||||
|
parent: Option<SpanToken>,
|
||||||
|
) -> Result<EligibilityCache> {
|
||||||
|
trace.timed(
|
||||||
|
"semantic.eligibility_filtered_scan_and_hash",
|
||||||
|
parent,
|
||||||
|
|trace, span| {
|
||||||
|
let result = collect_eligibility(connection, generation, revision, filter, path);
|
||||||
|
if let Ok(cache) = &result {
|
||||||
|
trace.counter(span, "eligible_chunks", cache.keys.len() as u64);
|
||||||
|
}
|
||||||
|
trace.counter(span, "single_dirty_file", u64::from(path.is_some()));
|
||||||
|
result
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn collect_eligibility(
|
fn collect_eligibility(
|
||||||
connection: &Connection,
|
connection: &Connection,
|
||||||
generation: i64,
|
generation: i64,
|
||||||
@@ -621,12 +906,15 @@ fn collect_eligibility(
|
|||||||
filter: &FilterKey,
|
filter: &FilterKey,
|
||||||
path: Option<&str>,
|
path: Option<&str>,
|
||||||
) -> Result<EligibilityCache> {
|
) -> Result<EligibilityCache> {
|
||||||
let mut statement=connection.prepare("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path FROM semantic_chunks c
|
let mut statement=connection.prepare("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c
|
||||||
JOIN passages p ON p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
|
LEFT JOIN passages p ON c.source_type='text' AND p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
|
||||||
JOIN files f ON f.path=c.path WHERE c.generation=?1
|
JOIN files f ON f.path=c.path WHERE c.generation=?1 AND ((c.source_type='text' AND p.rowid IS NOT NULL) OR (c.source_type!='text' AND c.file_size=f.size AND c.file_mtime=f.mtime))
|
||||||
AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2) AND (?3 IS NULL OR f.extension=?3)
|
AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2) AND (?3 IS NULL OR f.extension=?3)
|
||||||
AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
|
AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
|
||||||
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
|
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
|
||||||
|
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10))
|
||||||
|
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11))
|
||||||
|
AND (?12 IS NULL OR c.source_type=?12)
|
||||||
AND (?9 IS NULL OR c.path=?9) ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
|
AND (?9 IS NULL OR c.path=?9) ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
|
||||||
let mut rows = statement.query(params![
|
let mut rows = statement.query(params![
|
||||||
generation,
|
generation,
|
||||||
@@ -637,7 +925,10 @@ fn collect_eligibility(
|
|||||||
filter.modified_after,
|
filter.modified_after,
|
||||||
filter.modified_before,
|
filter.modified_before,
|
||||||
MAX_SCAN_CHUNKS as i64 + 1,
|
MAX_SCAN_CHUNKS as i64 + 1,
|
||||||
path
|
path,
|
||||||
|
filter.min_duration,
|
||||||
|
filter.max_duration,
|
||||||
|
filter.source_type
|
||||||
])?;
|
])?;
|
||||||
let mut hashes = HashMap::<i64, String>::new();
|
let mut hashes = HashMap::<i64, String>::new();
|
||||||
let mut keys = HashSet::new();
|
let mut keys = HashSet::new();
|
||||||
@@ -650,7 +941,9 @@ fn collect_eligibility(
|
|||||||
"Corpus exceeds {MAX_SCAN_CHUNKS} chunk resource budget"
|
"Corpus exceeds {MAX_SCAN_CHUNKS} chunk resource budget"
|
||||||
);
|
);
|
||||||
let passage_id: i64 = row.get(1)?;
|
let passage_id: i64 = row.get(1)?;
|
||||||
let hash = if let Some(hash) = hashes.get(&passage_id) {
|
let hash = if row.get::<_, String>(5)? != "text" {
|
||||||
|
row.get::<_, String>(2)?
|
||||||
|
} else if let Some(hash) = hashes.get(&passage_id) {
|
||||||
hash.clone()
|
hash.clone()
|
||||||
} else {
|
} else {
|
||||||
let hash = content_hash(&row.get::<_, String>(3)?);
|
let hash = content_hash(&row.get::<_, String>(3)?);
|
||||||
@@ -678,6 +971,340 @@ mod tests {
|
|||||||
use searchi_core::SearchIndex;
|
use searchi_core::SearchIndex;
|
||||||
const DIMENSIONS: usize = 768;
|
const DIMENSIONS: usize = 768;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn many_video_windows_do_not_crowd_out_other_parent_files() -> Result<()> {
|
||||||
|
// Geometry verifies grouped retrieval, not model inference or codecs.
|
||||||
|
let directory = tempfile::tempdir()?;
|
||||||
|
let root = directory.path().join("media");
|
||||||
|
std::fs::create_dir(&root)?;
|
||||||
|
for name in ["long.mp4", "other.mp4", "excluded.png"] {
|
||||||
|
std::fs::write(root.join(name), b"parent-grouping identity fixture")?;
|
||||||
|
}
|
||||||
|
let db = directory.path().join("index.db");
|
||||||
|
let mut lexical = SearchIndex::open(&db)?;
|
||||||
|
lexical.index_folder(&root)?;
|
||||||
|
let mut store = Store::open(&db, DIMENSIONS)?;
|
||||||
|
let generation = store.begin("grouped-retrieval-proof")?;
|
||||||
|
let mut query = vec![0.; DIMENSIONS];
|
||||||
|
query[0] = 1.;
|
||||||
|
let mut second = query.clone();
|
||||||
|
second[0] = 0.95;
|
||||||
|
second[1] = (1.0_f32 - 0.95_f32.powi(2)).sqrt();
|
||||||
|
for (id, path, _, _) in store.media_batch(0)? {
|
||||||
|
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
|
||||||
|
if path.ends_with("long.mp4") {
|
||||||
|
for ordinal in 0..96 {
|
||||||
|
store.insert_media(
|
||||||
|
generation,
|
||||||
|
id,
|
||||||
|
ordinal,
|
||||||
|
&identity,
|
||||||
|
"video",
|
||||||
|
ordinal as u32 * 2000,
|
||||||
|
&query,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
} else if path.ends_with("other.mp4") {
|
||||||
|
store.insert_media(generation, id, 0, &identity, "video", 4000, &second)?;
|
||||||
|
} else {
|
||||||
|
store.insert_media(generation, id, 0, &identity, "image", 0, &query)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
store.activate(generation)?;
|
||||||
|
let options = SearchOptions {
|
||||||
|
extension: Some("mp4".into()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
MEDIA_STAT_CALLS.with(|count| count.set(0));
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
let results = store.search_origin_traced(
|
||||||
|
&query,
|
||||||
|
"grouped-retrieval-proof",
|
||||||
|
2,
|
||||||
|
&options,
|
||||||
|
None,
|
||||||
|
&mut trace,
|
||||||
|
None,
|
||||||
|
)?;
|
||||||
|
assert_eq!(results.len(), 2);
|
||||||
|
assert!(results[0].path.ends_with("long.mp4"));
|
||||||
|
assert!(results[1].path.ends_with("other.mp4"));
|
||||||
|
assert_eq!(results[1].source.as_ref().unwrap().number, 4000);
|
||||||
|
assert_eq!(
|
||||||
|
MEDIA_STAT_CALLS.with(std::cell::Cell::get),
|
||||||
|
2,
|
||||||
|
"Each parent is stat-checked once despite repeated ANN expansions"
|
||||||
|
);
|
||||||
|
let metrics = trace.finish(results.len());
|
||||||
|
let spans = metrics.trace.unwrap();
|
||||||
|
assert!(
|
||||||
|
spans
|
||||||
|
.iter()
|
||||||
|
.filter(|s| s.name == "semantic.ann_pass")
|
||||||
|
.count()
|
||||||
|
> 1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
spans
|
||||||
|
.iter()
|
||||||
|
.filter(|s| s.name == "semantic.parent_filesystem_validation")
|
||||||
|
.count(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
spans
|
||||||
|
.iter()
|
||||||
|
.find(|s| s.name == "semantic.retrieval")
|
||||||
|
.unwrap()
|
||||||
|
.counters["ann_passes"]
|
||||||
|
> 1
|
||||||
|
);
|
||||||
|
for span in &spans {
|
||||||
|
if let Some(parent) = span.parent {
|
||||||
|
let parent = &spans[parent];
|
||||||
|
assert!(span.start_ms >= parent.start_ms);
|
||||||
|
assert!(
|
||||||
|
span.start_ms + span.duration_ms <= parent.start_ms + parent.duration_ms + 1e-6
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_origins_persist_filters_and_changes_remove_stale_candidates() -> Result<()> {
|
||||||
|
let directory = tempfile::tempdir()?;
|
||||||
|
let root = directory.path().join("media");
|
||||||
|
std::fs::create_dir(&root)?;
|
||||||
|
let file = root.join("clip.mp4");
|
||||||
|
std::fs::write(&file, b"storage identity fixture; not an inference test")?;
|
||||||
|
let db = directory.path().join("index.db");
|
||||||
|
let mut lexical = SearchIndex::open(&db)?;
|
||||||
|
lexical.index_folder(&root)?;
|
||||||
|
let mut store = Store::open(&db, DIMENSIONS)?;
|
||||||
|
let (id, path, _, _) = store.media_batch(0)?.remove(0);
|
||||||
|
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
|
||||||
|
lexical.set_media_metadata(
|
||||||
|
&path,
|
||||||
|
"artist: real extracted metadata fixture",
|
||||||
|
65000,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
)?;
|
||||||
|
let mut vector = vec![0.; DIMENSIONS];
|
||||||
|
vector[0] = 1.;
|
||||||
|
let generation = store.begin("media-storage-proof")?;
|
||||||
|
store.insert_media(generation, id, 0, &identity, "video", 2345, &vector)?;
|
||||||
|
store.insert_media(generation, id, 1, &identity, "audio", 28000, &vector)?;
|
||||||
|
store.activate(generation)?;
|
||||||
|
drop(store);
|
||||||
|
let mut store = Store::open(&db, DIMENSIONS)?;
|
||||||
|
let mut cold = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
let results = store.search_origin_traced(
|
||||||
|
&vector,
|
||||||
|
"media-storage-proof",
|
||||||
|
5,
|
||||||
|
&SearchOptions::default(),
|
||||||
|
None,
|
||||||
|
&mut cold,
|
||||||
|
None,
|
||||||
|
)?;
|
||||||
|
let cold = cold.finish(results.len()).trace.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
cold.iter()
|
||||||
|
.find(|s| s.name == "semantic.ann_load")
|
||||||
|
.unwrap()
|
||||||
|
.counters["cache_hit"],
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert!(cold.iter().any(|s| s.name == "semantic.ann_checksum"));
|
||||||
|
let mut warm = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
let warm_results = store.search_origin_traced(
|
||||||
|
&vector,
|
||||||
|
"media-storage-proof",
|
||||||
|
5,
|
||||||
|
&SearchOptions::default(),
|
||||||
|
None,
|
||||||
|
&mut warm,
|
||||||
|
None,
|
||||||
|
)?;
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&results)?,
|
||||||
|
serde_json::to_value(&warm_results)?
|
||||||
|
);
|
||||||
|
let warm = warm.finish(warm_results.len()).trace.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
warm.iter()
|
||||||
|
.find(|s| s.name == "semantic.ann_load")
|
||||||
|
.unwrap()
|
||||||
|
.counters["cache_hit"],
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
warm.iter()
|
||||||
|
.find(|s| s.name == "semantic.eligibility")
|
||||||
|
.unwrap()
|
||||||
|
.counters["cache_hit"],
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert!(!warm.iter().any(|s| s.name == "semantic.ann_blob_read"));
|
||||||
|
assert_eq!(results.len(), 1);
|
||||||
|
assert_eq!(results[0].source.as_ref().unwrap().kind, "timestamp_ms");
|
||||||
|
assert!([2345, 28000].contains(&results[0].source.as_ref().unwrap().number));
|
||||||
|
let audio = store.search_origin(
|
||||||
|
&vector,
|
||||||
|
"media-storage-proof",
|
||||||
|
5,
|
||||||
|
&SearchOptions::default(),
|
||||||
|
Some("audio"),
|
||||||
|
)?;
|
||||||
|
assert_eq!(audio.len(), 1);
|
||||||
|
assert_eq!(audio[0].match_kind, "semantic_audio");
|
||||||
|
assert_eq!(audio[0].source.as_ref().unwrap().number, 28000);
|
||||||
|
let video = store.search_origin(
|
||||||
|
&vector,
|
||||||
|
"media-storage-proof",
|
||||||
|
5,
|
||||||
|
&SearchOptions::default(),
|
||||||
|
Some("video"),
|
||||||
|
)?;
|
||||||
|
assert_eq!(video.len(), 1);
|
||||||
|
assert_eq!(video[0].match_kind, "semantic_video");
|
||||||
|
assert_eq!(video[0].source.as_ref().unwrap().number, 2345);
|
||||||
|
assert!(store
|
||||||
|
.search_origin(
|
||||||
|
&vector,
|
||||||
|
"media-storage-proof",
|
||||||
|
5,
|
||||||
|
&SearchOptions::default(),
|
||||||
|
Some("image")
|
||||||
|
)?
|
||||||
|
.is_empty());
|
||||||
|
assert!(
|
||||||
|
results[0].match_kind == "semantic_video" || results[0].match_kind == "semantic_audio"
|
||||||
|
);
|
||||||
|
let filter = SearchOptions {
|
||||||
|
extension: Some("png".into()),
|
||||||
|
..SearchOptions::default()
|
||||||
|
};
|
||||||
|
assert!(store
|
||||||
|
.search(&vector, "media-storage-proof", 5, &filter)?
|
||||||
|
.is_empty());
|
||||||
|
let duration_filter = SearchOptions {
|
||||||
|
min_duration_ms: Some(66000),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(store
|
||||||
|
.search(&vector, "media-storage-proof", 5, &duration_filter)?
|
||||||
|
.is_empty());
|
||||||
|
let duration_filter = SearchOptions {
|
||||||
|
min_duration_ms: Some(64000),
|
||||||
|
max_duration_ms: Some(66000),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.search(&vector, "media-storage-proof", 5, &duration_filter)?
|
||||||
|
.len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
std::fs::write(&file, b"changed")?;
|
||||||
|
// Foreground hits check only actual candidate metadata, before watcher
|
||||||
|
// reconciliation; they never rehash the complete filesystem corpus.
|
||||||
|
assert!(store
|
||||||
|
.search(&vector, "media-storage-proof", 5, &SearchOptions::default())?
|
||||||
|
.is_empty());
|
||||||
|
lexical.index_folder(&root)?;
|
||||||
|
assert!(store
|
||||||
|
.search(&vector, "media-storage-proof", 5, &SearchOptions::default())?
|
||||||
|
.is_empty());
|
||||||
|
assert!(store
|
||||||
|
.insert_media(generation, id, 2, &identity, "video", 0, &vector)
|
||||||
|
.is_err());
|
||||||
|
std::fs::remove_file(&file)?;
|
||||||
|
lexical.index_folder(&root)?;
|
||||||
|
assert!(store
|
||||||
|
.search(&vector, "media-storage-proof", 5, &SearchOptions::default())?
|
||||||
|
.is_empty());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_failure_removes_partial_vectors_and_persists_reason() -> Result<()> {
|
||||||
|
let directory = tempfile::tempdir()?;
|
||||||
|
let file = directory.path().join("picture.png");
|
||||||
|
std::fs::write(&file, b"identity fixture")?;
|
||||||
|
let db = directory.path().join("index.db");
|
||||||
|
let mut lexical = SearchIndex::open(&db)?;
|
||||||
|
lexical.index_folder(directory.path())?;
|
||||||
|
let mut store = Store::open(&db, DIMENSIONS)?;
|
||||||
|
let (id, path, _, _) = store
|
||||||
|
.media_batch(0)?
|
||||||
|
.into_iter()
|
||||||
|
.find(|(_, p, _, _)| p.ends_with("picture.png"))
|
||||||
|
.unwrap();
|
||||||
|
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
|
||||||
|
let mut vector = vec![0.; DIMENSIONS];
|
||||||
|
vector[0] = 1.;
|
||||||
|
let generation = store.begin("failure-proof")?;
|
||||||
|
store.insert_media(generation, id, 0, &identity, "image", 0, &vector)?;
|
||||||
|
store.media_failure(generation, &path, "Decoder rejected corrupt image")?;
|
||||||
|
let chunks: usize = store.connection.query_row(
|
||||||
|
"SELECT count(*) FROM semantic_chunks WHERE generation=?1",
|
||||||
|
[generation],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?;
|
||||||
|
assert_eq!(chunks, 0);
|
||||||
|
let reason: String = store.connection.query_row(
|
||||||
|
"SELECT message FROM semantic_media_errors WHERE generation=?1",
|
||||||
|
[generation],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?;
|
||||||
|
assert!(reason.contains("corrupt"));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn deleted_media_without_reconciliation_does_not_hide_later_valid_top_k() -> Result<()> {
|
||||||
|
let directory = tempfile::tempdir()?;
|
||||||
|
let root = directory.path().join("images");
|
||||||
|
std::fs::create_dir(&root)?;
|
||||||
|
for name in ["a.png", "b.png"] {
|
||||||
|
std::fs::write(root.join(name), b"storage fixture")?;
|
||||||
|
}
|
||||||
|
let db = directory.path().join("index.db");
|
||||||
|
let mut lexical = SearchIndex::open(&db)?;
|
||||||
|
lexical.index_folder(&root)?;
|
||||||
|
let mut store = Store::open(&db, DIMENSIONS)?;
|
||||||
|
let generation = store.begin("candidate-stat-proof")?;
|
||||||
|
let mut vector = vec![0.; DIMENSIONS];
|
||||||
|
vector[0] = 1.;
|
||||||
|
for (id, path, _, _) in store.media_batch(0)? {
|
||||||
|
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
|
||||||
|
store.insert_media(generation, id, 0, &identity, "image", 0, &vector)?;
|
||||||
|
}
|
||||||
|
store.activate(generation)?;
|
||||||
|
std::fs::remove_file(root.join("a.png"))?;
|
||||||
|
let results = store.search(
|
||||||
|
&vector,
|
||||||
|
"candidate-stat-proof",
|
||||||
|
1,
|
||||||
|
&SearchOptions::default(),
|
||||||
|
)?;
|
||||||
|
assert_eq!(results.len(), 1);
|
||||||
|
assert!(results[0].path.ends_with("b.png"));
|
||||||
|
std::fs::remove_file(root.join("b.png"))?;
|
||||||
|
assert!(store
|
||||||
|
.search(
|
||||||
|
&vector,
|
||||||
|
"candidate-stat-proof",
|
||||||
|
1,
|
||||||
|
&SearchOptions::default()
|
||||||
|
)?
|
||||||
|
.is_empty());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn generation_persists_failed_staging_preserves_active_and_changes_hide_stale_vectors(
|
fn generation_persists_failed_staging_preserves_active_and_changes_hide_stale_vectors(
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ The Python CMake package is needed only when CMake is absent from PATH.
|
|||||||
python -m pip install cmake
|
python -m pip install cmake
|
||||||
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
||||||
python tools/embeddinggemma2/stage_runtime.py --backend cpu
|
python tools/embeddinggemma2/stage_runtime.py --backend cpu
|
||||||
|
python tools/media-runtime/provision.py --stage
|
||||||
python tools/embeddinggemma2/download.py
|
python tools/embeddinggemma2/download.py
|
||||||
$env:SEARCHI_FORCE_CPU = '1'
|
$env:SEARCHI_FORCE_CPU = '1'
|
||||||
$env:SEARCHI_GEMMA_MODEL_DIR = (Resolve-Path .searchi/models/embeddinggemma2).Path
|
$env:SEARCHI_GEMMA_MODEL_DIR = (Resolve-Path .searchi/models/embeddinggemma2).Path
|
||||||
@@ -38,8 +39,7 @@ The worker is statically linked with the MSVC CRT on Windows. Staging copies
|
|||||||
it and the llama.cpp license into the desktop resources directory. Add
|
it and the llama.cpp license into the desktop resources directory. Add
|
||||||
`--bundle-model` to staging for an offline model bundle after downloading;
|
`--bundle-model` to staging for an offline model bundle after downloading;
|
||||||
otherwise desktop setup provisions verified models on first launch. Search
|
otherwise desktop setup provisions verified models on first launch. Search
|
||||||
itself performs no network inference or downloads. Media weights are staged
|
itself performs no network inference or downloads. The worker now uses the verified media projector for genuine image/audio embeddings. The FFmpeg script stages verified standalone decoders and their notices.
|
||||||
for future integration; the current worker supports text only.
|
|
||||||
|
|
||||||
The desktop is a standalone Cargo workspace and needs separate validation:
|
The desktop is a standalone Cargo workspace and needs separate validation:
|
||||||
|
|
||||||
@@ -50,7 +50,7 @@ npm ci
|
|||||||
npm run format:check
|
npm run format:check
|
||||||
npm test
|
npm test
|
||||||
npm run build
|
npm run build
|
||||||
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu'
|
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
|
||||||
npm run tauri -- dev
|
npm run tauri -- dev
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -67,13 +67,18 @@ Linux release. On a native Linux build host, run from the repository root:
|
|||||||
```sh
|
```sh
|
||||||
python3 tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
python3 tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
||||||
python3 tools/embeddinggemma2/download.py
|
python3 tools/embeddinggemma2/download.py
|
||||||
|
python3 tools/media-runtime/provision.py
|
||||||
export SEARCHI_FORCE_CPU=1
|
export SEARCHI_FORCE_CPU=1
|
||||||
cargo build --release -p searchi-cli --locked
|
cargo build --release -p searchi-cli --locked
|
||||||
./target/release/searchi index --db searchi.sqlite --root /path/to/documents
|
./target/release/searchi index --db searchi.sqlite --root /path/to/documents
|
||||||
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --cpu-threads 1
|
./target/release/searchi embed --db searchi.sqlite --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --ffmpeg .searchi/media-runtime/bin/ffmpeg --ffprobe .searchi/media-runtime/bin/ffprobe --cpu-threads 1
|
||||||
./target/release/searchi search --db searchi.sqlite 'supplier cancellation terms' --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --cpu-threads 1
|
./target/release/searchi search --db searchi.sqlite 'supplier cancellation terms' --model-dir .searchi/models/embeddinggemma2 --embedding-worker .searchi/runtime/cpu/searchi-gemma-worker --cpu-threads 1
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`embed` builds document, image, video and audio/music vectors by default. Use
|
||||||
|
`--text-only` explicitly to omit media. Default media indexing needs paired
|
||||||
|
FFmpeg/FFprobe paths or packaged helpers beside the installed CLI.
|
||||||
|
|
||||||
`searchi setup --model-dir PATH` is an alternative verified model-provisioning
|
`searchi setup --model-dir PATH` is an alternative verified model-provisioning
|
||||||
command. It provisions model assets, not the native worker. Package the worker
|
command. It provisions model assets, not the native worker. Package the worker
|
||||||
separately and pass its path to headless embedding/search. Each short-lived CLI
|
separately and pass its path to headless embedding/search. Each short-lived CLI
|
||||||
@@ -114,5 +119,9 @@ installers, stage the target worker, retain dependency/model notices, verify
|
|||||||
clean-machine launch and setup, run native Google smoke tests, and complete
|
clean-machine launch and setup, run native Google smoke tests, and complete
|
||||||
upgrade/migration, signatures, and rollback checks. Optional offline bundles
|
upgrade/migration, signatures, and rollback checks. Optional offline bundles
|
||||||
need both verified model files. See [semantic bounds and evidence](SEMANTIC.md)
|
need both verified model files. See [semantic bounds and evidence](SEMANTIC.md)
|
||||||
for current text-only, corpus, and performance limitations. Keep signing keys,
|
for current corpus, localization, and performance limitations. Keep signing keys,
|
||||||
repository tokens, and model credentials outside source control.
|
repository tokens, and model credentials outside source control.
|
||||||
|
FFmpeg provisioning supports manifest-pinned Windows/Linux x86-64 and ARM64 artifacts;
|
||||||
|
only Windows x86-64 has executed here. macOS decoder packaging remains a gate.
|
||||||
|
See [media runtime provenance](../tools/media-runtime/README.md) and
|
||||||
|
[measured media limitations](MEDIA-QUALITY.md).
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
# Media search quality
|
||||||
|
|
||||||
|
## Evaluation contract
|
||||||
|
|
||||||
|
The independent v1 corpus contains **20 real assets and 65 frozen queries**: 12 photos, four videos, two music performances, and two sound effects. Queries include English/German visual memories, visually similar hard negatives, timestamp-sensitive actions, spoken topics, exact artist/title metadata, strict filter constraints, and no-answer cases.
|
||||||
|
|
||||||
|
Labels were frozen before media inference with fixture SHA256 `b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393`. Original asset hashes and licenses are recorded in [fixture.json](../fixtures/media/fixture.json) and [provenance](../fixtures/media/README.md). All Windows VM evaluation uses the real Google worker on **CPU, one thread**, with CPU FFmpeg decoding. No simulated embeddings or inferred model captions enter the corpus labels.
|
||||||
|
|
||||||
|
The reference environment is Windows 11 Pro in a shared VM exposing four logical CPUs on an AMD Ryzen 7 7800X3D host, approximately 9.93 GiB visible memory. Inference deliberately uses one thread and `SEARCHI_FORCE_CPU=1`. [Recorded environment and worker hash](../fixtures/media/environment.json). Observed worker RAM varied around 0.9–1.2 GB during indexing; these snapshots are not a measured peak-memory guarantee.
|
||||||
|
|
||||||
|
The corpus has one assistant annotator. Actual image/video frames were visually inspected; speech timestamps use independently published NASA captions. Human audio listening and blinded second-annotator review remain unexecuted. Existing artist/title tags are retained and checked at the stream level, where Vorbis stores them. English/German pairs refer to the same assets and are correlated, not twice as many independent examples. There is only one spoken-topic video and two music files; category-perfect recall can therefore be easy and must not be extrapolated.
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
The first **direct real-model dense reference** completed on CPU with 94/95 candidate encodings. This is an embedding/retrieval diagnostic, not yet the production pipeline. Candidate filtering in that reference is a benchmark constraint; exact metadata was deliberately marked unsupported.
|
||||||
|
|
||||||
|
| Category | Queries | File recall@1 | File recall@5 | MRR | Precise timestamp hit@1 |
|
||||||
|
|---|---:|---:|---:|---:|---:|
|
||||||
|
| Photo memory | 24 | 100% | 100% | 1.000 | — |
|
||||||
|
| Video actions | 4 | 100% | 100% | 1.000 | 100% |
|
||||||
|
| Video scenes | 6 | 100% | 100% | 1.000 | 100% |
|
||||||
|
| Flower temporal states | 4 | 100% | 100% | 1.000 | 75% |
|
||||||
|
| Spoken topics in video | 8 | 100% | 100% | 1.000 | **25%** |
|
||||||
|
| Music memories | 4 | **50%** | **50%** | 0.500 | — |
|
||||||
|
| Sound effects | 4 | 100% | 100% | 1.000 | — |
|
||||||
|
|
||||||
|
Across 55 supported positive queries (including one reference extension constraint), recall@1/@5 and MRR were **96.36% / 96.36% / 0.964**. Across 22 temporal queries, segment overlap@1 was **95.45%**, but precise point hits were **68.18%**. [Full recorded summary and failures](../fixtures/media/reference-summary.json), [rankings](../fixtures/media/reference-results.json).
|
||||||
|
|
||||||
|
Important failures:
|
||||||
|
|
||||||
|
- **Music encoding harness:** the original jazz violin Vorbis decode briefly overshot normalized PCM range. The direct tool sent unclamped floats, and the worker correctly rejected that one candidate. The two associated music misses are an explicit preprocessing failure, not evidence that the model confuses piano with violin. The production decoder clamps its PCM and must be evaluated separately; preserve this failed reference rather than hiding it.
|
||||||
|
- **Speech localization:** all eight spoken-topic queries found the right file and an overlapping thirty-second audio segment, but six returned a segment start outside the labelled utterance. Correct-file retrieval therefore does **not** fulfill precise “jump to the bit where somebody mentions a topic.” Shorter local windows or another alignment mechanism need independent evaluation.
|
||||||
|
- **Temporal visual distinction:** one red-flower query selected the right clip but the wrong stage of opening.
|
||||||
|
- **No-answer behavior:** all six supported absent/negative queries returned nearest neighbors (**100% unsuppressed return rate**). No arbitrary cosine threshold was invented. This exposes missing abstention evidence, not a calibrated confidence score.
|
||||||
|
- **Exact metadata:** four identity/no-answer cases remain unsupported in the direct embedding lane. A separate actual FFprobe → guarded Searchi core FTS run passed all three frozen artist/title queries at rank 1 and returned no result for the absent artist: **recall@1/@5 = 100%, MRR = 1.0; absent-artist return rate = 0%**. [Metadata results](../fixtures/media/metadata-summary.json). These are three small lexical identity cases, not proof that hybrid fallback abstains or that all music collections work.
|
||||||
|
|
||||||
|
Encoding took **498.66 seconds** for the photo/frame/audio candidates in this CPU-one-thread run. Query p50/p95 were **92.78 / 122.93 ms** after encoding. Runtime reported `google/embeddinggemma-2`, 768 dimensions, CPU, one thread, `b11461-no-logits-v1`. Other concurrent VM work can affect timings; this is not a clean-room hardware benchmark.
|
||||||
|
|
||||||
|
### Actual production pipeline: 30-second audio windows
|
||||||
|
|
||||||
|
The real shared Searchi service, FFmpeg 9.0.2 decoder, metadata extraction, guarded core index, ANN retrieval and hybrid ranking completed **20/20 files and 118 chunks with zero encoding failures**. This rerun used the fixed decoder's exact sample trimming; the earlier direct harness failure remains recorded separately. Indexing took **668.04 seconds**, and warm query p50/p95 were **92.23 / 113.38 ms**, on one CPU thread. These serial evaluation runs can still share the VM with unrelated work.
|
||||||
|
|
||||||
|
Across 58 supported positive queries, actual file recall@1/@5 was **96.55% / 100%**, with **MRR 0.983**. EN recall@1 was 96.77% (31 positives), DE 96.30% (27 positives). Photo memories 24/24, video file retrieval 22/22, sound effects 4/4, artist/title metadata 3/3 and the positive extension filter 1/1 passed at rank 1. Music memories passed **2/4 at rank 1 and 4/4 in the first five**: both violin queries ranked piano first despite successful decoding, an actual music hard-negative confusion.
|
||||||
|
|
||||||
|
Temporal point accuracy was **12/22 (54.55%)**; segment overlap was 18/22 (81.82%). Video action/scenery point hits passed 10/10, flower state hits 2/4, and spoken-topic hits **0/8**. Two spoken orange-soil queries surfaced image/video-frame hits instead of audio. Remaining speech starts ranged from 0.539 to 15.777 seconds outside the labelled utterance. The 0/8 strict score does not mean every start is equally unusable; raw distance and wrong-channel failures are retained rather than tuning a tolerance to this test corpus.
|
||||||
|
|
||||||
|
All six supported no-answer cases returned results, including the absent artist through semantic fallback. The separate lexical-only metadata absence test passed, but that does **not** establish abstention for the hybrid product. A requested temporal-range negative was explicitly unsupported, rather than silently counted as a pass. [Production summary](../fixtures/media/pipeline-30s-summary.json), [complete actual rankings](../fixtures/media/pipeline-30s-results.json).
|
||||||
|
|
||||||
|
### Same-label shorter audio experiment
|
||||||
|
|
||||||
|
The same fixed decoder, files, queries and two-second overlap were rerun with ten-second audio windows. All 20 files indexed successfully into **154 chunks**, with zero failures. This is a predeclared policy comparison, not a new production default.
|
||||||
|
|
||||||
|
| Metric | 30-second windows | 10-second windows |
|
||||||
|
|---|---:|---:|
|
||||||
|
| CPU indexing time | 668.04 s | 671.51 s |
|
||||||
|
| Actual chunks | 118 | 154 |
|
||||||
|
| File recall@1 / @5 | 96.55% / 100% | 96.55% / 100% |
|
||||||
|
| MRR | 0.983 | 0.983 |
|
||||||
|
| Speech precise point hits | 0/8 | 1/8 |
|
||||||
|
| Speech segment overlap | 6/8 | 6/8 |
|
||||||
|
| Overall temporal point hits | 12/22 | 13/22 |
|
||||||
|
| Worst correct-channel timestamp distance | 15.777 s | 3.777 s |
|
||||||
|
| Warm query p50 / p95 | 92.23 / 113.38 ms | 90.65 / 110.42 ms |
|
||||||
|
| Music top-1 | 2/4 | 2/4 |
|
||||||
|
| No-answer returned results | 6/6 | 6/6 |
|
||||||
|
|
||||||
|
Shorter windows improve the worst raw localization distance, but only one strict utterance hit and unchanged wrong-channel failures are insufficient for the original spoken-topic navigation goal. Similar measured cost here does not establish cost equivalence for long recordings; window count, media mix and VM contention differ. The actual hybrid artist/title regression still passes all three identity cases. [Ten-second summary](../fixtures/media/pipeline-10s-summary.json), [complete rankings and policy fingerprint](../fixtures/media/pipeline-10s-results.json).
|
||||||
|
|
||||||
|
### Predeclared photo-budget comparison
|
||||||
|
|
||||||
|
The same 12 photos and 24 frozen EN/DE memories were encoded at 70, 280 and 1120 tokens. Query embeddings were computed once and shared; no labels or thresholds changed.
|
||||||
|
|
||||||
|
| Image tokens | Actual CPU batch encoding time | Recall@1 / @5 | MRR |
|
||||||
|
|---|---:|---:|---:|
|
||||||
|
| 70 | 47.61 s | 100% / 100% | 1.000 |
|
||||||
|
| 280 | 199.65 s | 100% / 100% | 1.000 |
|
||||||
|
| 1120 | 1493.32 s | 100% / 100% | 1.000 |
|
||||||
|
|
||||||
|
There was no retrieval improvement on this small corpus despite a substantial high-budget cost. The high-budget run overlapped independent native compilation/profiling in the shared CPU VM, so the table is an observed experiment, not a controlled hardware scaling law. A worker RAM snapshot reached approximately 1.5 GB; peak RAM was not instrumented. Equal recall here does not establish equivalent retrieval for small text, dense screenshots or subtle visual distinctions. Keep the existing production policy pending a broader quality/cost corpus; these observations do not justify making 1120 tokens the default. [Budget summary](../fixtures/media/image-budget-summary.json), [actual dimensions, rankings and worker report](../fixtures/media/image-budget-results.json).
|
||||||
|
|
||||||
|
Production defaults have not changed as part of these experiments. The current **self-rating is 4/10: experimental media retrieval**. Real visual retrieval is promising, but precise speech timestamps, music hard negatives and abstention remain essential gaps. This rating follows the rubric below rather than extrapolating the small corpus's high file-recall score. The production measurements above precede a subsequent spoken-cue channel-routing improvement; that change has not been rescored against this corpus and must not inherit a better measured score without a new run.
|
||||||
|
|
||||||
|
The existing **20-document/24-query synthetic regression** was rerun with the actual Google CPU worker and unchanged fixture SHA256 `3bab37fc0ec9c2e2e40684962b049172b5078358c6487c9033ed8697b8128dd7`:
|
||||||
|
|
||||||
|
| Lane | Top-1 | Recall@3 | Recall@10 |
|
||||||
|
|---|---:|---:|---:|
|
||||||
|
| Filename/keyword | 20.83% | 20.83% | 20.83% |
|
||||||
|
| Semantic | 100% | 100% | 100% |
|
||||||
|
| Hybrid | 100% | 100% | 100% |
|
||||||
|
|
||||||
|
Measured hybrid p95 was **146.83 ms** on this run. [Recorded output](../fixtures/media/document-baseline.json). This remains a synthetic seed, not independent production-quality evidence or a large-corpus performance claim.
|
||||||
|
|
||||||
|
## Scoring and release gates
|
||||||
|
|
||||||
|
Report recall@1/@5, MRR, language/type breakdowns, segment-overlap hits, precise timestamp point hits, and failure categories. No-answer return rate is distinct from positive recall. Do not invent an arbitrary cosine rejection threshold from these results; calibrate on a separate development set, then validate on held-out negatives.
|
||||||
|
|
||||||
|
Self-rating rubric is declared before scores: **1–2/10** broken core retrieval; **3–4/10** experimental retrieval with missing essential capabilities or unvalidated abstention/metadata; **5–6/10** useful limited beta after independent basic retrieval/filter/metadata checks; **7–8/10** strong pilot with high recall and timestamp quality on representative corpora; **9–10/10** requires production release evidence across all operating systems, large representative libraries, calibrated no-answer behavior, and complete lifecycle/security/resource verification. A small diagnostic alone cannot justify a production score.
|
||||||
|
|
||||||
|
Remaining gates: broader production metadata/filter enforcement beyond the four positive seed constraints, calibrated no-answer handling, reliable spoken-topic alignment and channel selection, realistic long videos, audio listening review, unseen music/accents, corrupt codec cases, missing files/renames, hardware resource bounds, and native Windows/Linux/macOS validation. The measured production pipeline and explicit failures above establish a reproducible baseline; they do not close these gates.
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
# Search performance diagnostics
|
||||||
|
|
||||||
|
Search performance must distinguish total request latency from the work inside
|
||||||
|
it. The shared Rust schema exposes basic metrics for routine searches and a
|
||||||
|
bounded nested trace for deliberate debugging. It uses monotonic `Instant`
|
||||||
|
timing, not wall-clock timestamps. No query text, source passages or file paths
|
||||||
|
belong in stage names/counters.
|
||||||
|
|
||||||
|
## Shared response
|
||||||
|
|
||||||
|
`SearchResponse<T>` contains `results: Vec<T>` and `metrics: SearchMetrics`.
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| total_ms | Elapsed time inside the collector's measured request boundary |
|
||||||
|
| result_count | Number of final returned results |
|
||||||
|
| trace | null in basic mode; debug spans in begin order |
|
||||||
|
| dropped_spans | Spans omitted when the 512-span ceiling or invalid parent is encountered |
|
||||||
|
|
||||||
|
A `SearchSpan` has a static `name`, relative `start_ms`, `duration_ms`, optional
|
||||||
|
parent index and integer `counters`. Parent indices reference earlier spans in
|
||||||
|
the same trace; child timings lie within their parent. Nested durations overlap,
|
||||||
|
so summing every span would double-count work. Counters are capped at 32 keys per
|
||||||
|
span, with 96-character labels. Missing spans due to the cap are visible through
|
||||||
|
`dropped_spans`, never presented as complete instrumentation.
|
||||||
|
|
||||||
|
Basic mode retains only the request start Instant and result count; there is no
|
||||||
|
per-stage span Vec or diagnostic string allocation. Debug mode captures up to
|
||||||
|
512 spans. Scope timings preserve normal closure return values and errors.
|
||||||
|
Already-ended or foreign tokens are harmless, and unfinished scopes close at
|
||||||
|
collector finish. Closing a parent closes its open descendants for containment.
|
||||||
|
|
||||||
|
## Instrumenting the shared engine
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use searchi_core::metrics::{MetricsMode, TraceCollector};
|
||||||
|
let mut trace = TraceCollector::new(MetricsMode::Debug);
|
||||||
|
let results = trace.timed("search", None, |trace, parent| {
|
||||||
|
trace.timed("candidate_selection", parent, |trace, token| {
|
||||||
|
trace.counter(token, "candidate_count", candidates.len() as u64);
|
||||||
|
retrieve_candidates(candidates)
|
||||||
|
})
|
||||||
|
});
|
||||||
|
let metrics = trace.finish(results.len());
|
||||||
|
```
|
||||||
|
|
||||||
|
This is an API illustration, not a measured benchmark. `begin(name,parent)` /
|
||||||
|
`end(token,counters)` supports explicit scopes spanning locks or separate calls;
|
||||||
|
`SpanToken` is Copy and exposes its index. `counter` records bounded integer
|
||||||
|
counts without source data. `elapsed_ms` reports elapsed request time. Capture
|
||||||
|
queue/lock boundaries in the caller, then pass that same collector into the
|
||||||
|
shared core/service so stages refer to one origin.
|
||||||
|
|
||||||
|
Useful actual boundaries include request admission/reader lock, filter snapshot,
|
||||||
|
filename/FTS retrieval, tokenizer/query inference, active ANN load, eligibility
|
||||||
|
refresh, filtered HNSW traversal, per-file deduplication, ranking/fusion and
|
||||||
|
result assembly. Name only work that is truly measured. Do not label the whole
|
||||||
|
worker round trip as pure neural execution or fabricate stage splits from a
|
||||||
|
single elapsed number. Model load/warmup and initial eligibility work should be
|
||||||
|
reported separately from a retained worker's first foreground and warm queries.
|
||||||
|
|
||||||
|
## CLI and desktop integration
|
||||||
|
|
||||||
|
The CLI exposes these actual modes:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
searchi search --db searchi.sqlite contract --json
|
||||||
|
searchi search --db searchi.sqlite contract --performance --json
|
||||||
|
searchi search --db searchi.sqlite contract --debug-performance --json
|
||||||
|
```
|
||||||
|
|
||||||
|
Plain `--json` preserves the legacy result array. `--performance --json` emits
|
||||||
|
`SearchResponse` with basic metrics; `--debug-performance --json` emits the
|
||||||
|
same envelope with detailed spans. Normal terminal output prints results and
|
||||||
|
reports elapsed time/count on stderr, including CLI index/session initialization.
|
||||||
|
Debug terminal mode also prints the detailed metrics on stderr. Legacy Vec
|
||||||
|
interfaces remain wrappers for compatibility. The desktop should use a compact ordinary summary and an optional
|
||||||
|
diagnostic view, without confusing score/similarity with elapsed time or confidence.
|
||||||
|
|
||||||
|
Both measured GUI commands start the native clock at command entry, include instrumented queue/lock waits, and finish before response serialization. UI timing separately captures input/debounce, lane/retry waits, invoke/round trip and synchronous DOM updates. Display latency ends at the DOM update; actual compositor paint and deferred media preview loading are not measured. Their overlapping boundaries must not be summed. Native metrics exclude frontend delays and post-return paint. A fast Rust request does not
|
||||||
|
prove a responsive first foreground interaction. Conversely, pending setup or
|
||||||
|
model loading must not be silently included in a warm-query claim.
|
||||||
|
|
||||||
|
## Reproducible performance validation
|
||||||
|
|
||||||
|
Record build profile, CPU/device and thread budget, VM/host resources, corpus and
|
||||||
|
active generation, filters, cold/warm state, concurrent indexing and setup costs.
|
||||||
|
Use multiple queries and p50/p95 distributions; retain raw reports. Compare basic
|
||||||
|
and debug mode on the same corpus to quantify instrumentation overhead. Debug
|
||||||
|
traces guide investigation but are not a substitute for representative benchmarks.
|
||||||
|
|
||||||
|
Current recorded filename, actual Google seed and synthetic ANN evidence is in
|
||||||
|
[VALIDATION](VALIDATION.md) and [SEMANTIC](SEMANTIC.md). Media quality/timing is in
|
||||||
|
[MEDIA-QUALITY](MEDIA-QUALITY.md). Synthetic vectors measure index behavior only;
|
||||||
|
real Google inference remains required for product semantic search. Platform,
|
||||||
|
large-corpus, concurrent-indexing and GPU performance gates remain independent.
|
||||||
|
## Worker round-trip diagnostics
|
||||||
|
|
||||||
|
Measured desktop failures return `{message, metrics}` and retain the trace for
|
||||||
|
the failed attempt. Busy retries are separate attempts. If an unexpected worker
|
||||||
|
task failure loses its collector, elapsed time is retained, the trace is null,
|
||||||
|
and a dropped-span marker makes the missing detail explicit. Error messages are
|
||||||
|
excluded from the exported timing report.
|
||||||
|
|
||||||
|
Embedding IPC spans expose named `*_us` integer counters for request encoding,
|
||||||
|
writing, worker wait/response decode, validation, and native tokenization, batch
|
||||||
|
preparation, inference and pooling/normalization when genuinely reported. They
|
||||||
|
also expose `worker_token_count` and the GPU-to-CPU restart count. Microsecond
|
||||||
|
counters are durations, not occurrence counts; their suffix records units.
|
||||||
|
|
||||||
|
The native worker clock is not aligned with Rust's collector origin. Native
|
||||||
|
stage values remain counters on the IPC span rather than invented positioned
|
||||||
|
child spans. Worker wait includes response-reader JSON parsing and scheduling;
|
||||||
|
it is not pure inference time. The encompassing round trip and its component
|
||||||
|
counters overlap. A fallback/restart may include setup outside the last worker's
|
||||||
|
reported neural pass; retain that distinction when diagnosing slow requests.
|
||||||
|
|
||||||
|
## Small instrumentation-overhead diagnostic
|
||||||
|
|
||||||
|
The deterministic example creates 1,000 local text files, builds/prewarms SQLite,
|
||||||
|
then alternates basic/debug order over 24 exact filename queries and 20 repeats.
|
||||||
|
Both modes must retrieve the correct file. It records 480 samples per mode,
|
||||||
|
outer-call p50/p95/mean and actual debug span counts, excluding fixture setup
|
||||||
|
and JSON serialization. It runs no model, media decoder or network.
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cargo run --release -p searchi-core --example trace_overhead -- .searchi/trace-overhead.json
|
||||||
|
```
|
||||||
|
|
||||||
|
This tiny warm lexical corpus measures diagnostic overhead under the current
|
||||||
|
VM conditions. It does not certify production search latency or model-query
|
||||||
|
tracing overhead. Keep the raw report and profile/device/concurrency caveats;
|
||||||
|
repeat on representative corpora and compare instrumentation policies before
|
||||||
|
claiming low overhead across every retrieval mode.
|
||||||
|
|
||||||
|
Windows CPU release diagnostic, 2026-10-12 (480 samples per mode):
|
||||||
|
|
||||||
|
| Mode | Mean | p50 | p95 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Basic | 0.556 ms | 0.440 ms | 0.934 ms |
|
||||||
|
| Detailed | 0.968 ms | 0.819 ms | 1.615 ms |
|
||||||
|
|
||||||
|
The detailed searches retained 14 spans each. These measurements include the
|
||||||
|
search itself and metrics assembly, and exclude fixture setup and JSON encoding.
|
||||||
|
Other work in the VM could contend for CPU. [Raw samples and measurement scope](evidence/windows-cpu-trace-overhead.json).
|
||||||
|
The recorded Windows x86-64 release run retained 480 raw samples per mode and
|
||||||
|
14 actual debug spans per request:
|
||||||
|
|
||||||
|
| Mode | Mean | p50 | p95 |
|
||||||
|
| --- | ---: | ---: | ---: |
|
||||||
|
| Basic | 0.556 ms | 0.440 ms | 0.934 ms |
|
||||||
|
| Debug | 0.968 ms | 0.819 ms | 1.615 ms |
|
||||||
|
|
||||||
|
[Raw samples and measurement contract](evidence/windows-cpu-trace-overhead.json)
|
||||||
|
include the active VM-contention caveat: native photo inference and other work
|
||||||
|
may overlap. This is an absolute small-corpus comparison, not an old-baseline
|
||||||
|
regression claim or proof of model/media tracing overhead. The extra cost
|
||||||
|
supports keeping detailed mode optional rather than claiming it is free.
|
||||||
@@ -38,10 +38,21 @@ Status: implementation started; no production release yet. Windows VM tests use
|
|||||||
- [ ] Watchers and reconciliation across Windows, macOS and Linux.
|
- [ ] Watchers and reconciliation across Windows, macOS and Linux.
|
||||||
|
|
||||||
## 4. Media
|
## 4. Media
|
||||||
|
- [x] Genuine Google image, audio and video embeddings integrated with the shared
|
||||||
|
search service and CPU-tested FFmpeg decoding; music tags and duration filters.
|
||||||
|
- [x] Frozen independent media corpus with retrieval, timestamp, negative-query
|
||||||
|
and indexing-cost measurements (see MEDIA-QUALITY.md).
|
||||||
|
- [ ] Improve precise spoken-passage localization, music hard negatives and
|
||||||
|
calibrated handling of queries without a relevant match.
|
||||||
|
- [ ] Package and verify decoders on every release OS; test production acceleration.
|
||||||
- [ ] CPU OCR, image retrieval, audio transcription, video segment locations.
|
- [ ] CPU OCR, image retrieval, audio transcription, video segment locations.
|
||||||
- [ ] Isolated extraction workers with timeouts and resource ceilings.
|
- [ ] Isolated extraction workers with timeouts and resource ceilings.
|
||||||
|
|
||||||
## 5. Release gates
|
## 5. Release gates
|
||||||
|
- [x] Shared monotonic Basic/Debug search metrics, granular bounded engine traces,
|
||||||
|
actual worker query diagnostics, and headless JSON performance output.
|
||||||
|
- [ ] Verify detailed performance UI interaction on native desktop platforms;
|
||||||
|
repeat overhead measurements on representative libraries and hardware.
|
||||||
- [ ] Formatting, lint, tests and security/dependency review.
|
- [ ] Formatting, lint, tests and security/dependency review.
|
||||||
- [ ] CPU benchmarks: filename p95 <50 ms/1M records; text <150 ms/100K documents; semantic <500 ms on declared hardware.
|
- [ ] CPU benchmarks: filename p95 <50 ms/1M records; text <150 ms/100K documents; semantic <500 ms on declared hardware.
|
||||||
- [ ] Measure cold start, indexing throughput, RAM, disk and idle CPU.
|
- [ ] Measure cold start, indexing throughput, RAM, disk and idle CPU.
|
||||||
|
|||||||
@@ -38,9 +38,7 @@ on other operating systems remain independent release checks.
|
|||||||
| embeddinggemma-2-Q8_0.gguf | 309,855,456 | 2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135 |
|
| embeddinggemma-2-Q8_0.gguf | 309,855,456 | 2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135 |
|
||||||
| mmproj-embeddinggemma-2-Q8_0.gguf | 554,821,024 | c4a8a52691ecef40618438928bdf9e68379b854e24166f292592353db0aab64f |
|
| mmproj-embeddinggemma-2-Q8_0.gguf | 554,821,024 | c4a8a52691ecef40618438928bdf9e68379b854e24166f292592353db0aab64f |
|
||||||
|
|
||||||
Setup downloads both pinned files. **The current worker implements text**;
|
Setup downloads both pinned files. The native worker implements genuine text, image pixels, and mono 16 kHz audio embeddings through the verified Google projector in the same 768-dimensional query space. The shared service now persists unified document, image, video-frame and soundtrack/audio/music vectors with source locations and metadata. Scoped desktop media previews and an independent retrieval evaluation are implemented; precise localization, broader quality, native playback QA and releases remain gates.
|
||||||
image/audio projection loading and image/audio/video embeddings remain required
|
|
||||||
work. Downloaded media weights do not establish functioning multimodal search.
|
|
||||||
|
|
||||||
Weights use Apache-2.0; llama.cpp and DigUp use MIT; USearch and NumKong use
|
Weights use Apache-2.0; llama.cpp and DigUp use MIT; USearch and NumKong use
|
||||||
Apache-2.0. Retain notices with distributions. Five DigUp text reference vectors
|
Apache-2.0. Retain notices with distributions. Five DigUp text reference vectors
|
||||||
@@ -64,14 +62,15 @@ HybridSearch is the shared CLI/desktop service:
|
|||||||
HybridSearch::open(db, model_dir, worker_executable, threads)
|
HybridSearch::open(db, model_dir, worker_executable, threads)
|
||||||
HybridSearch::embedding_status(db) // No model/runtime load or network
|
HybridSearch::embedding_status(db) // No model/runtime load or network
|
||||||
service.warmup() // Real query inference + ANN cache
|
service.warmup() // Real query inference + ANN cache
|
||||||
service.index_embeddings()
|
service.index_embeddings() // Explicit document-only generation
|
||||||
|
service.index_embeddings_with_media(&tools, &MediaLimits::default()) // Unified generation
|
||||||
service.search(query, limit, &SearchOptions)
|
service.search(query, limit, &SearchOptions)
|
||||||
service.search_semantic(query, limit, &SearchOptions) // Diagnostics
|
service.search_semantic(query, limit, &SearchOptions) // Diagnostics
|
||||||
service.worker_info() // Actual model/runtime/device/threads
|
service.worker_info() // Actual model/runtime/device/threads
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
Threads are bounded to 1–8; VM tests use one. Document batches contain at most
|
Threads are bounded to 1–8; VM tests use one. Document batches contain at most
|
||||||
eight windows. Request/response budgets are 2/4 MiB, startup has a 120-second
|
eight windows. Request/response budgets are 16/4 MiB, startup has a 120-second
|
||||||
deadline, and each request has a 180-second deadline. Malformed or interrupted
|
deadline, and each request has a 180-second deadline. Malformed or interrupted
|
||||||
IPC and invalid embeddings fail explicitly.
|
IPC and invalid embeddings fail explicitly.
|
||||||
|
|
||||||
@@ -177,3 +176,92 @@ The ignored ANN test deliberately requires release mode. Default tests require
|
|||||||
no model/network and cover protocol bounds, invalid vectors, UTF-8 coverage,
|
no model/network and cover protocol bounds, invalid vectors, UTF-8 coverage,
|
||||||
fusion, lifecycle, checksums, pre-top-k filtering, and dirty-path caching.
|
fusion, lifecycle, checksums, pre-top-k filtering, and dirty-path caching.
|
||||||
Linux/macOS and GPU checks are not inferred from this Windows CPU run.
|
Linux/macOS and GPU checks are not inferred from this Windows CPU run.
|
||||||
|
|
||||||
|
## Genuine image and audio adapter
|
||||||
|
|
||||||
|
`GemmaEmbedder::embed_images(&[&Path])` uses a provisional 280-token image
|
||||||
|
budget. `embed_images_with_budget(paths, budget)` supports 70 through 1120
|
||||||
|
Google visual tokens, including full-resolution model input. Use
|
||||||
|
`embed_images_with_metadata` for encoded dimensions and the chosen budget.
|
||||||
|
The aspect-preserving dimensions are multiples of 48; Rust performs bounded
|
||||||
|
PNG/JPEG/WebP/BMP decoding, applies EXIF orientation, and bicubic Catmull-Rom
|
||||||
|
resizing before sending RGB pixels. No captions or filenames produce vectors.
|
||||||
|
Bounds are eight paths per call, one image per inference operation, 64 MiB
|
||||||
|
input files, 64 Mi-pixel JPEG sources (scaled decoding above 16 Mi pixels),
|
||||||
|
16 Mi-pixel non-JPEG sources, and a 256 MiB decoding-buffer/output ceiling. Decoder dimensions
|
||||||
|
and output size are checked before pixel allocation. Tiny/extreme images still
|
||||||
|
receive a valid bounded grid. The same API embeds decoded video frame files.
|
||||||
|
|
||||||
|
`GemmaEmbedder::embed_audio_pcm(samples)` accepts normalized finite mono 16 kHz
|
||||||
|
PCM with 1–480,000 samples, at most 30 seconds, on a little-endian host. It
|
||||||
|
encodes actual sound through Google MTMD and the shared mean-pooled text model.
|
||||||
|
Callers must segment the entire recording/movie soundtrack and retain actual
|
||||||
|
window timestamps; this API never silently truncates audio. It provides no
|
||||||
|
transcript. Speech, music, and environmental sounds require independent quality
|
||||||
|
evaluation even though all use this genuine encoder.
|
||||||
|
|
||||||
|
The projector is size/SHA256 verified on first media use, then loaded lazily in
|
||||||
|
the retained native worker. Model initialization and returned inference failures
|
||||||
|
share the CPU fallback policy. Native video/FFmpeg spawning is disabled; the
|
||||||
|
media adapter owns extraction. Media generations must include
|
||||||
|
`GEMMA_MEDIA_FINGERPRINT` and image budget/window policy in their identity.
|
||||||
|
Text fingerprints remain unchanged. GPU media inference is untested here.
|
||||||
|
|
||||||
|
Actual Windows CPU parity with copied MIT DigUp fixtures passed for screenshot
|
||||||
|
280, photo 140, frame 70, and speech: cosine respectively **0.99994194,
|
||||||
|
0.99974583, 0.99921928, and 0.99983318**, all exceeding the 0.999 gate. The test
|
||||||
|
completed in 58.79 seconds including fresh model/projector startup. This proves
|
||||||
|
runtime parity on those inputs, not broad image/audio retrieval quality.
|
||||||
|
Run with the model/worker environment above:
|
||||||
|
|
||||||
|
~~~powershell
|
||||||
|
cargo test -p searchi-semantic --test gemma_media_cpu -- --ignored --nocapture
|
||||||
|
~~~
|
||||||
|
The large camera JPEG policy, established decoder tradeoffs, EXIF/resource test,
|
||||||
|
and memory-bound caveats are documented in [the semantic crate](../crates/searchi-semantic/README.md).
|
||||||
|
|
||||||
|
## Unified media indexing and setup
|
||||||
|
|
||||||
|
The desktop and default CLI `embed` command use `index_embeddings_with_media`.
|
||||||
|
Document-only indexing is an explicit `--text-only` choice. `searchi-media`
|
||||||
|
prepares actual image files, movie frames and mono 16 kHz audio windows with
|
||||||
|
verified caller-supplied FFmpeg/FFprobe executables. There is no transcription,
|
||||||
|
fabricated caption or model-derived filename substitution. Existing stream/file
|
||||||
|
artist/title metadata enters guarded keyword indexing.
|
||||||
|
|
||||||
|
Default media policy uses image 280/frame 70 visual tokens, 30-second primary
|
||||||
|
audio windows with two-second overlap, up to 240 distributed frames plus 120
|
||||||
|
scene candidates, up to 600 audio windows, a four-hour/16 GiB file ceiling and
|
||||||
|
45-second subprocess deadlines. Coverage includes the audio tail or reports an
|
||||||
|
explicit resource error; video sampling spans the movie without guaranteeing
|
||||||
|
every transient scene. Media files have stable content identity, persisted
|
||||||
|
vectors and sources (`image`, `timestamp_ms`); source changes/deletions suppress
|
||||||
|
stale results. Generations include preprocessing/window/sampling fingerprints.
|
||||||
|
Per-file decoding failures are reported and persisted rather than silently
|
||||||
|
counted as successful media indexing. Other filename/text matches stay useful.
|
||||||
|
|
||||||
|
Developer preparation from the repository root:
|
||||||
|
|
||||||
|
~~~powershell
|
||||||
|
python tools/embeddinggemma2/build_runtime.py --backend cpu --jobs 2
|
||||||
|
python tools/embeddinggemma2/stage_runtime.py --backend cpu
|
||||||
|
python tools/media-runtime/provision.py --stage
|
||||||
|
$env:SEARCHI_FORCE_CPU = '1'
|
||||||
|
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = '--disable-gpu --disable-accelerated-video-decode'
|
||||||
|
~~~
|
||||||
|
|
||||||
|
The decoder script verifies a pinned FFmpeg 9.0.2 LGPL artifact and stages tools,
|
||||||
|
notices and the manifest without global installation. Only Windows x86-64 has
|
||||||
|
been executed; Linux/ARM64 entries require native checks, and macOS decoder
|
||||||
|
packaging is still missing. End-user packages must include the worker/decoders;
|
||||||
|
Google weights are provisioned automatically and verified locally. Search does
|
||||||
|
not perform setup downloads. Headless CLI helper discovery and explicit paths
|
||||||
|
are documented in [README](../README.md) and [BUILDING](BUILDING.md).
|
||||||
|
|
||||||
|
[Independent media evaluation](MEDIA-QUALITY.md) preserves the frozen corpus,
|
||||||
|
actual production pipeline, direct-harness failure, music confusions, speech
|
||||||
|
point-localization failures and unsuppressed no-answer results. Strong file
|
||||||
|
recall is insufficient for accurate spoken-topic navigation. A ten-second audio
|
||||||
|
experiment reduces some distances but does not close that gate; it is not a
|
||||||
|
new default. Quality policies must be validated on held-out data rather than
|
||||||
|
inventing thresholds from the evaluation corpus.
|
||||||
@@ -1,64 +1,94 @@
|
|||||||
# Validation evidence
|
# Validation evidence
|
||||||
|
|
||||||
Development host: Windows VM, x86_64 MSVC, Rust stable 1.99.0.
|
Development host: Windows VM, x86_64 MSVC, stable Rust 1.99.0. All local
|
||||||
All local tests run on CPU; no GPU provider is configured.
|
inference, FFmpeg decoding and WebView tests use CPU only. UI launches set
|
||||||
|
`WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS='--disable-gpu --disable-accelerated-video-decode'`
|
||||||
|
and `SEARCHI_FORCE_CPU=1`.
|
||||||
|
|
||||||
## Current integration checkpoint (2026-10-12)
|
## Current integration checkpoint (2026-10-12)
|
||||||
|
|
||||||
- `cargo test --workspace --locked`: 41 default tests passed after final core integration. Four asset-dependent
|
- Native desktop: **9/9 tests passed**, including structured trace/basic responses.
|
||||||
or explicit benchmark tests remain opt-in; a default pass does not run them.
|
Post-generation cached query warmup is implemented but its interactive first-query gate remains open.
|
||||||
- Native desktop tests: 7 passed with SEARCHI_FORCE_CPU=1 and WebView2 GPU disabled.
|
Frontend: **18/18 tests passed**, TypeScript/Vite production build and
|
||||||
- Frontend: 7 async-state tests passed, Prettier passed, TypeScript/Vite build passed.
|
formatting passed. Preview policy tests cover fractional timestamp seeking,
|
||||||
- Genuine Google CPU reference/window test passed separately: five DigUp reference
|
image labels, one active/latest pending request, stale replies and Escape.
|
||||||
vectors had cosine similarity above 0.999; actual multilingual retrieval and
|
- Semantic adapter: **8 default tests passed**; the explicit 48 MP resource test
|
||||||
tokenizer tail-window coverage passed. MiniLM evidence below is historical.
|
remains opt-in. Strict targeted all-target Clippy and formatting passed.
|
||||||
- Repository description/topics were read back after update; the branded avatar
|
Protocol bounds, invalid vectors, artifact size/checksum mismatch, old-worker
|
||||||
upload returned HTTP 204 and the repository API returned its avatar URL.
|
capability defaults and oversized JPEG rejection are covered.
|
||||||
- New desktop styling is in progress. `design-qa.md` records the visual gate as
|
- The CPU native helper rebuilt and staged successfully with genuine Google
|
||||||
blocked because browser bootstrap and native screen capture are unavailable.
|
MTMD image/audio support. Its handshake reports `media_supported:true`;
|
||||||
No visual match or complete UI interaction verification is claimed.
|
older text workers default false rather than claiming media readiness.
|
||||||
|
- Actual Google CPU media parity passed separately in **52.85 seconds**:
|
||||||
|
screenshot 280 cosine 0.99994194, photo 140: 0.99974583, frame 70: 0.99921928 and
|
||||||
|
speech: 0.99983318 against pinned MIT DigUp vectors. All exceed the 0.999 gate.
|
||||||
|
This verifies runtime parity, not broad retrieval quality.
|
||||||
|
- The release resource test encodes/decodes an actual 8000x6000 JPEG with EXIF
|
||||||
|
rotation and verifies region colors after scaled decoding/resizing. Decode
|
||||||
|
plus resize took 176–220 ms. A sampled whole-test peak was 146.86 MiB, including
|
||||||
|
source fixture generation; it is not decoder-only RSS or neural quality.
|
||||||
|
- The independent production media pipeline has run on 20 real assets/65 frozen
|
||||||
|
queries using actual Google inference, FFmpeg, metadata, persistence and HNSW.
|
||||||
|
It encoded 20/20 files with no failures in the recorded 30-second-window run.
|
||||||
|
Broad file recall is promising; precise speech timestamps, music confusion
|
||||||
|
and no-answer behavior remain failed quality gates. Preserve the earlier
|
||||||
|
direct harness failure separately. [Raw results and limitations](MEDIA-QUALITY.md).
|
||||||
|
- Actual document seed regression passed 24/24 semantic/hybrid queries. This is
|
||||||
|
a small synthetic regression, not a production relevance corpus.
|
||||||
|
- The explicit 100,000-vector release ANN diagnostic passed 98.5% recall@10,
|
||||||
|
HNSW p95 1.37 ms and warm service p95 6.40 ms with zero warm passage hashes.
|
||||||
|
Vectors are synthetic index benchmark data, never production embeddings.
|
||||||
|
- The fixed one-million-file first-input report records 40 inputs below 50 ms in
|
||||||
|
that run: [recorded benchmark](../crates/searchi-core/benchmarks/results/windows-cpu-full-candidate-first-final.json).
|
||||||
|
These measurements do not establish every query or concurrent-indexing goal.
|
||||||
|
|
||||||
Final UI integration: the branded compact launcher, local Inter/Fluent assets,
|
The integrated CPU workspace checkpoint passed **62 default tests** (8 assets, 6 CLI, 23 core, 4 media, 8 semantic, 13 service), with **10 explicit opt-in tests ignored**. Workspace and native all-target Clippy passed with warnings denied. The later semantic CPU-restart counter change passed workspace Clippy and the actual CPU query diagnostics test. Earlier aggregate counts are historical.
|
||||||
themes and motion implementation passed frontend build, formatting, and seven
|
|
||||||
state tests. Native launch succeeded; accessibility text showed real controls
|
|
||||||
and a warmed Google CPU worker. UI input and screenshots remain blocked by
|
|
||||||
Windows access/capture errors, as recorded in design-qa.md. Workspace formatting
|
|
||||||
and strict all-target Clippy passed. The fixed full-corpus first-input report
|
|
||||||
is in crates/searchi-core/benchmarks/results/windows-cpu-full-candidate-first-final.json;
|
|
||||||
all 40 measured filename inputs were below 50 ms in that run.
|
|
||||||
|
|
||||||
The active parallel development branches of work may add tests or change counts
|
The updated pinned C++ helper compiled in an independent CPU build directory and
|
||||||
after this checkpoint. Run the documented checks again after integration.
|
was staged for desktop resources without interrupting media evaluation. Actual
|
||||||
|
Google query diagnostics passed separately in 18.09 seconds including model
|
||||||
|
initialization and two queries: 13 tokens, tokenization 149 us, batch preparation
|
||||||
|
22 us, inference 112914 us, pooling/normalization 17 us. Rust worker wait/response
|
||||||
|
decode measured 113960 us. Debug/normal embedding cosine was 0.99999999. These
|
||||||
|
are one diagnostic sample under VM contention, not production latency evidence.
|
||||||
|
See [performance boundaries and raw overhead samples](PERFORMANCE.md).
|
||||||
|
|
||||||
## Historical foundation evidence
|
## Native UI and platform limits
|
||||||
|
|
||||||
Initial foundation checks (2026-10-12):
|
The branded launcher launched natively. Accessibility inspection showed real
|
||||||
- `cargo test --workspace`: CLI parser/executable checks and engine retrieval,
|
controls and a warmed Google CPU worker. Native screenshots, input automation
|
||||||
lifecycle, exclusions, filters, migration and concurrent-reader checks passed.
|
and complete image/audio/video playback interactions remain blocked by Windows
|
||||||
- `cargo clippy --workspace --all-targets -- -D warnings`: passed.
|
access/capture errors recorded in [design QA](../design-qa.md). Passing pure
|
||||||
- Rust formatting applied; final formatting verification follows integration.
|
preview policy tests does not establish real WebView codec playback or visual
|
||||||
- Desktop TypeScript/Vite build passed; npm audit reported zero vulnerabilities.
|
fidelity. Default-application opening is not exact timestamp navigation.
|
||||||
- Initial Windows desktop native check, embedded app build, strict Clippy and
|
|
||||||
two desktop tests passed (index gate recovery and guarded source opening).
|
|
||||||
- Desktop launched with WebView2 `--disable-gpu`. UI automation could not capture
|
|
||||||
the window: `GetCursorPos` access denied, then monitor capture error
|
|
||||||
`0x80070057`. Visual layout and end-to-end UI interactions remain unverified.
|
|
||||||
- Real Python CPU ONNX embedding proof completed on 8 documents / 6 queries:
|
|
||||||
5/6 top-1 accuracy, 6/6 recall@3. The strict top-1 quality gate is not met.
|
|
||||||
See `tools/semantic-proof/windows-cpu-result.json` for raw timings and rankings.
|
|
||||||
This tiny fixture is not evidence of large-corpus performance.
|
|
||||||
- Native Rust CPU embedding adapter compiled, passed its three unit tests and
|
|
||||||
the explicit local-model smoke test. The smoke reproduced the same 5/6 top-1
|
|
||||||
limitation. Default CI omits this asset-dependent test; it must be executed
|
|
||||||
separately with provisioned local artifacts.
|
|
||||||
- Integrated workspace validation: 19 default tests passed; 1 local-model test
|
|
||||||
is opt-in. Rust formatting and strict Clippy passed after lockfile refresh.
|
|
||||||
- PDF/DOCX fixture tests include matching source locations, corrupt replacement,
|
|
||||||
encrypted PDF behavior and compressed-input limits.
|
|
||||||
|
|
||||||
Not yet established: production latency targets, semantic inference, desktop
|
Windows x86-64 verified decoder provisioning/staging has executed. Manifest
|
||||||
runtime behavior, Linux/macOS execution, installer signing or upgrades.
|
entries for Linux and ARM64 are configuration, not evidence of native execution.
|
||||||
CI configuration is source configuration, not evidence that remote jobs ran.
|
A vetted macOS decoder package remains missing. Linux/macOS worker/desktop
|
||||||
Gitea workflows require matching Windows, Linux and macOS runners; runner
|
execution, production GPU parity/fallback, installer signing, upgrade/rollback,
|
||||||
availability has not been established. GitHub workflows are also supplied for
|
licenses/source distribution, large libraries and calibrated absence handling
|
||||||
future mirroring, but do not run merely because code was pushed to Gitea.
|
remain unexecuted release gates. FFmpeg subprocess memory is not sandbox-capped;
|
||||||
|
bounded outputs/timeouts are not an absolute RSS guarantee.
|
||||||
|
|
||||||
|
Gitea needs matching registered runners. GitHub workflows are supplied for
|
||||||
|
mirroring; they do not run merely because code is pushed to Gitea. The current
|
||||||
|
workflow checks do not automatically certify downloaded Google assets or the
|
||||||
|
separately built worker. CI configuration is not remote job evidence.
|
||||||
|
|
||||||
|
## Historical evidence
|
||||||
|
|
||||||
|
Initial engine/CLI checks covered retrieval, persistence, filesystem changes,
|
||||||
|
exclusions/filters, migration, concurrent readers, PDF pages, DOCX paragraphs,
|
||||||
|
corrupt replacement, encrypted PDF errors and bounded compressed input.
|
||||||
|
The first actual Google text adapter matched five pinned reference vectors
|
||||||
|
above 0.999, including multilingual queries and long-document window coverage.
|
||||||
|
|
||||||
|
The rejected MiniLM/ONNX experiment achieved 5/6 top-1 on its six-query fixture;
|
||||||
|
it is retained under the opt-in historical proof only and is not a production
|
||||||
|
fallback. [Historical experiment](../tools/semantic-proof/HISTORICAL.md).
|
||||||
|
Repository metadata/avatar updates were read back successfully. Historical
|
||||||
|
foundation/default test totals changed as implementation expanded and must not
|
||||||
|
be presented as the current aggregate.
|
||||||
|
|
||||||
|
Reproduction commands and exact interfaces are in [BUILDING.md](BUILDING.md),
|
||||||
|
[SEMANTIC.md](SEMANTIC.md), and [MEDIA-QUALITY.md](MEDIA-QUALITY.md).
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-10-12",
|
||||||
|
"model": "google/embeddinggemma-2",
|
||||||
|
"runtime": "llama.cpp b11461 with pinned no-logits patch",
|
||||||
|
"device": "cpu",
|
||||||
|
"threads": 1,
|
||||||
|
"test": "google_cpu_debug_query_reports_actual_worker_stages_without_changing_vectors",
|
||||||
|
"test_elapsed_seconds_including_initialization_and_two_queries": 18.09,
|
||||||
|
"debug_normal_cosine": 0.99999999,
|
||||||
|
"diagnostics": {
|
||||||
|
"request_encode_us": 24,
|
||||||
|
"request_write_us": 52,
|
||||||
|
"response_validate_us": 92,
|
||||||
|
"worker_batch_prepare_us": 22,
|
||||||
|
"worker_inference_us": 112914,
|
||||||
|
"worker_pool_normalize_us": 17,
|
||||||
|
"worker_token_count": 13,
|
||||||
|
"worker_tokenization_us": 149,
|
||||||
|
"worker_wait_and_response_decode_us": 113960
|
||||||
|
},
|
||||||
|
"scope": "One actual query diagnostic; native worker clocks are not aligned to Rust. VM had concurrent media inference. No production latency or broad overhead claim."
|
||||||
|
}
|
||||||
@@ -0,0 +1,990 @@
|
|||||||
|
{
|
||||||
|
"arch": "x86_64",
|
||||||
|
"basic": {
|
||||||
|
"mean_ms": 0.5563772916666668,
|
||||||
|
"p50_ms": 0.4403,
|
||||||
|
"p95_ms": 0.9344,
|
||||||
|
"samples": 480
|
||||||
|
},
|
||||||
|
"basic_samples_ms": [
|
||||||
|
0.434,
|
||||||
|
0.4441,
|
||||||
|
0.5638000000000001,
|
||||||
|
0.41350000000000003,
|
||||||
|
0.4078,
|
||||||
|
0.44470000000000004,
|
||||||
|
0.4078,
|
||||||
|
0.5291,
|
||||||
|
0.416,
|
||||||
|
0.4223,
|
||||||
|
0.43360000000000004,
|
||||||
|
0.4181,
|
||||||
|
0.5680999999999999,
|
||||||
|
0.44630000000000003,
|
||||||
|
0.4105,
|
||||||
|
0.3995,
|
||||||
|
0.43829999999999997,
|
||||||
|
0.4223,
|
||||||
|
0.4192,
|
||||||
|
0.4153,
|
||||||
|
0.5199999999999999,
|
||||||
|
0.4115,
|
||||||
|
0.482,
|
||||||
|
0.5616,
|
||||||
|
0.417,
|
||||||
|
0.4108,
|
||||||
|
0.412,
|
||||||
|
0.3987,
|
||||||
|
0.4224,
|
||||||
|
0.5749000000000001,
|
||||||
|
0.39780000000000004,
|
||||||
|
0.40240000000000004,
|
||||||
|
0.4188,
|
||||||
|
0.39809999999999995,
|
||||||
|
0.3923,
|
||||||
|
0.532,
|
||||||
|
0.44880000000000003,
|
||||||
|
0.5005,
|
||||||
|
0.4576,
|
||||||
|
0.4341,
|
||||||
|
0.5256,
|
||||||
|
0.4138,
|
||||||
|
0.5062000000000001,
|
||||||
|
0.4152,
|
||||||
|
0.3983,
|
||||||
|
0.5354,
|
||||||
|
0.4163,
|
||||||
|
0.5561999999999999,
|
||||||
|
0.7023,
|
||||||
|
2.0184,
|
||||||
|
0.483,
|
||||||
|
0.5897,
|
||||||
|
0.5286,
|
||||||
|
0.4496,
|
||||||
|
0.41750000000000004,
|
||||||
|
2.5051,
|
||||||
|
0.4249,
|
||||||
|
0.8365999999999999,
|
||||||
|
0.5505,
|
||||||
|
0.4127,
|
||||||
|
0.5488999999999999,
|
||||||
|
0.47009999999999996,
|
||||||
|
0.6473,
|
||||||
|
0.4203,
|
||||||
|
0.6688,
|
||||||
|
0.4081,
|
||||||
|
0.46749999999999997,
|
||||||
|
0.4396,
|
||||||
|
0.4638,
|
||||||
|
0.4878,
|
||||||
|
0.5753,
|
||||||
|
0.8668,
|
||||||
|
0.4817,
|
||||||
|
0.41829999999999995,
|
||||||
|
0.4106,
|
||||||
|
0.482,
|
||||||
|
0.4837,
|
||||||
|
0.3935,
|
||||||
|
0.5321,
|
||||||
|
0.42860000000000004,
|
||||||
|
0.41859999999999997,
|
||||||
|
0.3914,
|
||||||
|
0.3881,
|
||||||
|
0.522,
|
||||||
|
0.4021,
|
||||||
|
0.3938,
|
||||||
|
0.3918,
|
||||||
|
0.38739999999999997,
|
||||||
|
0.5252,
|
||||||
|
0.39740000000000003,
|
||||||
|
0.3877,
|
||||||
|
0.421,
|
||||||
|
0.40850000000000003,
|
||||||
|
0.5257000000000001,
|
||||||
|
0.40299999999999997,
|
||||||
|
0.4779,
|
||||||
|
1.9503000000000001,
|
||||||
|
0.416,
|
||||||
|
0.4799,
|
||||||
|
0.5547,
|
||||||
|
0.41209999999999997,
|
||||||
|
0.4281,
|
||||||
|
0.6578,
|
||||||
|
0.4112,
|
||||||
|
0.704,
|
||||||
|
0.5748,
|
||||||
|
0.4073,
|
||||||
|
3.9541999999999997,
|
||||||
|
0.5597,
|
||||||
|
0.6794,
|
||||||
|
0.46690000000000004,
|
||||||
|
0.4059,
|
||||||
|
0.5046999999999999,
|
||||||
|
0.4214,
|
||||||
|
0.5058,
|
||||||
|
0.4331,
|
||||||
|
0.4039,
|
||||||
|
0.41100000000000003,
|
||||||
|
2.5677,
|
||||||
|
0.4401,
|
||||||
|
0.4058,
|
||||||
|
0.4004,
|
||||||
|
0.4566,
|
||||||
|
0.4249,
|
||||||
|
0.5615,
|
||||||
|
0.3956,
|
||||||
|
0.6221,
|
||||||
|
0.53,
|
||||||
|
0.4347,
|
||||||
|
0.5176999999999999,
|
||||||
|
0.3959,
|
||||||
|
0.4193,
|
||||||
|
0.627,
|
||||||
|
0.42719999999999997,
|
||||||
|
1.1775,
|
||||||
|
0.6066999999999999,
|
||||||
|
0.5718,
|
||||||
|
0.45,
|
||||||
|
0.4439,
|
||||||
|
0.5364,
|
||||||
|
0.5308,
|
||||||
|
0.4416,
|
||||||
|
0.5453,
|
||||||
|
0.4424,
|
||||||
|
0.5058,
|
||||||
|
0.4075,
|
||||||
|
0.46099999999999997,
|
||||||
|
0.5885,
|
||||||
|
0.5032,
|
||||||
|
1.3806999999999998,
|
||||||
|
0.4668,
|
||||||
|
0.5145,
|
||||||
|
0.40959999999999996,
|
||||||
|
0.5816,
|
||||||
|
0.4249,
|
||||||
|
0.3986,
|
||||||
|
0.6983999999999999,
|
||||||
|
2.6059,
|
||||||
|
0.4276,
|
||||||
|
0.5638000000000001,
|
||||||
|
0.5913,
|
||||||
|
0.4168,
|
||||||
|
0.4255,
|
||||||
|
1.4086999999999998,
|
||||||
|
1.5349000000000002,
|
||||||
|
0.4954,
|
||||||
|
0.5483,
|
||||||
|
0.41350000000000003,
|
||||||
|
0.42450000000000004,
|
||||||
|
1.1743999999999999,
|
||||||
|
0.6155,
|
||||||
|
0.48319999999999996,
|
||||||
|
0.6431,
|
||||||
|
0.4142,
|
||||||
|
0.43370000000000003,
|
||||||
|
0.5195,
|
||||||
|
0.3939,
|
||||||
|
0.41609999999999997,
|
||||||
|
0.41619999999999996,
|
||||||
|
0.38949999999999996,
|
||||||
|
1.9449999999999998,
|
||||||
|
0.5754,
|
||||||
|
0.4729,
|
||||||
|
0.5639,
|
||||||
|
0.40719999999999995,
|
||||||
|
1.8393000000000002,
|
||||||
|
0.46490000000000004,
|
||||||
|
0.5271,
|
||||||
|
0.4041,
|
||||||
|
0.5678000000000001,
|
||||||
|
0.5701,
|
||||||
|
0.4301,
|
||||||
|
0.9139,
|
||||||
|
0.4016,
|
||||||
|
0.4486,
|
||||||
|
0.41659999999999997,
|
||||||
|
0.6462,
|
||||||
|
0.7081999999999999,
|
||||||
|
0.6969,
|
||||||
|
0.6972,
|
||||||
|
0.5947,
|
||||||
|
0.4736,
|
||||||
|
0.6512,
|
||||||
|
0.4312,
|
||||||
|
0.4189,
|
||||||
|
0.5238,
|
||||||
|
0.39130000000000004,
|
||||||
|
0.4522,
|
||||||
|
0.4057,
|
||||||
|
0.5138,
|
||||||
|
0.8949,
|
||||||
|
0.9114,
|
||||||
|
0.5758,
|
||||||
|
0.5318999999999999,
|
||||||
|
0.4661,
|
||||||
|
0.4411,
|
||||||
|
1.6324,
|
||||||
|
0.4342,
|
||||||
|
2.5791999999999997,
|
||||||
|
0.4112,
|
||||||
|
0.4033,
|
||||||
|
3.1165,
|
||||||
|
0.424,
|
||||||
|
0.5136000000000001,
|
||||||
|
0.40549999999999997,
|
||||||
|
0.631,
|
||||||
|
0.4153,
|
||||||
|
2.9606,
|
||||||
|
0.4458,
|
||||||
|
0.4709,
|
||||||
|
0.4316,
|
||||||
|
0.4279,
|
||||||
|
0.45790000000000003,
|
||||||
|
0.4303,
|
||||||
|
0.397,
|
||||||
|
1.674,
|
||||||
|
1.2691,
|
||||||
|
0.7015,
|
||||||
|
0.5775,
|
||||||
|
0.5308,
|
||||||
|
0.4457,
|
||||||
|
0.7706,
|
||||||
|
0.7252,
|
||||||
|
0.49720000000000003,
|
||||||
|
0.5576,
|
||||||
|
0.5545,
|
||||||
|
0.4265,
|
||||||
|
0.5495,
|
||||||
|
0.40690000000000004,
|
||||||
|
0.4043,
|
||||||
|
0.3976,
|
||||||
|
0.3864,
|
||||||
|
0.3887,
|
||||||
|
0.4118,
|
||||||
|
0.7454000000000001,
|
||||||
|
0.7821,
|
||||||
|
0.7821,
|
||||||
|
0.627,
|
||||||
|
0.623,
|
||||||
|
0.41590000000000005,
|
||||||
|
0.4231,
|
||||||
|
0.45130000000000003,
|
||||||
|
0.4584,
|
||||||
|
0.5218,
|
||||||
|
0.4885,
|
||||||
|
0.38880000000000003,
|
||||||
|
0.6336,
|
||||||
|
0.4138,
|
||||||
|
0.3964,
|
||||||
|
0.5286,
|
||||||
|
0.4095,
|
||||||
|
0.40440000000000004,
|
||||||
|
0.7086,
|
||||||
|
0.5809,
|
||||||
|
0.6849,
|
||||||
|
0.4264,
|
||||||
|
0.5157,
|
||||||
|
0.4281,
|
||||||
|
0.4668,
|
||||||
|
0.6745,
|
||||||
|
0.7035,
|
||||||
|
0.7287,
|
||||||
|
0.6853,
|
||||||
|
0.415,
|
||||||
|
0.5748,
|
||||||
|
0.42319999999999997,
|
||||||
|
0.3932,
|
||||||
|
0.4022,
|
||||||
|
0.5082,
|
||||||
|
0.39599999999999996,
|
||||||
|
0.42700000000000005,
|
||||||
|
0.4045,
|
||||||
|
0.3875,
|
||||||
|
0.5247999999999999,
|
||||||
|
0.4077,
|
||||||
|
0.4099,
|
||||||
|
0.4001,
|
||||||
|
0.3945,
|
||||||
|
1.2222,
|
||||||
|
0.423,
|
||||||
|
0.4063,
|
||||||
|
0.4059,
|
||||||
|
0.40890000000000004,
|
||||||
|
0.528,
|
||||||
|
0.4066,
|
||||||
|
0.402,
|
||||||
|
0.39530000000000004,
|
||||||
|
0.3986,
|
||||||
|
0.3931,
|
||||||
|
0.48069999999999996,
|
||||||
|
0.4489,
|
||||||
|
0.4098,
|
||||||
|
0.40059999999999996,
|
||||||
|
0.5538000000000001,
|
||||||
|
0.4022,
|
||||||
|
0.4113,
|
||||||
|
0.6244,
|
||||||
|
0.40440000000000004,
|
||||||
|
0.4828,
|
||||||
|
0.40349999999999997,
|
||||||
|
0.4216,
|
||||||
|
0.41800000000000004,
|
||||||
|
0.3955,
|
||||||
|
0.3921,
|
||||||
|
0.5814,
|
||||||
|
0.4569,
|
||||||
|
0.4053,
|
||||||
|
0.47869999999999996,
|
||||||
|
0.4563,
|
||||||
|
0.5081,
|
||||||
|
0.5031,
|
||||||
|
0.44270000000000004,
|
||||||
|
0.6819999999999999,
|
||||||
|
0.44670000000000004,
|
||||||
|
0.42719999999999997,
|
||||||
|
0.5216,
|
||||||
|
0.41619999999999996,
|
||||||
|
0.4459,
|
||||||
|
0.4079,
|
||||||
|
0.5176000000000001,
|
||||||
|
0.4091,
|
||||||
|
0.4403,
|
||||||
|
0.4271,
|
||||||
|
0.399,
|
||||||
|
0.5476,
|
||||||
|
0.4369,
|
||||||
|
1.5406,
|
||||||
|
0.4229,
|
||||||
|
0.5256,
|
||||||
|
0.4348,
|
||||||
|
0.40190000000000003,
|
||||||
|
0.491,
|
||||||
|
0.6232,
|
||||||
|
0.4165,
|
||||||
|
0.5216,
|
||||||
|
0.3937,
|
||||||
|
0.41619999999999996,
|
||||||
|
0.4152,
|
||||||
|
0.40640000000000004,
|
||||||
|
0.5112,
|
||||||
|
0.381,
|
||||||
|
0.38230000000000003,
|
||||||
|
0.5358,
|
||||||
|
0.4103,
|
||||||
|
0.3923,
|
||||||
|
0.3892,
|
||||||
|
0.4074,
|
||||||
|
0.5296,
|
||||||
|
0.40900000000000003,
|
||||||
|
0.419,
|
||||||
|
0.3923,
|
||||||
|
0.4018,
|
||||||
|
0.518,
|
||||||
|
0.3943,
|
||||||
|
0.443,
|
||||||
|
0.3878,
|
||||||
|
0.3998,
|
||||||
|
0.39120000000000005,
|
||||||
|
0.4237,
|
||||||
|
0.5232,
|
||||||
|
0.3859,
|
||||||
|
0.4138,
|
||||||
|
0.4401,
|
||||||
|
0.4011,
|
||||||
|
0.39530000000000004,
|
||||||
|
0.4001,
|
||||||
|
0.3907,
|
||||||
|
0.4242,
|
||||||
|
0.5098,
|
||||||
|
0.3998,
|
||||||
|
0.5058,
|
||||||
|
0.4314,
|
||||||
|
0.388,
|
||||||
|
0.3998,
|
||||||
|
0.3986,
|
||||||
|
0.5114,
|
||||||
|
0.3856,
|
||||||
|
0.39490000000000003,
|
||||||
|
0.4326,
|
||||||
|
0.4249,
|
||||||
|
0.5303,
|
||||||
|
0.3922,
|
||||||
|
0.3905,
|
||||||
|
0.4299,
|
||||||
|
0.3973,
|
||||||
|
0.47409999999999997,
|
||||||
|
0.38939999999999997,
|
||||||
|
0.5234000000000001,
|
||||||
|
0.4118,
|
||||||
|
0.4851,
|
||||||
|
0.4848,
|
||||||
|
0.4082,
|
||||||
|
0.4038,
|
||||||
|
0.48410000000000003,
|
||||||
|
0.4568,
|
||||||
|
0.4204,
|
||||||
|
0.7379,
|
||||||
|
0.7857,
|
||||||
|
0.7318,
|
||||||
|
0.9344,
|
||||||
|
0.9292,
|
||||||
|
0.883,
|
||||||
|
0.8180000000000001,
|
||||||
|
0.7191,
|
||||||
|
1.4266999999999999,
|
||||||
|
0.43629999999999997,
|
||||||
|
0.5381,
|
||||||
|
0.4556,
|
||||||
|
0.6560999999999999,
|
||||||
|
0.5197999999999999,
|
||||||
|
0.4394,
|
||||||
|
0.46309999999999996,
|
||||||
|
0.5307000000000001,
|
||||||
|
0.42319999999999997,
|
||||||
|
0.4075,
|
||||||
|
0.3964,
|
||||||
|
0.5482,
|
||||||
|
0.3926,
|
||||||
|
0.388,
|
||||||
|
0.4299,
|
||||||
|
0.44499999999999995,
|
||||||
|
0.7485999999999999,
|
||||||
|
0.44470000000000004,
|
||||||
|
0.40759999999999996,
|
||||||
|
0.5051,
|
||||||
|
0.3889,
|
||||||
|
0.5062000000000001,
|
||||||
|
0.40700000000000003,
|
||||||
|
0.4002,
|
||||||
|
0.3934,
|
||||||
|
0.3889,
|
||||||
|
0.5202,
|
||||||
|
0.40049999999999997,
|
||||||
|
0.40650000000000003,
|
||||||
|
0.3976,
|
||||||
|
0.4211,
|
||||||
|
0.4279,
|
||||||
|
0.5324,
|
||||||
|
0.40059999999999996,
|
||||||
|
1.4782,
|
||||||
|
0.6034999999999999,
|
||||||
|
0.43679999999999997,
|
||||||
|
0.4105,
|
||||||
|
0.408,
|
||||||
|
0.7263000000000001,
|
||||||
|
0.4593,
|
||||||
|
0.6996,
|
||||||
|
0.4271,
|
||||||
|
0.5413,
|
||||||
|
0.3998,
|
||||||
|
0.4329,
|
||||||
|
0.7967,
|
||||||
|
0.9076000000000001,
|
||||||
|
0.8878,
|
||||||
|
0.4333,
|
||||||
|
0.431,
|
||||||
|
0.8344999999999999,
|
||||||
|
1.1369,
|
||||||
|
0.42410000000000003,
|
||||||
|
0.4025
|
||||||
|
],
|
||||||
|
"debug": {
|
||||||
|
"mean_ms": 0.967733333333333,
|
||||||
|
"p50_ms": 0.8194,
|
||||||
|
"p95_ms": 1.6146,
|
||||||
|
"samples": 480
|
||||||
|
},
|
||||||
|
"debug_samples_ms": [
|
||||||
|
0.8167,
|
||||||
|
0.7293,
|
||||||
|
0.9592,
|
||||||
|
0.8642,
|
||||||
|
0.7673,
|
||||||
|
0.8222,
|
||||||
|
0.7207,
|
||||||
|
0.7231,
|
||||||
|
0.8286,
|
||||||
|
0.7298,
|
||||||
|
0.8266,
|
||||||
|
0.7201,
|
||||||
|
0.7448,
|
||||||
|
0.8205,
|
||||||
|
0.7162,
|
||||||
|
1.0768,
|
||||||
|
0.8357,
|
||||||
|
0.7537,
|
||||||
|
0.8582000000000001,
|
||||||
|
0.7343999999999999,
|
||||||
|
0.7179000000000001,
|
||||||
|
4.9764,
|
||||||
|
0.8008,
|
||||||
|
0.8557,
|
||||||
|
0.7527,
|
||||||
|
0.9688,
|
||||||
|
0.8746999999999999,
|
||||||
|
0.727,
|
||||||
|
0.8711,
|
||||||
|
0.7105,
|
||||||
|
0.7215,
|
||||||
|
0.8220000000000001,
|
||||||
|
0.7176,
|
||||||
|
0.8272,
|
||||||
|
0.7191,
|
||||||
|
0.9924999999999999,
|
||||||
|
0.9016000000000001,
|
||||||
|
0.7423,
|
||||||
|
1.2084000000000001,
|
||||||
|
0.8556,
|
||||||
|
0.7384,
|
||||||
|
0.7409,
|
||||||
|
0.7081999999999999,
|
||||||
|
0.8427,
|
||||||
|
0.8474,
|
||||||
|
0.7284,
|
||||||
|
0.7603,
|
||||||
|
3.1463,
|
||||||
|
1.6184999999999998,
|
||||||
|
1.1203999999999998,
|
||||||
|
0.8165,
|
||||||
|
1.4208,
|
||||||
|
0.924,
|
||||||
|
0.85,
|
||||||
|
0.7396,
|
||||||
|
0.8244,
|
||||||
|
0.9145,
|
||||||
|
0.7588,
|
||||||
|
0.812,
|
||||||
|
1.0149,
|
||||||
|
0.7919,
|
||||||
|
0.9867999999999999,
|
||||||
|
0.8725999999999999,
|
||||||
|
0.7468,
|
||||||
|
0.8335,
|
||||||
|
1.1976,
|
||||||
|
0.9091,
|
||||||
|
0.7374999999999999,
|
||||||
|
0.8851,
|
||||||
|
1.9234,
|
||||||
|
1.1454,
|
||||||
|
1.2746000000000002,
|
||||||
|
1.474,
|
||||||
|
0.7353000000000001,
|
||||||
|
0.8841,
|
||||||
|
0.9075,
|
||||||
|
0.7607999999999999,
|
||||||
|
0.8289000000000001,
|
||||||
|
0.8323999999999999,
|
||||||
|
0.8009000000000001,
|
||||||
|
0.8202,
|
||||||
|
0.7681,
|
||||||
|
0.8252999999999999,
|
||||||
|
0.8329,
|
||||||
|
0.7232999999999999,
|
||||||
|
0.8313,
|
||||||
|
0.7251,
|
||||||
|
0.8301,
|
||||||
|
0.7288,
|
||||||
|
0.7092,
|
||||||
|
0.8169,
|
||||||
|
0.7212,
|
||||||
|
0.882,
|
||||||
|
0.7234,
|
||||||
|
0.7201,
|
||||||
|
3.2547,
|
||||||
|
0.7303000000000001,
|
||||||
|
0.9255,
|
||||||
|
0.755,
|
||||||
|
0.7154,
|
||||||
|
0.9,
|
||||||
|
0.8394,
|
||||||
|
0.7272,
|
||||||
|
0.9393,
|
||||||
|
0.8936000000000001,
|
||||||
|
0.733,
|
||||||
|
6.3655,
|
||||||
|
2.8007,
|
||||||
|
0.8151999999999999,
|
||||||
|
0.7929999999999999,
|
||||||
|
0.8479,
|
||||||
|
1.1844,
|
||||||
|
0.8132,
|
||||||
|
0.9790000000000001,
|
||||||
|
0.8571,
|
||||||
|
0.7755000000000001,
|
||||||
|
0.8656,
|
||||||
|
0.73,
|
||||||
|
0.7969999999999999,
|
||||||
|
0.8276,
|
||||||
|
0.7308,
|
||||||
|
0.883,
|
||||||
|
0.7273,
|
||||||
|
0.8855,
|
||||||
|
0.7218,
|
||||||
|
0.7303999999999999,
|
||||||
|
0.8796999999999999,
|
||||||
|
0.9757,
|
||||||
|
2.321,
|
||||||
|
0.7583,
|
||||||
|
0.7388,
|
||||||
|
0.8352,
|
||||||
|
0.7089000000000001,
|
||||||
|
1.1594000000000002,
|
||||||
|
0.9188999999999999,
|
||||||
|
1.6146,
|
||||||
|
7.5508,
|
||||||
|
0.8053,
|
||||||
|
1.2301,
|
||||||
|
0.7657,
|
||||||
|
1.4819,
|
||||||
|
1.0386,
|
||||||
|
0.7665000000000001,
|
||||||
|
0.7372000000000001,
|
||||||
|
0.7472,
|
||||||
|
0.9138,
|
||||||
|
0.782,
|
||||||
|
0.8009999999999999,
|
||||||
|
0.9136,
|
||||||
|
0.9881,
|
||||||
|
0.846,
|
||||||
|
0.7302,
|
||||||
|
0.7045,
|
||||||
|
0.7411,
|
||||||
|
0.8925,
|
||||||
|
0.7107,
|
||||||
|
3.5707,
|
||||||
|
0.7821,
|
||||||
|
0.8599,
|
||||||
|
0.8071,
|
||||||
|
0.7342,
|
||||||
|
0.8309,
|
||||||
|
0.7102999999999999,
|
||||||
|
2.6215,
|
||||||
|
0.8529,
|
||||||
|
0.7293,
|
||||||
|
0.7728,
|
||||||
|
0.8239000000000001,
|
||||||
|
1.3806,
|
||||||
|
3.0549,
|
||||||
|
1.4774,
|
||||||
|
1.4057,
|
||||||
|
1.4697,
|
||||||
|
0.8741000000000001,
|
||||||
|
1.2217,
|
||||||
|
0.8339000000000001,
|
||||||
|
0.7242000000000001,
|
||||||
|
0.8094,
|
||||||
|
0.7016,
|
||||||
|
0.8065,
|
||||||
|
0.7062999999999999,
|
||||||
|
1.1670999999999998,
|
||||||
|
0.7271,
|
||||||
|
1.4295,
|
||||||
|
0.7381,
|
||||||
|
2.2805,
|
||||||
|
1.673,
|
||||||
|
0.7472,
|
||||||
|
0.7219,
|
||||||
|
3.197,
|
||||||
|
0.7538999999999999,
|
||||||
|
0.8595999999999999,
|
||||||
|
0.7359,
|
||||||
|
0.8168,
|
||||||
|
0.7067,
|
||||||
|
0.7971,
|
||||||
|
2.1982999999999997,
|
||||||
|
1.2154,
|
||||||
|
1.27,
|
||||||
|
3.3795,
|
||||||
|
1.1157000000000001,
|
||||||
|
0.7726999999999999,
|
||||||
|
0.8442000000000001,
|
||||||
|
0.8882,
|
||||||
|
0.7017,
|
||||||
|
0.7211,
|
||||||
|
1.2641,
|
||||||
|
0.8506,
|
||||||
|
0.723,
|
||||||
|
3.8099000000000003,
|
||||||
|
1.1316,
|
||||||
|
1.0410000000000001,
|
||||||
|
0.8524999999999999,
|
||||||
|
1.0685,
|
||||||
|
0.7464999999999999,
|
||||||
|
0.8846999999999999,
|
||||||
|
1.0812,
|
||||||
|
1.107,
|
||||||
|
1.093,
|
||||||
|
1.1158,
|
||||||
|
0.7374,
|
||||||
|
1.5673,
|
||||||
|
1.0413,
|
||||||
|
0.8394999999999999,
|
||||||
|
0.7274,
|
||||||
|
0.8664000000000001,
|
||||||
|
1.0905,
|
||||||
|
0.7726000000000001,
|
||||||
|
1.5097,
|
||||||
|
0.9828,
|
||||||
|
0.9055,
|
||||||
|
0.7475,
|
||||||
|
0.9872,
|
||||||
|
0.9333,
|
||||||
|
0.8344999999999999,
|
||||||
|
0.7199,
|
||||||
|
1.4668,
|
||||||
|
2.1616999999999997,
|
||||||
|
1.0192999999999999,
|
||||||
|
0.7895000000000001,
|
||||||
|
1.8851,
|
||||||
|
1.3231,
|
||||||
|
1.2965,
|
||||||
|
0.7644000000000001,
|
||||||
|
0.7393000000000001,
|
||||||
|
0.9441,
|
||||||
|
0.7245,
|
||||||
|
0.7081000000000001,
|
||||||
|
0.8285,
|
||||||
|
0.7283,
|
||||||
|
0.8108,
|
||||||
|
0.7171000000000001,
|
||||||
|
0.7234999999999999,
|
||||||
|
0.7841,
|
||||||
|
1.0449000000000002,
|
||||||
|
1.0277,
|
||||||
|
1.0941,
|
||||||
|
1.2095,
|
||||||
|
1.158,
|
||||||
|
0.8891,
|
||||||
|
0.7197,
|
||||||
|
0.882,
|
||||||
|
0.7267,
|
||||||
|
0.7133,
|
||||||
|
0.7043,
|
||||||
|
0.727,
|
||||||
|
0.7603,
|
||||||
|
0.8140000000000001,
|
||||||
|
0.8274,
|
||||||
|
0.7349,
|
||||||
|
0.723,
|
||||||
|
0.8446,
|
||||||
|
0.9939,
|
||||||
|
0.9430999999999999,
|
||||||
|
1.2679,
|
||||||
|
0.8774,
|
||||||
|
0.7139,
|
||||||
|
0.7448999999999999,
|
||||||
|
0.7282000000000001,
|
||||||
|
0.7496,
|
||||||
|
1.2924,
|
||||||
|
1.2619,
|
||||||
|
1.2734999999999999,
|
||||||
|
0.8732,
|
||||||
|
0.9319000000000001,
|
||||||
|
0.7432,
|
||||||
|
0.842,
|
||||||
|
0.7283,
|
||||||
|
0.7115,
|
||||||
|
0.8179,
|
||||||
|
0.7108,
|
||||||
|
0.8030999999999999,
|
||||||
|
0.7085,
|
||||||
|
0.7425,
|
||||||
|
0.9125,
|
||||||
|
0.7362,
|
||||||
|
0.8239000000000001,
|
||||||
|
0.7027,
|
||||||
|
0.7646000000000001,
|
||||||
|
0.841,
|
||||||
|
0.727,
|
||||||
|
0.8307,
|
||||||
|
0.7421,
|
||||||
|
0.7456,
|
||||||
|
0.8228000000000001,
|
||||||
|
0.7144,
|
||||||
|
0.8956,
|
||||||
|
0.7155,
|
||||||
|
0.8194,
|
||||||
|
1.1168,
|
||||||
|
0.8448,
|
||||||
|
0.7318,
|
||||||
|
0.8819,
|
||||||
|
0.7425,
|
||||||
|
0.7328,
|
||||||
|
0.8884,
|
||||||
|
0.7295999999999999,
|
||||||
|
0.7747,
|
||||||
|
0.8804000000000001,
|
||||||
|
0.7343999999999999,
|
||||||
|
1.1522,
|
||||||
|
0.9186,
|
||||||
|
0.7333,
|
||||||
|
0.8239000000000001,
|
||||||
|
0.8241999999999999,
|
||||||
|
0.759,
|
||||||
|
0.861,
|
||||||
|
0.7278,
|
||||||
|
0.8876,
|
||||||
|
0.7179000000000001,
|
||||||
|
0.7827,
|
||||||
|
0.8582000000000001,
|
||||||
|
0.7512000000000001,
|
||||||
|
1.0376,
|
||||||
|
0.8305,
|
||||||
|
0.7426,
|
||||||
|
0.7012999999999999,
|
||||||
|
0.8982,
|
||||||
|
0.7402,
|
||||||
|
0.7324,
|
||||||
|
0.8277,
|
||||||
|
0.7297,
|
||||||
|
0.8775000000000001,
|
||||||
|
0.7464999999999999,
|
||||||
|
0.7191,
|
||||||
|
1.0924,
|
||||||
|
0.8524999999999999,
|
||||||
|
0.7319,
|
||||||
|
0.7194,
|
||||||
|
0.8802,
|
||||||
|
0.7388,
|
||||||
|
1.2646,
|
||||||
|
0.8413,
|
||||||
|
0.7404999999999999,
|
||||||
|
0.7242000000000001,
|
||||||
|
0.8245,
|
||||||
|
0.725,
|
||||||
|
0.8404,
|
||||||
|
0.7255,
|
||||||
|
0.7195,
|
||||||
|
0.7114,
|
||||||
|
0.8140000000000001,
|
||||||
|
0.7406,
|
||||||
|
0.7184,
|
||||||
|
0.8338,
|
||||||
|
0.7412,
|
||||||
|
0.8614,
|
||||||
|
0.7208,
|
||||||
|
0.7693,
|
||||||
|
0.8305,
|
||||||
|
0.7565000000000001,
|
||||||
|
0.8252,
|
||||||
|
0.7134,
|
||||||
|
0.7383,
|
||||||
|
0.8237,
|
||||||
|
0.7262,
|
||||||
|
0.8196,
|
||||||
|
0.7092,
|
||||||
|
0.8099,
|
||||||
|
0.7102999999999999,
|
||||||
|
0.7117,
|
||||||
|
0.837,
|
||||||
|
0.7314,
|
||||||
|
0.8402000000000001,
|
||||||
|
0.8168,
|
||||||
|
0.7099,
|
||||||
|
0.8225,
|
||||||
|
0.7115,
|
||||||
|
0.713,
|
||||||
|
0.7081999999999999,
|
||||||
|
0.7036,
|
||||||
|
0.8197,
|
||||||
|
0.7163,
|
||||||
|
0.8418,
|
||||||
|
0.7012,
|
||||||
|
0.7218,
|
||||||
|
0.8236,
|
||||||
|
0.7665000000000001,
|
||||||
|
0.8529,
|
||||||
|
0.759,
|
||||||
|
0.7396,
|
||||||
|
0.8301,
|
||||||
|
0.7272,
|
||||||
|
0.828,
|
||||||
|
0.7037,
|
||||||
|
0.7199,
|
||||||
|
0.7303999999999999,
|
||||||
|
1.0776,
|
||||||
|
0.8279,
|
||||||
|
0.8061,
|
||||||
|
0.7394,
|
||||||
|
0.7934,
|
||||||
|
0.805,
|
||||||
|
0.8004,
|
||||||
|
0.9452,
|
||||||
|
0.791,
|
||||||
|
1.0659,
|
||||||
|
1.2731000000000001,
|
||||||
|
1.2185,
|
||||||
|
1.1789,
|
||||||
|
1.4793,
|
||||||
|
1.3232000000000002,
|
||||||
|
1.3123,
|
||||||
|
1.1511,
|
||||||
|
4.9007,
|
||||||
|
1.9954999999999998,
|
||||||
|
0.7574,
|
||||||
|
0.7156,
|
||||||
|
1.063,
|
||||||
|
0.7659,
|
||||||
|
0.7311000000000001,
|
||||||
|
0.8957,
|
||||||
|
0.9449000000000001,
|
||||||
|
0.7726999999999999,
|
||||||
|
0.7187,
|
||||||
|
0.7123,
|
||||||
|
0.7200000000000001,
|
||||||
|
0.8279,
|
||||||
|
0.7073,
|
||||||
|
0.7967,
|
||||||
|
0.785,
|
||||||
|
0.7429,
|
||||||
|
0.8313,
|
||||||
|
0.7189,
|
||||||
|
0.7351000000000001,
|
||||||
|
0.7239,
|
||||||
|
0.7041000000000001,
|
||||||
|
0.8423,
|
||||||
|
0.7414999999999999,
|
||||||
|
0.8124,
|
||||||
|
0.7135,
|
||||||
|
0.7106,
|
||||||
|
0.8559,
|
||||||
|
0.7453,
|
||||||
|
0.9529,
|
||||||
|
0.7197,
|
||||||
|
0.8180000000000001,
|
||||||
|
0.7674,
|
||||||
|
0.7205,
|
||||||
|
0.8537,
|
||||||
|
1.0651,
|
||||||
|
0.862,
|
||||||
|
0.7998000000000001,
|
||||||
|
0.8413,
|
||||||
|
0.7829,
|
||||||
|
0.7494999999999999,
|
||||||
|
0.9843000000000001,
|
||||||
|
1.0847,
|
||||||
|
0.7273,
|
||||||
|
0.7253999999999999,
|
||||||
|
0.8928,
|
||||||
|
0.722,
|
||||||
|
1.1499,
|
||||||
|
1.1878,
|
||||||
|
0.8394,
|
||||||
|
0.8515,
|
||||||
|
1.642,
|
||||||
|
1.8285,
|
||||||
|
0.7554000000000001,
|
||||||
|
0.8442999999999999
|
||||||
|
],
|
||||||
|
"debug_span_count_max": 14,
|
||||||
|
"debug_span_count_min": 14,
|
||||||
|
"files": 1000,
|
||||||
|
"kind": "prepared_sqlite_trace_overhead",
|
||||||
|
"limits": "tiny warm lexical corpus; harness serial/no model/media/GPU; other VM work including native photo inference may contend; not isolated production performance",
|
||||||
|
"measurement": "outer Rust call including metrics assembly; excludes fixture setup/JSON serialization",
|
||||||
|
"os": "windows",
|
||||||
|
"production_claim": false,
|
||||||
|
"profile": "release",
|
||||||
|
"queries": 24,
|
||||||
|
"repetitions": 20
|
||||||
|
}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
derived/
|
||||||
|
inspection/
|
||||||
|
*.sqlite*
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# Independently labelled media diagnostic v1
|
||||||
|
|
||||||
|
`fixture.json` and `labels.lock.json` were frozen before media inference. The fixture contains 20 original public/licensed files and 65 English/German queries. Neutral filenames prevent filename semantics from giving away visual answers. Do not edit v1 labels after seeing rankings; corrections require a new version and another pre-scoring lock.
|
||||||
|
|
||||||
|
The assistant inspected every photo and sampled real video frames through `view_image`. Cat-action/flower-state intervals came from these frames. Spoken-topic intervals came from NASA's publisher-provided SRT, not a model transcript. Music/sound labels came from independent publisher descriptions and original embedded tags; human listening verification is **not** claimed. One annotator authored these labels, so a blinded second human review remains necessary.
|
||||||
|
|
||||||
|
## Provenance and redistribution
|
||||||
|
|
||||||
|
Every asset has its exact source URL, description page, license, byte count, SHA256, and modification record in the fixture. The assets are a small educational search benchmark, not a stock-media product.
|
||||||
|
|
||||||
|
- Twelve 640px photos are publisher-generated derivatives under the [Pexels License](https://www.pexels.com/license/); no local pixel modifications were made. Pexels IDs identify the sources; attribution is not required by that license. Creator names are not guessed.
|
||||||
|
- `clip01.webm`: **Cat opening door**, Astropulse, [source](https://commons.wikimedia.org/wiki/File:Cat_opening_door.webm), [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/). The original is unmodified. Reproduced frames are derivatives and retain that license and attribution.
|
||||||
|
- `clip02.webm`: **Water flowing strem**, BhotaRohit, [source](https://commons.wikimedia.org/wiki/File:Water_flowing_strem.webm), [CC0](https://creativecommons.org/publicdomain/zero/1.0/). Visual inspection reveals a misty waterfall; labels deliberately follow the pixels rather than the filename.
|
||||||
|
- `clip03.webm` and `sound02.mp3`: MDN contributors, original [interactive-example assets](https://github.com/mdn/interactive-examples), repository [CC0 notice](https://raw.githubusercontent.com/mdn/interactive-examples/main/LICENSE). The files are the real flower timelapse and roaring-creature effect.
|
||||||
|
- `clip04.mp4`: NASA **Artemis Origins**, [source](https://images.nasa.gov/details/Artemis%20Origins), used for educational/informational evaluation under [NASA media usage permission](https://www.nasa.gov/nasa-brand-center/images-and-media/). No endorsement is implied; NASA insignia/trademarks are not used as Searchi branding. The footage is unmodified.
|
||||||
|
- `music01.ogg`: **Variatio 13 a 2 Clav.**, Kimiko Ishizaka, [source](https://commons.wikimedia.org/wiki/File:Kimiko_Ishizaka_-_14_-_Variatio_13_a_2_Clav.ogg), [CC0](https://creativecommons.org/publicdomain/zero/1.0/). The project's primary [release description](https://github.com/robertDouglass/opengoldbergvariations.org/blob/master/config.toml) confirms the public-domain release, independently of the Commons pending-review banner. Original audio and Vorbis metadata are retained.
|
||||||
|
- `music02.ogg`: **Jazz Violin Solo**, VWD, [source](https://commons.wikimedia.org/wiki/File:00_Jazz_Violin_Solo.ogg), author's worldwide public-domain dedication on that page. Original tags identify the artist as **Violinist WTD**, which is intentionally the strict metadata query rather than a guessed identity.
|
||||||
|
- `sound01.wav`: **Dog barking mono**, Brandon Morris, uploaded by HaelDB, [source](https://opengameart.org/content/dog-barking-mono), [CC0](https://creativecommons.org/publicdomain/zero/1.0/). Original waveform retained.
|
||||||
|
|
||||||
|
All source licenses remain attached to their files. Root repository MIT licensing does not replace these third-party media terms. `derived/` and `inspection/` are ignored; generated derivatives carry their source licenses when redistributed.
|
||||||
|
|
||||||
|
This collection is deliberately tiny. Twenty public assets do not establish quality on a user's complete photo library, music collection, recorded meetings, or long movies.
|
||||||
|
After Width: | Height: | Size: 27 KiB |
|
After Width: | Height: | Size: 67 KiB |
|
After Width: | Height: | Size: 34 KiB |
|
After Width: | Height: | Size: 37 KiB |
|
After Width: | Height: | Size: 50 KiB |
|
After Width: | Height: | Size: 27 KiB |
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 64 KiB |
|
After Width: | Height: | Size: 67 KiB |
|
After Width: | Height: | Size: 23 KiB |
|
After Width: | Height: | Size: 27 KiB |
|
After Width: | Height: | Size: 14 KiB |
@@ -0,0 +1,852 @@
|
|||||||
|
{
|
||||||
|
"background_prewarm_ms": 133.0573,
|
||||||
|
"device": "cpu",
|
||||||
|
"documents": 20,
|
||||||
|
"embedding_report": {
|
||||||
|
"chunks": 20,
|
||||||
|
"generation": 1,
|
||||||
|
"model_fingerprint": "google/embeddinggemma-2:bfcd298762cc34d0357ece5ebdd31791a3a374d8:q8_0:llama-b11461-no-logits-v1:mean:l2:query512:doc2048:windows-overlap128:prompts-v1:byte64k-overlap512-title512-v1",
|
||||||
|
"passages": 20
|
||||||
|
},
|
||||||
|
"first_foreground_ms": 108.6225,
|
||||||
|
"first_foreground_top1": "supplier-agreement.txt",
|
||||||
|
"hybrid": {
|
||||||
|
"recall_at10": 1.0,
|
||||||
|
"recall_at3": 1.0,
|
||||||
|
"top1_accuracy": 1.0
|
||||||
|
},
|
||||||
|
"hybrid_p50_ms": 107.3451,
|
||||||
|
"hybrid_p95_ms": 146.82819999999998,
|
||||||
|
"indexing_ms": 4840.859100000001,
|
||||||
|
"lexical": {
|
||||||
|
"recall_at10": 0.20833333333333334,
|
||||||
|
"recall_at3": 0.20833333333333334,
|
||||||
|
"top1_accuracy": 0.20833333333333334
|
||||||
|
},
|
||||||
|
"limitation": "Synthetic seed corpus only; no production quality or large-corpus latency certification",
|
||||||
|
"load_ms": 8644.3299,
|
||||||
|
"model": "google/embeddinggemma-2",
|
||||||
|
"queries": 24,
|
||||||
|
"rankings": [
|
||||||
|
{
|
||||||
|
"expected": "supplier-contract",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "supplier-agreement.txt",
|
||||||
|
"match_kind": "exact_filename",
|
||||||
|
"source": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "INV-2026-1042.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "exact",
|
||||||
|
"lexical_rank": 1,
|
||||||
|
"query": "supplier-agreement.txt",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "invoice",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "INV-2026-1042.txt",
|
||||||
|
"match_kind": "filename",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "network-incident.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "identifier",
|
||||||
|
"lexical_rank": 1,
|
||||||
|
"query": "INV-2026-1042",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "error-code",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "support-error.txt",
|
||||||
|
"match_kind": "exact_identifier",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "search-engine-design.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "INV-2026-1042.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "identifier",
|
||||||
|
"lexical_rank": 1,
|
||||||
|
"query": "E_SEARCH_071",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "supplier-contract",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "supplier-agreement.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "retry-design.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "How can we end our relationship with the supplier?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "supplier-contract",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "supplier-agreement.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "mietvertrag.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "de-crosslingual",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Wie kündigen wir die Zusammenarbeit mit unserem Lieferanten?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "apartment-lease",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "mietvertrag.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "support-error.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "urlaubsrichtlinie.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en-crosslingual",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Can I keep a pet in the rented apartment?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "apartment-lease",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "mietvertrag.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "reisekosten.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "monitor-warranty.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "de",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Wo steht etwas über die Kaution für die Wohnung?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "travel-expenses",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "reisekosten.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "INV-2026-1042.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "conference-agenda.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en-crosslingual",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "How do I claim my train and hotel costs?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "conference-agenda",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "conference-agenda.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "support-error.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "search-engine-design.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "de-crosslingual",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Wann ist der Workshop zur Datenbankleistung?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "deployment",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "deployment-runbook.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "product-launch.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "support-error.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "What should we do if the new release fails its health check?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "backup-restore",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "backup-recovery.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "network-incident.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "de-crosslingual",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Ich habe Dateien gelöscht und brauche sie aus der Sicherung zurück",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "api-retry",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "retry-design.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "deployment-runbook.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "network-incident.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "How should the app retry a temporary server failure?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "oauth",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "authentication-notes.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "retry-design.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "backup-recovery.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "How do we obtain another access token after it expires?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "network-outage",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "network-incident.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "backup-recovery.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "de-crosslingual",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Warum war der Netzwerkswitch nicht erreichbar?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "launch-plan",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "product-launch.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "research-budget.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "deployment-runbook.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "The plan for announcing the new product after beta testing",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "research-budget",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "research-budget.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "INV-2026-1042.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "retry-design.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Money allocated for interviews and prototype usability studies",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "vacation-policy",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "urlaubsrichtlinie.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "INV-2026-1042.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en-crosslingual",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Who approves my time off and who covers urgent work?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "recipe",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "tomato-soup.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "gartenplanung.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "INV-2026-1042.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "A soup made with roasted tomatoes and basil",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "renovation",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "renovation-estimate.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "INV-2026-1042.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Does the kitchen renovation quote include new flooring?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "warranty",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "monitor-warranty.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "network-incident.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "backup-recovery.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "The document about replacing a defective display under warranty",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "search-architecture",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "search-engine-design.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "support-error.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "backup-recovery.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "How can keyword results and embeddings be combined for file search?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "garden",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "gartenplanung.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "renovation-estimate.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en-crosslingual",
|
||||||
|
"lexical_rank": null,
|
||||||
|
"query": "Where is the plan for the raised vegetable bed and rainwater barrel?",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "cancellation-email",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "hybrid",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "urlaubsrichtlinie.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "conference-agenda.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "en",
|
||||||
|
"lexical_rank": 1,
|
||||||
|
"query": "Cancel tomorrow's planning meeting",
|
||||||
|
"semantic_rank": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expected": "invoice",
|
||||||
|
"hybrid_rank": 1,
|
||||||
|
"hybrid_top3": [
|
||||||
|
{
|
||||||
|
"filename": "INV-2026-1042.txt",
|
||||||
|
"match_kind": "exact_identifier",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "meeting-cancellation.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"filename": "search-engine-design.txt",
|
||||||
|
"match_kind": "semantic",
|
||||||
|
"source": {
|
||||||
|
"kind": "line",
|
||||||
|
"number": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"language": "identifier",
|
||||||
|
"lexical_rank": 1,
|
||||||
|
"query": "OFFICE-CHAIRS-42",
|
||||||
|
"semantic_rank": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"semantic": {
|
||||||
|
"recall_at10": 1.0,
|
||||||
|
"recall_at3": 1.0,
|
||||||
|
"top1_accuracy": 1.0
|
||||||
|
},
|
||||||
|
"startup_init_ms": 8286.3466,
|
||||||
|
"threads": 1
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-10-12",
|
||||||
|
"os": "Microsoft Windows 11 Pro",
|
||||||
|
"os_version": "10.0.26300",
|
||||||
|
"visible_memory_kib": 10416556,
|
||||||
|
"cpu_name": "AMD Ryzen 7 7800X3D 8-Core Processor ",
|
||||||
|
"vm_vcpus": 4,
|
||||||
|
"model_threads": 1,
|
||||||
|
"device": "cpu",
|
||||||
|
"force_cpu": "1",
|
||||||
|
"ffmpeg_version": "9.0.2",
|
||||||
|
"worker_sha256": "f2ad020307469371519b848c9fa92ae4c51ee8ae8a0ce917f8f408f5756e9086",
|
||||||
|
"scope": "Shared Windows VM; timing affected by other work; RAM observations are not measured peak"
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
{
|
||||||
|
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
|
||||||
|
"worker": {
|
||||||
|
"device": "cpu",
|
||||||
|
"dimensions": 768,
|
||||||
|
"media_supported": true,
|
||||||
|
"model": "google/embeddinggemma-2",
|
||||||
|
"ready": true,
|
||||||
|
"runtime": "b11461-no-logits-v1",
|
||||||
|
"threads": 1
|
||||||
|
},
|
||||||
|
"photos": 12,
|
||||||
|
"queries": 24,
|
||||||
|
"query_embedding_ms": 2262.7884,
|
||||||
|
"budgets": [
|
||||||
|
{
|
||||||
|
"token_budget": 70,
|
||||||
|
"encoding_ms": 47612.655699999996,
|
||||||
|
"metrics": {
|
||||||
|
"positive_queries": 24,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"token_budget": 280,
|
||||||
|
"encoding_ms": 199652.08899999998,
|
||||||
|
"metrics": {
|
||||||
|
"positive_queries": 24,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"token_budget": 1120,
|
||||||
|
"encoding_ms": 1493321.6608,
|
||||||
|
"metrics": {
|
||||||
|
"positive_queries": 24,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"interpretation": "Same frozen 12 photos and 24 correlated EN/DE queries. Query embeddings shared across budgets. Encoding time is CPU batch cost; ranking-only time excludes inference. Equal small-corpus recall does not prove equal quality on larger or more difficult images."
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
|
||||||
|
"frozen_before_media_scoring": true,
|
||||||
|
"document_seed_sha256": "3bab37fc0ec9c2e2e40684962b049172b5078358c6487c9033ed8697b8128dd7",
|
||||||
|
"query_count": 65,
|
||||||
|
"asset_count": 20
|
||||||
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
{
|
||||||
|
"encoding_failures": [],
|
||||||
|
"encoding_ms": 1863.0235000000002,
|
||||||
|
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
|
||||||
|
"rows": [
|
||||||
|
{
|
||||||
|
"file_rankings": [
|
||||||
|
{
|
||||||
|
"candidate": {
|
||||||
|
"asset": "music01",
|
||||||
|
"match_kind": "media_metadata",
|
||||||
|
"modality": "metadata"
|
||||||
|
},
|
||||||
|
"score": 2.847686714371868
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"query": {
|
||||||
|
"category": "exact-metadata",
|
||||||
|
"expected": [
|
||||||
|
"music01"
|
||||||
|
],
|
||||||
|
"id": "q055",
|
||||||
|
"language": "en",
|
||||||
|
"metadata": {
|
||||||
|
"artist": "Kimiko Ishizaka"
|
||||||
|
},
|
||||||
|
"text": "Kimiko Ishizaka"
|
||||||
|
},
|
||||||
|
"query_ms": 1.0931,
|
||||||
|
"segment_top5": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file_rankings": [
|
||||||
|
{
|
||||||
|
"candidate": {
|
||||||
|
"asset": "music02",
|
||||||
|
"match_kind": "media_metadata",
|
||||||
|
"modality": "metadata"
|
||||||
|
},
|
||||||
|
"score": 4.671020217612865
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"query": {
|
||||||
|
"category": "exact-metadata",
|
||||||
|
"expected": [
|
||||||
|
"music02"
|
||||||
|
],
|
||||||
|
"id": "q056",
|
||||||
|
"language": "en",
|
||||||
|
"metadata": {
|
||||||
|
"title": "Jazz Violin Solo"
|
||||||
|
},
|
||||||
|
"text": "Jazz Violin Solo"
|
||||||
|
},
|
||||||
|
"query_ms": 0.7408,
|
||||||
|
"segment_top5": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file_rankings": [
|
||||||
|
{
|
||||||
|
"candidate": {
|
||||||
|
"asset": "music02",
|
||||||
|
"match_kind": "media_metadata",
|
||||||
|
"modality": "metadata"
|
||||||
|
},
|
||||||
|
"score": 3.6759070488571206
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"query": {
|
||||||
|
"category": "exact-metadata",
|
||||||
|
"expected": [
|
||||||
|
"music02"
|
||||||
|
],
|
||||||
|
"id": "q057",
|
||||||
|
"language": "en",
|
||||||
|
"metadata": {
|
||||||
|
"artist": "Violinist WTD"
|
||||||
|
},
|
||||||
|
"text": "Violinist WTD"
|
||||||
|
},
|
||||||
|
"query_ms": 0.6757,
|
||||||
|
"segment_top5": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file_rankings": [],
|
||||||
|
"query": {
|
||||||
|
"category": "metadata-no-answer",
|
||||||
|
"expected": [],
|
||||||
|
"id": "q065",
|
||||||
|
"language": "en",
|
||||||
|
"metadata": {
|
||||||
|
"artist": "Adele"
|
||||||
|
},
|
||||||
|
"text": "Adele"
|
||||||
|
},
|
||||||
|
"query_ms": 0.4148,
|
||||||
|
"segment_top5": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"scope": "Actual FFprobe stream/format metadata + guarded Searchi core FTS; lexical identity queries, no embedding fallback; encoding_ms measures metadata extraction, not model inference",
|
||||||
|
"worker": null
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
{
|
||||||
|
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
|
||||||
|
"worker": null,
|
||||||
|
"overall": {
|
||||||
|
"positive_queries": 3,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 1,
|
||||||
|
"no_answer_return_rate": 0.0
|
||||||
|
},
|
||||||
|
"by_category": {
|
||||||
|
"exact-metadata": {
|
||||||
|
"positive_queries": 3,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"metadata-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 1,
|
||||||
|
"no_answer_return_rate": 0.0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"by_language": {
|
||||||
|
"en": {
|
||||||
|
"positive_queries": 3,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 1,
|
||||||
|
"no_answer_return_rate": 0.0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"encoding_failures": [],
|
||||||
|
"encoding_ms": 1863.0235000000002,
|
||||||
|
"query_p50_ms": 0.7408,
|
||||||
|
"query_p95_ms": 1.0931,
|
||||||
|
"failures": [],
|
||||||
|
"source_scope": "Actual FFprobe stream/format metadata + guarded Searchi core FTS; lexical identity queries, no embedding fallback; encoding_ms measures metadata extraction, not model inference",
|
||||||
|
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
|
||||||
|
}
|
||||||
@@ -0,0 +1,561 @@
|
|||||||
|
{
|
||||||
|
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
|
||||||
|
"worker": {
|
||||||
|
"device": "cpu",
|
||||||
|
"dimensions": 768,
|
||||||
|
"media_supported": true,
|
||||||
|
"model": "google/embeddinggemma-2",
|
||||||
|
"ready": true,
|
||||||
|
"runtime": "b11461-no-logits-v1",
|
||||||
|
"threads": 1
|
||||||
|
},
|
||||||
|
"overall": {
|
||||||
|
"positive_queries": 58,
|
||||||
|
"unsupported_queries": 1,
|
||||||
|
"recall_at1": 0.9655172413793104,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 0.9827586206896551,
|
||||||
|
"temporal_queries": 22,
|
||||||
|
"segment_overlap_hit_at1": 0.8181818181818182,
|
||||||
|
"timestamp_point_hit_at1": 0.5909090909090909,
|
||||||
|
"segment_overlap_hit_at5": 0.8181818181818182,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 3.777000000000001,
|
||||||
|
"no_answer_queries": 6,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"by_category": {
|
||||||
|
"content-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 2,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"exact-filter": {
|
||||||
|
"positive_queries": 1,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"exact-metadata": {
|
||||||
|
"positive_queries": 3,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"filter-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 1,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"image-memory": {
|
||||||
|
"positive_queries": 24,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"metadata-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 1,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"music-memory": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 0.5,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 0.75,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"sound-memory": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"speech-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 2,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"temporal-negative": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 1,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-action": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 4,
|
||||||
|
"segment_overlap_hit_at1": 1.0,
|
||||||
|
"timestamp_point_hit_at1": 1.0,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 0,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-scene": {
|
||||||
|
"positive_queries": 6,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 6,
|
||||||
|
"segment_overlap_hit_at1": 1.0,
|
||||||
|
"timestamp_point_hit_at1": 1.0,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 0,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-speech": {
|
||||||
|
"positive_queries": 8,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 8,
|
||||||
|
"segment_overlap_hit_at1": 0.75,
|
||||||
|
"timestamp_point_hit_at1": 0.125,
|
||||||
|
"segment_overlap_hit_at5": 0.75,
|
||||||
|
"timestamp_distance_median_s": 1.551,
|
||||||
|
"timestamp_distance_max_s": 3.777000000000001,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-temporal": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 4,
|
||||||
|
"segment_overlap_hit_at1": 0.5,
|
||||||
|
"timestamp_point_hit_at1": 0.5,
|
||||||
|
"segment_overlap_hit_at5": 0.5,
|
||||||
|
"timestamp_distance_median_s": 1.005,
|
||||||
|
"timestamp_distance_max_s": 1.005,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"by_language": {
|
||||||
|
"de": {
|
||||||
|
"positive_queries": 27,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 0.9629629629629629,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 0.9814814814814815,
|
||||||
|
"temporal_queries": 11,
|
||||||
|
"segment_overlap_hit_at1": 0.8181818181818182,
|
||||||
|
"timestamp_point_hit_at1": 0.6363636363636364,
|
||||||
|
"segment_overlap_hit_at5": 0.8181818181818182,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 1.551,
|
||||||
|
"no_answer_queries": 2,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"en": {
|
||||||
|
"positive_queries": 31,
|
||||||
|
"unsupported_queries": 1,
|
||||||
|
"recall_at1": 0.967741935483871,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 0.9838709677419355,
|
||||||
|
"temporal_queries": 11,
|
||||||
|
"segment_overlap_hit_at1": 0.8181818181818182,
|
||||||
|
"timestamp_point_hit_at1": 0.5454545454545454,
|
||||||
|
"segment_overlap_hit_at5": 0.8181818181818182,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 3.777000000000001,
|
||||||
|
"no_answer_queries": 4,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"encoding_failures": [],
|
||||||
|
"encoding_ms": 671510.0948000001,
|
||||||
|
"query_p50_ms": 90.6467,
|
||||||
|
"query_p95_ms": 110.4152,
|
||||||
|
"failures": [
|
||||||
|
{
|
||||||
|
"query": "q029",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip03",
|
||||||
|
"end_s": 4.005,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_video",
|
||||||
|
"modality": "image",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 2005
|
||||||
|
},
|
||||||
|
"start_s": 2.005
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip03": [
|
||||||
|
[
|
||||||
|
0,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.005
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q030",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip03",
|
||||||
|
"end_s": 4.005,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_video",
|
||||||
|
"modality": "image",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 2005
|
||||||
|
},
|
||||||
|
"start_s": 2.005
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip03": [
|
||||||
|
[
|
||||||
|
0,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.005
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q039",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 10.0,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 0
|
||||||
|
},
|
||||||
|
"start_s": 0.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
1.551,
|
||||||
|
3.603
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.551
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q040",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 10.0,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 0
|
||||||
|
},
|
||||||
|
"start_s": 0.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
1.551,
|
||||||
|
3.603
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.551
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q041",
|
||||||
|
"category": "wrong-media-channel",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 26.167,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_video",
|
||||||
|
"modality": "image",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 24167
|
||||||
|
},
|
||||||
|
"start_s": 24.167
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
13.129,
|
||||||
|
16.316
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q042",
|
||||||
|
"category": "wrong-media-channel",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 14.467,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_video",
|
||||||
|
"modality": "image",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 12467
|
||||||
|
},
|
||||||
|
"start_s": 12.467
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
13.129,
|
||||||
|
16.316
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q043",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 50.0,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 40000
|
||||||
|
},
|
||||||
|
"start_s": 40.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
43.777,
|
||||||
|
52.419
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 3.777000000000001
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q045",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 66.0,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 56000
|
||||||
|
},
|
||||||
|
"start_s": 56.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
56.539,
|
||||||
|
63.496
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 0.5390000000000015
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q046",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 66.0,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 56000
|
||||||
|
},
|
||||||
|
"start_s": 56.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
56.539,
|
||||||
|
63.496
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 0.5390000000000015
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q049",
|
||||||
|
"category": "hard-negative-confusion",
|
||||||
|
"winner": "music01",
|
||||||
|
"expected": [
|
||||||
|
"music02"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q050",
|
||||||
|
"category": "hard-negative-confusion",
|
||||||
|
"winner": "music01",
|
||||||
|
"expected": [
|
||||||
|
"music02"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q059",
|
||||||
|
"category": "unsupported-query-capability",
|
||||||
|
"detail": "Production query API has no temporal-range filter yet"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q060",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "music01",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q061",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "music01",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q062",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "music01",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q063",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "clip04",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q064",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "clip04",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q065",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "clip04",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source_scope": "Actual Searchi hybrid pipeline and native extension filters; temporal-range filter unsupported; segment ends inferred from configured audio/frame policy, exact point timestamps come from actual result API",
|
||||||
|
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
|
||||||
|
}
|
||||||
@@ -0,0 +1,586 @@
|
|||||||
|
{
|
||||||
|
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
|
||||||
|
"worker": {
|
||||||
|
"device": "cpu",
|
||||||
|
"dimensions": 768,
|
||||||
|
"media_supported": true,
|
||||||
|
"model": "google/embeddinggemma-2",
|
||||||
|
"ready": true,
|
||||||
|
"runtime": "b11461-no-logits-v1",
|
||||||
|
"threads": 1
|
||||||
|
},
|
||||||
|
"overall": {
|
||||||
|
"positive_queries": 58,
|
||||||
|
"unsupported_queries": 1,
|
||||||
|
"recall_at1": 0.9655172413793104,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 0.9827586206896551,
|
||||||
|
"temporal_queries": 22,
|
||||||
|
"segment_overlap_hit_at1": 0.8181818181818182,
|
||||||
|
"timestamp_point_hit_at1": 0.5454545454545454,
|
||||||
|
"segment_overlap_hit_at5": 0.8181818181818182,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 15.777000000000001,
|
||||||
|
"no_answer_queries": 6,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"by_category": {
|
||||||
|
"content-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 2,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"exact-filter": {
|
||||||
|
"positive_queries": 1,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"exact-metadata": {
|
||||||
|
"positive_queries": 3,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"filter-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 1,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"image-memory": {
|
||||||
|
"positive_queries": 24,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"metadata-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 1,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"music-memory": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 0.5,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 0.75,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"sound-memory": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"speech-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 2,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"temporal-negative": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 1,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-action": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 4,
|
||||||
|
"segment_overlap_hit_at1": 1.0,
|
||||||
|
"timestamp_point_hit_at1": 1.0,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 0,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-scene": {
|
||||||
|
"positive_queries": 6,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 6,
|
||||||
|
"segment_overlap_hit_at1": 1.0,
|
||||||
|
"timestamp_point_hit_at1": 1.0,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 0,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-speech": {
|
||||||
|
"positive_queries": 8,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 8,
|
||||||
|
"segment_overlap_hit_at1": 0.75,
|
||||||
|
"timestamp_point_hit_at1": 0.0,
|
||||||
|
"segment_overlap_hit_at5": 0.75,
|
||||||
|
"timestamp_distance_median_s": 1.551,
|
||||||
|
"timestamp_distance_max_s": 15.777000000000001,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-temporal": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 4,
|
||||||
|
"segment_overlap_hit_at1": 0.5,
|
||||||
|
"timestamp_point_hit_at1": 0.5,
|
||||||
|
"segment_overlap_hit_at5": 0.5,
|
||||||
|
"timestamp_distance_median_s": 1.005,
|
||||||
|
"timestamp_distance_max_s": 1.005,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"by_language": {
|
||||||
|
"de": {
|
||||||
|
"positive_queries": 27,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 0.9629629629629629,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 0.9814814814814815,
|
||||||
|
"temporal_queries": 11,
|
||||||
|
"segment_overlap_hit_at1": 0.8181818181818182,
|
||||||
|
"timestamp_point_hit_at1": 0.5454545454545454,
|
||||||
|
"segment_overlap_hit_at5": 0.8181818181818182,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 15.777000000000001,
|
||||||
|
"no_answer_queries": 2,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"en": {
|
||||||
|
"positive_queries": 31,
|
||||||
|
"unsupported_queries": 1,
|
||||||
|
"recall_at1": 0.967741935483871,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 0.9838709677419355,
|
||||||
|
"temporal_queries": 11,
|
||||||
|
"segment_overlap_hit_at1": 0.8181818181818182,
|
||||||
|
"timestamp_point_hit_at1": 0.5454545454545454,
|
||||||
|
"segment_overlap_hit_at5": 0.8181818181818182,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 15.777000000000001,
|
||||||
|
"no_answer_queries": 4,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"encoding_failures": [],
|
||||||
|
"encoding_ms": 668040.7295,
|
||||||
|
"query_p50_ms": 92.2329,
|
||||||
|
"query_p95_ms": 113.3751,
|
||||||
|
"failures": [
|
||||||
|
{
|
||||||
|
"query": "q029",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip03",
|
||||||
|
"end_s": 4.005,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_video",
|
||||||
|
"modality": "image",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 2005
|
||||||
|
},
|
||||||
|
"start_s": 2.005
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip03": [
|
||||||
|
[
|
||||||
|
0,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.005
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q030",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip03",
|
||||||
|
"end_s": 4.005,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_video",
|
||||||
|
"modality": "image",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 2005
|
||||||
|
},
|
||||||
|
"start_s": 2.005
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip03": [
|
||||||
|
[
|
||||||
|
0,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.005
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q039",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 30.0,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 0
|
||||||
|
},
|
||||||
|
"start_s": 0.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
1.551,
|
||||||
|
3.603
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.551
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q040",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 30.0,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 0
|
||||||
|
},
|
||||||
|
"start_s": 0.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
1.551,
|
||||||
|
3.603
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.551
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q041",
|
||||||
|
"category": "wrong-media-channel",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 26.167,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_video",
|
||||||
|
"modality": "image",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 24167
|
||||||
|
},
|
||||||
|
"start_s": 24.167
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
13.129,
|
||||||
|
16.316
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q042",
|
||||||
|
"category": "wrong-media-channel",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 14.467,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_video",
|
||||||
|
"modality": "image",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 12467
|
||||||
|
},
|
||||||
|
"start_s": 12.467
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
13.129,
|
||||||
|
16.316
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q043",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 58.0,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 28000
|
||||||
|
},
|
||||||
|
"start_s": 28.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
43.777,
|
||||||
|
52.419
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 15.777000000000001
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q044",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 58.0,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 28000
|
||||||
|
},
|
||||||
|
"start_s": 28.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
43.777,
|
||||||
|
52.419
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 15.777000000000001
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q045",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 80.63,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 56000
|
||||||
|
},
|
||||||
|
"start_s": 56.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
56.539,
|
||||||
|
63.496
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 0.5390000000000015
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q046",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 80.63,
|
||||||
|
"interval_note": "End inferred from configured pipeline window policy, actual API exposes only start timestamp",
|
||||||
|
"match_kind": "semantic_audio",
|
||||||
|
"modality": "audio",
|
||||||
|
"source": {
|
||||||
|
"kind": "timestamp_ms",
|
||||||
|
"number": 56000
|
||||||
|
},
|
||||||
|
"start_s": 56.0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
56.539,
|
||||||
|
63.496
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 0.5390000000000015
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q049",
|
||||||
|
"category": "hard-negative-confusion",
|
||||||
|
"winner": "music01",
|
||||||
|
"expected": [
|
||||||
|
"music02"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q050",
|
||||||
|
"category": "hard-negative-confusion",
|
||||||
|
"winner": "music01",
|
||||||
|
"expected": [
|
||||||
|
"music02"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q059",
|
||||||
|
"category": "unsupported-query-capability",
|
||||||
|
"detail": "Production query API has no temporal-range filter yet"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q060",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "music01",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q061",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "music02",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q062",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "music02",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q063",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "clip04",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q064",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "clip04",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q065",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "clip04",
|
||||||
|
"score": 0.01639344262295082
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source_scope": "Actual Searchi hybrid pipeline and native extension filters; temporal-range filter unsupported; segment ends inferred from configured audio/frame policy, exact point timestamps come from actual result API",
|
||||||
|
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
|
||||||
|
}
|
||||||
@@ -0,0 +1,502 @@
|
|||||||
|
{
|
||||||
|
"fixture_sha256": "b4e465a0eb9582b826eebd2a20797e1680e4907e336d02d0ef2194e2ba0af393",
|
||||||
|
"worker": {
|
||||||
|
"device": "cpu",
|
||||||
|
"dimensions": 768,
|
||||||
|
"model": "google/embeddinggemma-2",
|
||||||
|
"ready": true,
|
||||||
|
"runtime": "b11461-no-logits-v1",
|
||||||
|
"threads": 1
|
||||||
|
},
|
||||||
|
"overall": {
|
||||||
|
"positive_queries": 55,
|
||||||
|
"unsupported_queries": 4,
|
||||||
|
"recall_at1": 0.9636363636363636,
|
||||||
|
"recall_at5": 0.9636363636363636,
|
||||||
|
"mrr": 0.9636363636363636,
|
||||||
|
"temporal_queries": 22,
|
||||||
|
"segment_overlap_hit_at1": 0.9545454545454546,
|
||||||
|
"timestamp_point_hit_at1": 0.6818181818181818,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 13.777000000000001,
|
||||||
|
"no_answer_queries": 6,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"by_category": {
|
||||||
|
"content-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 2,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"exact-filter": {
|
||||||
|
"positive_queries": 1,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"exact-metadata": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 3,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"filter-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 1,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"image-memory": {
|
||||||
|
"positive_queries": 24,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"metadata-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 1,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"music-memory": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 0.5,
|
||||||
|
"recall_at5": 0.5,
|
||||||
|
"mrr": 0.5,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"sound-memory": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"speech-no-answer": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 2,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"temporal-negative": {
|
||||||
|
"positive_queries": 0,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": null,
|
||||||
|
"recall_at5": null,
|
||||||
|
"mrr": null,
|
||||||
|
"temporal_queries": 0,
|
||||||
|
"segment_overlap_hit_at1": null,
|
||||||
|
"timestamp_point_hit_at1": null,
|
||||||
|
"segment_overlap_hit_at5": null,
|
||||||
|
"timestamp_distance_median_s": null,
|
||||||
|
"timestamp_distance_max_s": null,
|
||||||
|
"no_answer_queries": 1,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"video-action": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 4,
|
||||||
|
"segment_overlap_hit_at1": 1.0,
|
||||||
|
"timestamp_point_hit_at1": 1.0,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 0,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-scene": {
|
||||||
|
"positive_queries": 6,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 6,
|
||||||
|
"segment_overlap_hit_at1": 1.0,
|
||||||
|
"timestamp_point_hit_at1": 1.0,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 0,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-speech": {
|
||||||
|
"positive_queries": 8,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 8,
|
||||||
|
"segment_overlap_hit_at1": 1.0,
|
||||||
|
"timestamp_point_hit_at1": 0.25,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 13.129,
|
||||||
|
"timestamp_distance_max_s": 13.777000000000001,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
},
|
||||||
|
"video-temporal": {
|
||||||
|
"positive_queries": 4,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 1.0,
|
||||||
|
"recall_at5": 1.0,
|
||||||
|
"mrr": 1.0,
|
||||||
|
"temporal_queries": 4,
|
||||||
|
"segment_overlap_hit_at1": 0.75,
|
||||||
|
"timestamp_point_hit_at1": 0.75,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 1.0,
|
||||||
|
"no_answer_queries": 0,
|
||||||
|
"no_answer_return_rate": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"by_language": {
|
||||||
|
"de": {
|
||||||
|
"positive_queries": 27,
|
||||||
|
"unsupported_queries": 0,
|
||||||
|
"recall_at1": 0.9629629629629629,
|
||||||
|
"recall_at5": 0.9629629629629629,
|
||||||
|
"mrr": 0.9629629629629629,
|
||||||
|
"temporal_queries": 11,
|
||||||
|
"segment_overlap_hit_at1": 0.9090909090909091,
|
||||||
|
"timestamp_point_hit_at1": 0.6363636363636364,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 13.777000000000001,
|
||||||
|
"no_answer_queries": 2,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
},
|
||||||
|
"en": {
|
||||||
|
"positive_queries": 28,
|
||||||
|
"unsupported_queries": 4,
|
||||||
|
"recall_at1": 0.9642857142857143,
|
||||||
|
"recall_at5": 0.9642857142857143,
|
||||||
|
"mrr": 0.9642857142857143,
|
||||||
|
"temporal_queries": 11,
|
||||||
|
"segment_overlap_hit_at1": 1.0,
|
||||||
|
"timestamp_point_hit_at1": 0.7272727272727273,
|
||||||
|
"segment_overlap_hit_at5": 1.0,
|
||||||
|
"timestamp_distance_median_s": 0,
|
||||||
|
"timestamp_distance_max_s": 13.777000000000001,
|
||||||
|
"no_answer_queries": 4,
|
||||||
|
"no_answer_return_rate": 1.0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"encoding_failures": [
|
||||||
|
{
|
||||||
|
"candidate": {
|
||||||
|
"asset": "music02",
|
||||||
|
"end_s": 30,
|
||||||
|
"modality": "audio",
|
||||||
|
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\music02-audio-0000.f32",
|
||||||
|
"start_s": 0
|
||||||
|
},
|
||||||
|
"error": "Invalid normalized audio PCM"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"encoding_ms": 498657.0532,
|
||||||
|
"query_p50_ms": 92.78150000000001,
|
||||||
|
"query_p95_ms": 122.9277,
|
||||||
|
"failures": [
|
||||||
|
{
|
||||||
|
"query": "q030",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip03",
|
||||||
|
"end_s": 4,
|
||||||
|
"modality": "image",
|
||||||
|
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip03-frame-0002.png",
|
||||||
|
"start_s": 2,
|
||||||
|
"token_budget": 70
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip03": [
|
||||||
|
[
|
||||||
|
0,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q039",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 30,
|
||||||
|
"modality": "audio",
|
||||||
|
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
|
||||||
|
"start_s": 0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
1.551,
|
||||||
|
3.603
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.551
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q040",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 30,
|
||||||
|
"modality": "audio",
|
||||||
|
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
|
||||||
|
"start_s": 0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
1.551,
|
||||||
|
3.603
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 1.551
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q041",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 30,
|
||||||
|
"modality": "audio",
|
||||||
|
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
|
||||||
|
"start_s": 0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
13.129,
|
||||||
|
16.316
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 13.129
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q042",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 30,
|
||||||
|
"modality": "audio",
|
||||||
|
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0000.f32",
|
||||||
|
"start_s": 0
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
13.129,
|
||||||
|
16.316
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 13.129
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q043",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 60,
|
||||||
|
"modality": "audio",
|
||||||
|
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0030.f32",
|
||||||
|
"start_s": 30
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
43.777,
|
||||||
|
52.419
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 13.777000000000001
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q044",
|
||||||
|
"category": "timestamp-localization-miss",
|
||||||
|
"candidate": {
|
||||||
|
"asset": "clip04",
|
||||||
|
"end_s": 60,
|
||||||
|
"modality": "audio",
|
||||||
|
"path": "C:\\Users\\karl\\searchi\\fixtures\\media\\derived\\clip04-audio-0030.f32",
|
||||||
|
"start_s": 30
|
||||||
|
},
|
||||||
|
"expected_intervals": {
|
||||||
|
"clip04": [
|
||||||
|
[
|
||||||
|
43.777,
|
||||||
|
52.419
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timestamp_distance_s": 13.777000000000001
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q049",
|
||||||
|
"category": "hard-negative-confusion",
|
||||||
|
"winner": "music01",
|
||||||
|
"expected": [
|
||||||
|
"music02"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q050",
|
||||||
|
"category": "hard-negative-confusion",
|
||||||
|
"winner": "music01",
|
||||||
|
"expected": [
|
||||||
|
"music02"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q055",
|
||||||
|
"category": "unsupported-query-capability",
|
||||||
|
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q056",
|
||||||
|
"category": "unsupported-query-capability",
|
||||||
|
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q057",
|
||||||
|
"category": "unsupported-query-capability",
|
||||||
|
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q059",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "clip01",
|
||||||
|
"score": 0.750250822387441
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q060",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "music01",
|
||||||
|
"score": 0.5716409940327322
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q061",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "music01",
|
||||||
|
"score": 0.6223961635527826
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q062",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "music01",
|
||||||
|
"score": 0.6181057666050869
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q063",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "clip04",
|
||||||
|
"score": 0.6229413326862353
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q064",
|
||||||
|
"category": "no-answer-nearest-neighbor-return",
|
||||||
|
"winner": "clip03",
|
||||||
|
"score": 0.5715326739972073
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"query": "q065",
|
||||||
|
"category": "unsupported-query-capability",
|
||||||
|
"detail": "Exact metadata requires production lexical lane; direct audio embeddings cannot verify artist/title identities"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source_scope": "Direct real-model dense reference baseline; candidate filters here are benchmark reference constraints, not production-filter certification; no rejection threshold is tuned",
|
||||||
|
"interpretation": "Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality."
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
.venv/
|
||||||
|
target/
|
||||||
|
_*.py
|
||||||
|
__pycache__/
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
[package]
|
||||||
|
name = "searchi-media-eval"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
publish = false
|
||||||
|
|
||||||
|
[workspace]
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
anyhow = "1"
|
||||||
|
serde_json = "1"
|
||||||
|
sha2 = "0.10"
|
||||||
|
searchi-semantic = { path = "../../crates/searchi-semantic" }
|
||||||
|
searchi-core = { path = "../../crates/searchi-core" }
|
||||||
|
searchi-service = { path = "../../crates/searchi-service" }
|
||||||
|
searchi-media = { path = "../../crates/searchi-media" }
|
||||||
|
tempfile = "3"
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# Real-media quality evaluation
|
||||||
|
|
||||||
|
This independent tool owns no production/UI code. It feeds real pixels and decoded PCM to the actual Google CPU worker, keeps frozen source labels outside inference input, and reports ranking/localization failures.
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$env:SEARCHI_FORCE_CPU = '1'
|
||||||
|
python tools/media-eval/download.py
|
||||||
|
python tools/media-eval/prepare.py --ffmpeg .searchi/media-runtime/bin/ffmpeg.exe --ffprobe .searchi/media-runtime/bin/ffprobe.exe
|
||||||
|
cargo run --manifest-path tools/media-eval/Cargo.toml --bin searchi-media-eval -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe fixtures/media/reference-results.json
|
||||||
|
python tools/media-eval/score.py fixtures/media/reference-results.json fixtures/media/reference-summary.json
|
||||||
|
cargo run --manifest-path tools/media-eval/Cargo.toml --bin metadata -- fixtures/media .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe fixtures/media/metadata-results.json
|
||||||
|
python tools/media-eval/score.py fixtures/media/metadata-results.json fixtures/media/metadata-summary.json
|
||||||
|
cargo run --manifest-path tools/media-eval/Cargo.toml --bin pipeline -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe fixtures/media/pipeline-30s-results.json 30
|
||||||
|
cargo run --manifest-path tools/media-eval/Cargo.toml --bin pipeline -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe .searchi/media-runtime/bin/ffmpeg.exe .searchi/media-runtime/bin/ffprobe.exe fixtures/media/pipeline-10s-results.json 10
|
||||||
|
python tools/media-eval/score.py fixtures/media/pipeline-30s-results.json fixtures/media/pipeline-30s-summary.json
|
||||||
|
python tools/media-eval/score.py fixtures/media/pipeline-10s-results.json fixtures/media/pipeline-10s-summary.json
|
||||||
|
cargo run --manifest-path tools/media-eval/Cargo.toml --bin image_budgets -- fixtures/media .searchi/models/embeddinggemma2 .searchi/runtime/cpu/Release/searchi-gemma-worker.exe fixtures/media/image-budget-results.json
|
||||||
|
python tools/media-eval/score_budgets.py fixtures/media/image-budget-results.json fixtures/media/image-budget-summary.json
|
||||||
|
python -m unittest discover -s tools/media-eval -p test_*.py
|
||||||
|
```
|
||||||
|
|
||||||
|
The standalone Cargo workspace keeps evaluation dependencies separate. Native inference and FFmpeg decoding use CPU only and one thread. No test enables a GPU provider. The model/worker/decoder paths above refer to explicitly provisioned local developer artifacts; they are not end-user setup requirements.
|
||||||
|
|
||||||
|
The dense reference samples video every two seconds and PCM in thirty-second windows. Photos use 140 image tokens and frames 70. The separate comparison fixes the same 12 photos/24 queries and varies only photo token budgets **70, 280, 1120**. It does not change labels or choose a threshold after observing scores.
|
||||||
|
|
||||||
|
## Interpretation
|
||||||
|
|
||||||
|
- Recall@1/@5 and MRR rank distinct source files, choosing each file's best segment. Per-category/per-language metrics remain separate.
|
||||||
|
- Segment overlap credits a returned segment that actually intersects the labelled event in the correct file. Point hits require the returned start timestamp itself to fall inside the event. A thirty-second speech window may pass overlap while opening far too early; both metrics are required.
|
||||||
|
- No-answer queries remain separate from positive recall. The tool exposes returned neighbors and scores without inventing an abstention threshold. A reject threshold needs independent calibration and held-out validation.
|
||||||
|
- Direct model candidate filters are **reference pool constraints**, not evidence that production filters work. Artist/title identity cannot be recovered reliably from a performance embedding; exact metadata rows are explicitly unsupported until evaluated through Searchi's real lexical metadata lane.
|
||||||
|
- `metadata` tests the actual FFprobe/guarded core FTS lane; `pipeline` tests the actual unified Searchi media generation and hybrid ranking. The optional final argument varies audio window seconds (30 versus 10) while retaining two-second overlap, the same real files, decoder and frozen labels. It does not change production defaults.
|
||||||
|
- Model agreement with another implementation, synthetic document-seed scores, and this small-media benchmark each answer different questions. None is production quality certification.
|
||||||
|
|
||||||
|
Detailed provenance is in [the fixture README](../../fixtures/media/README.md). The final assessment belongs in [MEDIA-QUALITY.md](../../docs/MEDIA-QUALITY.md).
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
"""Reacquire the locked licensed corpus. Updated bytes are never accepted."""
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--root", type=Path, default=Path("fixtures/media"))
|
||||||
|
args = parser.parse_args()
|
||||||
|
root = args.root.resolve()
|
||||||
|
fixture_path = root / "fixture.json"
|
||||||
|
lock = json.loads((root / "labels.lock.json").read_text())
|
||||||
|
assert hashlib.sha256(fixture_path.read_bytes()).hexdigest() == lock["fixture_sha256"]
|
||||||
|
for asset in json.loads(fixture_path.read_text(encoding="utf-8"))["assets"]:
|
||||||
|
destination = (root / asset["path"]).resolve()
|
||||||
|
assert destination.is_relative_to(root)
|
||||||
|
if destination.is_file() and hashlib.sha256(destination.read_bytes()).hexdigest() == asset["sha256"]:
|
||||||
|
continue
|
||||||
|
request = urllib.request.Request(asset["source_url"], headers={"User-Agent": "SearchiMediaEvaluation/1.0"})
|
||||||
|
with urllib.request.urlopen(request, timeout=60) as response:
|
||||||
|
data = response.read(asset["bytes"] + 1)
|
||||||
|
assert len(data) == asset["bytes"] and hashlib.sha256(data).hexdigest() == asset["sha256"], f"Publisher changed {asset['id']}; keep frozen corpus, investigate a new version"
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
destination.write_bytes(data)
|
||||||
|
print("Verified", asset["id"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""Prepare real CPU-decoded media candidates after verifying frozen labels.
|
||||||
|
|
||||||
|
No generated captions, placeholder vectors, or model calls occur in preparation.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
def verify(root):
|
||||||
|
fixture_path = root / "fixture.json"
|
||||||
|
lock = json.loads((root / "labels.lock.json").read_text())
|
||||||
|
assert hashlib.sha256(fixture_path.read_bytes()).hexdigest() == lock["fixture_sha256"], "Frozen labels changed"
|
||||||
|
fixture = json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||||
|
for asset in fixture["assets"]:
|
||||||
|
path = (root / asset["path"]).resolve()
|
||||||
|
assert path.is_relative_to(root.resolve()), "Asset escaped corpus root"
|
||||||
|
assert path.stat().st_size == asset["bytes"], asset["id"]
|
||||||
|
assert hashlib.sha256(path.read_bytes()).hexdigest() == asset["sha256"], asset["id"]
|
||||||
|
return fixture
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--root", type=Path, default=Path("fixtures/media"))
|
||||||
|
parser.add_argument("--ffmpeg", type=Path, required=True)
|
||||||
|
parser.add_argument("--ffprobe", type=Path, required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
root = args.root.resolve()
|
||||||
|
fixture = verify(root)
|
||||||
|
derived = root / "derived"
|
||||||
|
derived.mkdir(exist_ok=True)
|
||||||
|
candidates = []
|
||||||
|
for asset in fixture["assets"]:
|
||||||
|
source = root / asset["path"]
|
||||||
|
if asset["kind"] == "image":
|
||||||
|
candidates.append(dict(asset=asset["id"], path=str(source), modality="image", token_budget=140))
|
||||||
|
continue
|
||||||
|
probe = json.loads(subprocess.check_output([str(args.ffprobe), "-v", "error", "-show_format", "-show_streams", "-of", "json", str(source)], text=True))
|
||||||
|
duration = float(probe["format"]["duration"])
|
||||||
|
if asset["kind"] == "video":
|
||||||
|
for stamp in range(0, int(duration) + 1, 2):
|
||||||
|
output = derived / f"{asset['id']}-frame-{stamp:04}.png"
|
||||||
|
subprocess.run([str(args.ffmpeg), "-hide_banner", "-loglevel", "error", "-hwaccel", "none", "-threads", "1", "-ss", str(stamp), "-i", str(source), "-frames:v", "1", "-vf", "scale=480:-1", "-threads", "1", "-y", str(output)], check=True)
|
||||||
|
if output.is_file():
|
||||||
|
candidates.append(dict(asset=asset["id"], path=str(output), modality="image", token_budget=70, start_s=stamp, end_s=min(stamp + 2, duration)))
|
||||||
|
if any(stream["codec_type"] == "audio" for stream in probe["streams"]):
|
||||||
|
for stamp in range(0, int(duration) + 1, 30):
|
||||||
|
output = derived / f"{asset['id']}-audio-{stamp:04}.f32"
|
||||||
|
subprocess.run([str(args.ffmpeg), "-hide_banner", "-loglevel", "error", "-hwaccel", "none", "-threads", "1", "-ss", str(stamp), "-i", str(source), "-t", "30", "-vn", "-ac", "1", "-ar", "16000", "-f", "f32le", "-threads", "1", "-y", str(output)], check=True)
|
||||||
|
if output.stat().st_size:
|
||||||
|
assert output.stat().st_size <= 480000 * 4, "Audio exceeds worker budget"
|
||||||
|
candidates.append(dict(asset=asset["id"], path=str(output), modality="audio", start_s=stamp, end_s=min(stamp + 30, duration)))
|
||||||
|
manifest = dict(fixture_sha256=hashlib.sha256((root / "fixture.json").read_bytes()).hexdigest(), ffmpeg_version=subprocess.check_output([str(args.ffmpeg), "-version"], text=True).splitlines()[0], policy="CPU-only dense reference: photos140/video70 tokens, uniform2s frames, mono16kHz30s PCM", candidates=candidates)
|
||||||
|
(derived / "candidates.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
||||||
|
print(json.dumps(dict(candidates=len(candidates), images=sum(item["modality"] == "image" for item in candidates), audio=sum(item["modality"] == "audio" for item in candidates))))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
"""Score frozen relevance without fitting an abstention threshold."""
|
||||||
|
import argparse
|
||||||
|
from collections import defaultdict
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def temporal_hit(candidate, query, point=False):
|
||||||
|
asset = candidate.get("asset")
|
||||||
|
if asset not in query.get("expected", []):
|
||||||
|
return False
|
||||||
|
if query.get("required_modality") and candidate.get("modality") != query["required_modality"]:
|
||||||
|
return False
|
||||||
|
ranges = query.get("intervals", {}).get(asset, [])
|
||||||
|
start = candidate.get("start_s")
|
||||||
|
end = candidate.get("end_s", start)
|
||||||
|
if start is None:
|
||||||
|
return False
|
||||||
|
if point:
|
||||||
|
return any(left <= start <= right for left, right in ranges)
|
||||||
|
return any(min(end, right) > max(start, left) for left, right in ranges)
|
||||||
|
|
||||||
|
|
||||||
|
def timestamp_distance(candidate, query):
|
||||||
|
"""Raw seconds from returned start to the nearest labelled event interval."""
|
||||||
|
if candidate.get("asset") not in query.get("expected", []) or candidate.get("start_s") is None:
|
||||||
|
return None
|
||||||
|
if query.get("required_modality") and candidate.get("modality") != query["required_modality"]:
|
||||||
|
return None
|
||||||
|
ranges = query.get("intervals", {}).get(candidate["asset"], [])
|
||||||
|
stamp = candidate["start_s"]
|
||||||
|
return min((max(left - stamp, stamp - right, 0) for left, right in ranges), default=None)
|
||||||
|
|
||||||
|
|
||||||
|
def metrics(rows):
|
||||||
|
positive = [row for row in rows if row["query"]["expected"] and not row.get("unsupported")]
|
||||||
|
negative = [row for row in rows if not row["query"]["expected"] and not row.get("unsupported")]
|
||||||
|
temporal = [row for row in positive if row["query"]["category"].startswith("video-")]
|
||||||
|
distances = sorted(distance for row in temporal if row["file_rankings"] and (distance := timestamp_distance(row["file_rankings"][0]["candidate"], row["query"])) is not None)
|
||||||
|
ranks = []
|
||||||
|
for row in positive:
|
||||||
|
expected = row["query"]["expected"]
|
||||||
|
ranks.append(next((i + 1 for i, hit in enumerate(row["file_rankings"]) if hit["candidate"]["asset"] in expected), None))
|
||||||
|
fraction = lambda count, total: count / total if total else None
|
||||||
|
return dict(
|
||||||
|
positive_queries=len(positive),
|
||||||
|
unsupported_queries=sum(bool(row.get("unsupported")) for row in rows),
|
||||||
|
recall_at1=fraction(sum(rank == 1 for rank in ranks), len(ranks)),
|
||||||
|
recall_at5=fraction(sum(rank is not None and rank <= 5 for rank in ranks), len(ranks)),
|
||||||
|
mrr=fraction(sum(1 / rank if rank else 0 for rank in ranks), len(ranks)),
|
||||||
|
temporal_queries=len(temporal),
|
||||||
|
segment_overlap_hit_at1=fraction(sum(bool(row["file_rankings"]) and temporal_hit(row["file_rankings"][0]["candidate"], row["query"]) for row in temporal), len(temporal)),
|
||||||
|
timestamp_point_hit_at1=fraction(sum(bool(row["file_rankings"]) and temporal_hit(row["file_rankings"][0]["candidate"], row["query"], point=True) for row in temporal), len(temporal)),
|
||||||
|
segment_overlap_hit_at5=fraction(sum(any(temporal_hit(hit["candidate"], row["query"]) for hit in row["segment_top5"]) for row in temporal), len(temporal)),
|
||||||
|
timestamp_distance_median_s=distances[len(distances) // 2] if distances else None,
|
||||||
|
timestamp_distance_max_s=max(distances) if distances else None,
|
||||||
|
no_answer_queries=len(negative),
|
||||||
|
no_answer_return_rate=fraction(sum(bool(row["file_rankings"]) for row in negative), len(negative)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def summarize(report):
|
||||||
|
rows = report["rows"]
|
||||||
|
by_category = defaultdict(list)
|
||||||
|
by_language = defaultdict(list)
|
||||||
|
failures = []
|
||||||
|
for row in rows:
|
||||||
|
query = row["query"]
|
||||||
|
by_category[query["category"]].append(row)
|
||||||
|
by_language[query["language"]].append(row)
|
||||||
|
if row.get("unsupported"):
|
||||||
|
failures.append(dict(query=query["id"], category="unsupported-query-capability", detail=row["unsupported"]))
|
||||||
|
continue
|
||||||
|
hits = row["file_rankings"]
|
||||||
|
if not query["expected"]:
|
||||||
|
if hits:
|
||||||
|
failures.append(dict(query=query["id"], category="no-answer-nearest-neighbor-return", winner=hits[0]["candidate"]["asset"], score=hits[0]["score"]))
|
||||||
|
elif not hits or hits[0]["candidate"]["asset"] not in query["expected"]:
|
||||||
|
winner = hits[0]["candidate"]["asset"] if hits else None
|
||||||
|
failures.append(dict(query=query["id"], category="hard-negative-confusion" if winner in query.get("hard_negatives", []) else "retrieval-top1-miss", winner=winner, expected=query["expected"]))
|
||||||
|
elif query["category"].startswith("video-") and not temporal_hit(hits[0]["candidate"], query, point=True):
|
||||||
|
candidate = hits[0]["candidate"]
|
||||||
|
wrong_channel = query.get("required_modality") and candidate.get("modality") != query["required_modality"]
|
||||||
|
failures.append(dict(query=query["id"], category="wrong-media-channel" if wrong_channel else "timestamp-localization-miss", candidate=candidate, expected_intervals=query["intervals"], timestamp_distance_s=timestamp_distance(candidate, query)))
|
||||||
|
durations = sorted(row["query_ms"] for row in rows if "query_ms" in row)
|
||||||
|
return dict(
|
||||||
|
fixture_sha256=report["fixture_sha256"], worker=report["worker"],
|
||||||
|
overall=metrics(rows), by_category={key: metrics(value) for key, value in sorted(by_category.items())},
|
||||||
|
by_language={key: metrics(value) for key, value in sorted(by_language.items())},
|
||||||
|
encoding_failures=report["encoding_failures"], encoding_ms=report["encoding_ms"],
|
||||||
|
query_p50_ms=durations[len(durations) // 2] if durations else None,
|
||||||
|
query_p95_ms=durations[min(len(durations) * 95 // 100, len(durations) - 1)] if durations else None,
|
||||||
|
failures=failures,
|
||||||
|
source_scope=report.get("scope", "Direct dense model baseline"),
|
||||||
|
interpretation="Small independently labelled public-media diagnostic; source_scope identifies the measured lane. No-answer return rate measures unsuppressed output, not a calibrated confidence policy. Audio interval overlap can pass while the returned start timestamp is too early; point hits expose this. This corpus alone cannot certify production quality.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("report", type=Path)
|
||||||
|
parser.add_argument("output", type=Path)
|
||||||
|
args = parser.parse_args()
|
||||||
|
report = json.loads(args.report.read_text(encoding="utf-8"))
|
||||||
|
fixture = Path("fixtures/media/fixture.json")
|
||||||
|
assert report["fixture_sha256"] == hashlib.sha256(fixture.read_bytes()).hexdigest()
|
||||||
|
summary = summarize(report)
|
||||||
|
args.output.write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||||||
|
print(json.dumps(summary["overall"]))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||