From 6a5a2dc40d69926edb41664a74f9414dedc73c8f Mon Sep 17 00:00:00 2001 From: karl Date: Tue, 13 Oct 2026 07:06:39 +0200 Subject: [PATCH] Freeze mixed-file retrieval diagnostic and shared-engine evaluator --- docs/MIXED-QUALITY.md | 32 + fixtures/mixed/LLAMA-LICENSE.txt | 21 + fixtures/mixed/README.md | 34 + fixtures/mixed/fixture.json | 1830 +++++++++++++++++ fixtures/mixed/labels.lock.json | 5 + .../apps__desktop__src__search-performance.ts | 230 +++ .../apps__desktop__src__selection-state.ts | 24 + .../crates__searchi-core__src__extraction.rs | 320 +++ ...tes__searchi-service__src__coordination.rs | 417 ++++ .../crates__searchi-service__src__routing.rs | 218 ++ ...earchi-service__src__store__incremental.rs | 967 +++++++++ .../mixed/sources/docs__AUTOMATIC-INDEXING.md | 100 + fixtures/mixed/sources/docs__CODE-SEARCH.md | 80 + .../sources/docs__INCREMENTAL-SEMANTIC.md | 115 ++ fixtures/mixed/sources/docs__PERFORMANCE.md | 219 ++ fixtures/mixed/sources/llama__ggml-alloc.c | 1173 +++++++++++ fixtures/mixed/sources/llama__unicode.cpp | 1564 ++++++++++++++ tools/mixed-eval/README.md | 35 + tools/mixed-eval/evaluate.py | 156 ++ tools/mixed-eval/test_evaluate.py | 57 + 20 files changed, 7597 insertions(+) create mode 100644 docs/MIXED-QUALITY.md create mode 100644 fixtures/mixed/LLAMA-LICENSE.txt create mode 100644 fixtures/mixed/README.md create mode 100644 fixtures/mixed/fixture.json create mode 100644 fixtures/mixed/labels.lock.json create mode 100644 fixtures/mixed/sources/apps__desktop__src__search-performance.ts create mode 100644 fixtures/mixed/sources/apps__desktop__src__selection-state.ts create mode 100644 fixtures/mixed/sources/crates__searchi-core__src__extraction.rs create mode 100644 fixtures/mixed/sources/crates__searchi-service__src__coordination.rs create mode 100644 fixtures/mixed/sources/crates__searchi-service__src__routing.rs create mode 100644 fixtures/mixed/sources/crates__searchi-service__src__store__incremental.rs create mode 100644 fixtures/mixed/sources/docs__AUTOMATIC-INDEXING.md create mode 100644 fixtures/mixed/sources/docs__CODE-SEARCH.md create mode 100644 fixtures/mixed/sources/docs__INCREMENTAL-SEMANTIC.md create mode 100644 fixtures/mixed/sources/docs__PERFORMANCE.md create mode 100644 fixtures/mixed/sources/llama__ggml-alloc.c create mode 100644 fixtures/mixed/sources/llama__unicode.cpp create mode 100644 tools/mixed-eval/README.md create mode 100644 tools/mixed-eval/evaluate.py create mode 100644 tools/mixed-eval/test_evaluate.py diff --git a/docs/MIXED-QUALITY.md b/docs/MIXED-QUALITY.md new file mode 100644 index 0000000..97be64b --- /dev/null +++ b/docs/MIXED-QUALITY.md @@ -0,0 +1,32 @@ +# Mixed retrieval quality gate + +A frozen mixed diagnostic is prepared in [fixtures/mixed](../fixtures/mixed/README.md). +It addresses the missing comparison between natural-language All-files queries +and explicit Code scope while images, music, speech, videos and actual documents +compete. It contains 32 real files and 91 English/German queries, including two +pinned upstream llama.cpp implementations. The existing media labels are reused +with their licenses and attribution. They are not an untouched held-out set. + +**No actual mixed inference results are recorded yet.** The frozen manifest +SHA-256 is `f83d27227582f614159ff153654d5bb5a47bf74998141e389557f3950ea0b924`. +Asset/hash verification and six meaningful scoring/provenance tests passed. +These prove harness behavior only; they establish no retrieval accuracy. + +The [evaluator](../tools/mixed-eval/README.md) calls the shared production engine +through CLI indexing and MCP search. It compares lexical/hybrid retrieval and +All-files/Code scope without changing ranking or tuning a cutoff against these +labels. File-level recall and no-answer returned-result rates remain separate +from spoken timestamp localization. Raw evidence must preserve actual failures, +encoding errors and unsupported filters. + +Remaining work: + +- Run the frozen corpus with the verified Google model and CPU runtime in a + coordinated slot; record exact CLI/worker hashes and actual raw results. +- Inspect failures by modality, language, requested artifact type and scope; + distinguish correct-file retrieval from a useful passage/timestamp. +- Author a larger independently reviewed held-out corpus before ranking changes; + frozen diagnostics must not become the sole optimization target. +- Establish recall and no-answer acceptance targets on representative user + libraries, unrelated repositories and longer recordings. Tiny diagnostics + cannot close the production relevance gate. diff --git a/fixtures/mixed/LLAMA-LICENSE.txt b/fixtures/mixed/LLAMA-LICENSE.txt new file mode 100644 index 0000000..e7dca55 --- /dev/null +++ b/fixtures/mixed/LLAMA-LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023-2026 The ggml authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/fixtures/mixed/README.md b/fixtures/mixed/README.md new file mode 100644 index 0000000..bdd8ac8 --- /dev/null +++ b/fixtures/mixed/README.md @@ -0,0 +1,34 @@ +# Mixed All-files diagnostic v1 + +This is a frozen, small diagnostic, **not a production quality score**. No mixed +model run has been performed yet. + +The corpus contains 32 actual files: six searchi implementations, four searchi +technical documents, two complete pinned llama.cpp source files, and twenty +existing licensed media files. Source snapshots retain exact original bytes; +`fixture.json` records upstream paths, commits, SHA-256, sizes, and licenses. +The llama.cpp copies come from committed upstream objects, excluding local +runtime patches. Its MIT notice is in `LLAMA-LICENSE.txt`. Media files are +referenced from `../media/assets` rather than duplicated. Their source URLs, +licenses, independently observed labels and attribution remain in +[the media fixture](../media/README.md). + +There are 91 frozen queries: English/German implementation and document pairs, +65 reused media queries, four additional absent-content queries and four pinned +upstream implementation queries. Code queries run in both All files and Code +scope. Documents about the implementation coexist with the code, providing +realistic competing content. Speech, music, images and videos compete with +both. The case asking for code that chooses speech search is intentional: it +can expose audio-intent routing that suppresses a relevant implementation. + +Labels were written by one annotator before **mixed** inference. Existing media +results had already been observed in their separate evaluation, so this is not +an untouched held-out media benchmark. searchi sources are the application's +own repository; the two llama.cpp files add external code, but do not establish +quality across unrelated real projects. Never adjust v1 labels after seeing +mixed results. Corrections require a new fixture version, explicit reasoning, +and a new pre-inference lock. A second blinded human review remains open. + +Run `python tools/mixed-eval/evaluate.py --verify-only` to verify labels and all +source/media bytes without invoking a model. See +[the evaluator instructions](../../tools/mixed-eval/README.md) for real runs. diff --git a/fixtures/mixed/fixture.json b/fixtures/mixed/fixture.json new file mode 100644 index 0000000..edf9c32 --- /dev/null +++ b/fixtures/mixed/fixture.json @@ -0,0 +1,1830 @@ +{ + "version": 1, + "label_policy": "Frozen before any mixed-model run. One annotator, source implementation inspections plus reused independently frozen media labels. Previously observed media outcomes mean this is a new mixed diagnostic, not an untouched held-out media benchmark. Never change v1 labels after mixed rankings.", + "assets": [ + { + "id": "source01", + "path": "sources/crates__searchi-service__src__coordination.rs", + "kind": "code", + "source_path": "crates/searchi-service/src/coordination.rs", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "481b4a33fdba844bd0677dc85a273eb21d10c9fd0667086215dd206972aef827", + "bytes": 15233, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "source02", + "path": "sources/crates__searchi-service__src__routing.rs", + "kind": "code", + "source_path": "crates/searchi-service/src/routing.rs", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "613299748f08248f6f9be694f19698434029aa8f4440fbf1a5618730314abbd6", + "bytes": 6647, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "source03", + "path": "sources/crates__searchi-core__src__extraction.rs", + "kind": "code", + "source_path": "crates/searchi-core/src/extraction.rs", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "269ba55dd648d2ad396488a30ace3d670b149969506f04387fd89b0b4ed1372a", + "bytes": 10899, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "source04", + "path": "sources/apps__desktop__src__selection-state.ts", + "kind": "code", + "source_path": "apps/desktop/src/selection-state.ts", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "9793541bc5da0ad1fd27e74ab9e5ad646242a717cebe0fb0e058a23d1959befd", + "bytes": 705, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "source05", + "path": "sources/apps__desktop__src__search-performance.ts", + "kind": "code", + "source_path": "apps/desktop/src/search-performance.ts", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "9e07ef01da623b34d3dc9f36f68488f19422b425bde3f98f9031a4dc67646fe1", + "bytes": 7198, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "source06", + "path": "sources/crates__searchi-service__src__store__incremental.rs", + "kind": "code", + "source_path": "crates/searchi-service/src/store/incremental.rs", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "597b8c30f7f86be13f36b16f8e2db992ee56cb6591532fb1b9e3be24dc7e60f0", + "bytes": 45030, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "source07", + "path": "sources/docs__PERFORMANCE.md", + "kind": "document", + "source_path": "docs/PERFORMANCE.md", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "a5147832daf8c9c1d51e297b6498e4627e069ab9de58862c9baa3e51b8566321", + "bytes": 12757, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "source08", + "path": "sources/docs__INCREMENTAL-SEMANTIC.md", + "kind": "document", + "source_path": "docs/INCREMENTAL-SEMANTIC.md", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "0291ab5cabce5ef0cb503dadc39696b6bb9198d17247673a24ce6c5d3f60cb9a", + "bytes": 6883, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "source09", + "path": "sources/docs__AUTOMATIC-INDEXING.md", + "kind": "document", + "source_path": "docs/AUTOMATIC-INDEXING.md", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "9de756e96c04a9d82915c78d851c928b375a9b6c044f672e6d23a487b0e15fa8", + "bytes": 6106, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "source10", + "path": "sources/docs__CODE-SEARCH.md", + "kind": "document", + "source_path": "docs/CODE-SEARCH.md", + "source_commit": "bba34de7aa84fe53501d938e5a50021476dabae4", + "sha256": "de3f230d33403ecc8627b18c048a0ca406dbec25c969b40c857f9fddcb79b3e9", + "bytes": 4216, + "license": "MIT; repository LICENSE", + "changes": "Exact checked-out bytes, no edits" + }, + { + "id": "image01", + "path": "../media/assets/image01.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/126407/pexels-photo-126407.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/126407/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Long-haired tabby cat lying flat indoors, eyes toward camera.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 27390, + "sha256": "c0b2ebddafcff4cd60ae1d2f1b8e53d4e0f11643ac53d0113605f333523b21ed" + }, + { + "id": "image02", + "path": "../media/assets/image02.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/1099680/pexels-photo-1099680.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/1099680/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Fruit bowl with raspberries, mango, blackberries, strawberries and almonds, top down.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 68905, + "sha256": "72b46006e7f55dcad5bec5af977171a369917030cf2ff99e544806af939ffec1" + }, + { + "id": "image03", + "path": "../media/assets/image03.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/459225/pexels-photo-459225.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/459225/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Jagged mountains and fir trees reflected in a daylight lake.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 34309, + "sha256": "c4abbb061e6fcbcef25d36d7bb7f24018c0c768b23176fb2a77a9eb8f5b86f21" + }, + { + "id": "image04", + "path": "../media/assets/image04.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/355465/pexels-photo-355465.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/355465/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Dense blue star field and Milky Way above a dark mountain landscape.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 37735, + "sha256": "8999f22131c22ee1c29d79dda352c2654131eb1b4f61ba7f8b8d21dbbc5cc791" + }, + { + "id": "image05", + "path": "../media/assets/image05.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/414612/pexels-photo-414612.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/414612/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Wooden pier into still lake at night under a bright moon.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 51106, + "sha256": "1cfd9d9c3dea4ec4e1733c55424eef93983bce1167f60777768cae6c549fd889" + }, + { + "id": "image06", + "path": "../media/assets/image06.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/1323550/pexels-photo-1323550.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/1323550/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Layered purple mountains under a yellow-orange sunset, tiny person on ridge.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 27259, + "sha256": "91947e87067c923db2e560f67cdd0c1edec55c9bb0fbab6a83ee7c7adf88860d" + }, + { + "id": "image07", + "path": "../media/assets/image07.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/841303/pexels-photo-841303.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/841303/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Cows grazing in green rolling hills.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 50633, + "sha256": "0b0cc8c211680d19ebdaf2b797af430120c4e65f951143bbe06544eaa0691942" + }, + { + "id": "image08", + "path": "../media/assets/image08.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/210019/pexels-photo-210019.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/210019/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Red sports coupe on a wooded road, motion-blurred background.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 65362, + "sha256": "12b5fd662c054d1337e4adf733751da3e32c1fd40f88dfccc0bd73af4f620369" + }, + { + "id": "image09", + "path": "../media/assets/image09.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/1640777/pexels-photo-1640777.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/1640777/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Bowl of quinoa and vegetables, sliced carrots and zucchini on pale table.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 68469, + "sha256": "6112af5f9ed170d91c7f8e1e5de95142c0190e2e2f8fbbbccf2f106553371817" + }, + { + "id": "image10", + "path": "../media/assets/image10.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/302899/pexels-photo-302899.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/302899/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Milk pouring from metal jug into cup of coffee with latte art.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 23609, + "sha256": "632926d3bd0f4691ae7fd0a8e2cb8b455e0252511eebf322e91c64c17cdf0e2a" + }, + { + "id": "image11", + "path": "../media/assets/image11.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/248797/pexels-photo-248797.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/248797/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Turquoise tropical sea and white beach fringed by green vegetation.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 27716, + "sha256": "c02ad4a8bbe8091cbc3a9a0abc602a905cb46056efa94141950f3af1c749c038" + }, + { + "id": "image12", + "path": "../media/assets/image12.jpg", + "kind": "image", + "source_url": "https://images.pexels.com/photos/167964/pexels-photo-167964.jpeg?auto=compress&cs=tinysrgb&w=640", + "source_page": "https://www.pexels.com/photo/167964/", + "license": "Pexels License", + "license_url": "https://www.pexels.com/license/", + "changes": "Publisher-provided 640px compressed derivative, no local pixel changes.", + "observation": "Black and white silhouette of a person reaching a hand toward camera.", + "independent_inspection": "view_image before any media embedding inference", + "bytes": 14256, + "sha256": "dd3bc409128a8702522bac7b953a5eb691662eebf21ac3ba0b3b15c70f86d54b" + }, + { + "id": "clip01", + "path": "../media/assets/clip01.webm", + "kind": "video", + "duration_s": 22.204, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/c/cc/Cat_opening_door.webm", + "source_page": "https://commons.wikimedia.org/wiki/File:Cat_opening_door.webm", + "license": "CC-BY-SA-4.0", + "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", + "author": "Astropulse", + "changes": "None; downloaded original file.", + "observation": "Orange tabby cat twice jumps against white door near its lever, then crouches.", + "independent_inspection": "Real frames viewed before inference; NASA speech labels use publisher SRT, never model outputs.", + "bytes": 2253125, + "sha256": "1a1f420559b306574ba8a04aee0175c38c2e20815b1bd46a45aae86d17f3ad53" + }, + { + "id": "clip02", + "path": "../media/assets/clip02.webm", + "kind": "video", + "duration_s": 13.81, + "source_url": "https://upload.wikimedia.org/wikipedia/commons/c/c9/Water_flowing_strem.webm", + "source_page": "https://commons.wikimedia.org/wiki/File:Water_flowing_strem.webm", + "license": "CC0-1.0", + "license_url": "https://creativecommons.org/publicdomain/zero/1.0/", + "author": "BhotaRohit", + "changes": "None; downloaded original file.", + "observation": "Tall waterfall descending misty rocky hillside; a tiny person appears near its base.", + "independent_inspection": "Real frames viewed before inference; NASA speech labels use publisher SRT, never model outputs.", + "bytes": 6001205, + "sha256": "5652e6490bea85dfb7a1da05e00fc7f3d8cc2cbd54790c03a48223f2256bc3d1" + }, + { + "id": "clip03", + "path": "../media/assets/clip03.webm", + "kind": "video", + "duration_s": 5.056, + "source_url": "https://interactive-examples.mdn.mozilla.net/media/cc0-videos/flower.webm", + "source_page": "https://github.com/mdn/interactive-examples", + "license": "CC0-1.0", + "license_url": "https://raw.githubusercontent.com/mdn/interactive-examples/main/LICENSE", + "author": "MDN contributors", + "changes": "None; downloaded original file.", + "observation": "Timelapse of closed red flower bud opening among green purple-edged leaves.", + "independent_inspection": "Real frames viewed before inference; NASA speech labels use publisher SRT, never model outputs.", + "bytes": 554058, + "sha256": "c6f8a348953395598a9a73b9bab1676436410797bce9f398f4be1531d6e76dda" + }, + { + "id": "clip04", + "path": "../media/assets/clip04.mp4", + "kind": "video", + "duration_s": 80.63, + "source_url": "https://images-assets.nasa.gov/video/Artemis%20Origins/Artemis%20Origins~small.mp4", + "source_page": "https://images.nasa.gov/details/Artemis%20Origins", + "license": "NASA informational media permission", + "license_url": "https://www.nasa.gov/nasa-brand-center/images-and-media/", + "author": "NASA", + "changes": "None; downloaded original file.", + "observation": "Artemis Origins montage: Apollo astronauts, lunar surface/rover, a speech and future-space imagery. Speech intervals independently labelled from NASA publisher SRT.", + "independent_inspection": "Real frames viewed before inference; NASA speech labels use publisher SRT, never model outputs.", + "bytes": 16710310, + "sha256": "5fa4af30f984f4d7d53b50990d1e21e41df50b62a8b3ec3a2a1d44f30431f0f1" + }, + { + "id": "music01", + "path": "../media/assets/music01.ogg", + "kind": "music", + "source_url": "https://upload.wikimedia.org/wikipedia/commons/4/49/Kimiko_Ishizaka_-_14_-_Variatio_13_a_2_Clav.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:Kimiko_Ishizaka_-_14_-_Variatio_13_a_2_Clav.ogg", + "license": "CC0-1.0", + "license_url": "https://creativecommons.org/publicdomain/zero/1.0/", + "author": "Kimiko Ishizaka", + "changes": "None; original waveform and embedded metadata retained.", + "observation": "Soft solo piano performance of Bach Goldberg Variation13, original Vorbis artist/title tags retained.", + "duration_s": 254.453333, + "tags": { + "encoder": "Lavc58.54.100 libvorbis", + "title": "Variatio 13 a 2 Clav.", + "artist": "Kimiko Ishizaka", + "track": "14", + "album": "The Open Goldberg Variations", + "disc": "1", + "date": "2012-05-29T17:49:26", + "genre": "Classical", + "copyright": "CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/", + "tdat": "2012-05-29 17:49:26", + "comment": "URL: http://freemusicarchive.org/music/Kimiko_Ishizaka/The_Open_Goldberg_Variations/KIMIKO_ISHIZAKA_-_Goldberg_Variations_BWV_988_-_14_-_Variatio_13_a_2_Clav__44k-24b\r\nComments: http://freemusicarchive.org/\r\nCurator: ccCommunity\r\nCopyright: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/\r\nComposer: J.S. Bach", + "tlen": "254453", + "tsrc": "QMNYZ1200014", + "composer": "J.S. Bach", + "album_artist": "Kimiko Ishizaka" + }, + "independent_inspection": "Publisher descriptions and original FFprobe tags checked before inference. Audio perception not claimed as human listening.", + "bytes": 4697594, + "sha256": "29fcd20414ee19c2a5aa4339eaf6375079dc87959051ab0aea32c76df52f34dc" + }, + { + "id": "music02", + "path": "../media/assets/music02.ogg", + "kind": "music", + "source_url": "https://upload.wikimedia.org/wikipedia/commons/3/37/00_Jazz_Violin_Solo.ogg", + "source_page": "https://commons.wikimedia.org/wiki/File:00_Jazz_Violin_Solo.ogg", + "license": "PD-self", + "license_url": "https://commons.wikimedia.org/wiki/File:00_Jazz_Violin_Solo.ogg", + "author": "VWD", + "changes": "None; original waveform and embedded metadata retained.", + "observation": "Jazz violin solo, original Vorbis tags identify artist Violinist WTD and title Jazz Violin Solo.", + "duration_s": 30.0, + "tags": { + "album": "2007 Spring Concert Series", + "artist": "Violinist WTD", + "composer": "Violinist WTD/LPC Jazz Band", + "date": "2007", + "encoder": "Max 0.7.1", + "encoding": "Vorbis settings: VBR(q=3.000000)", + "genre": "Jazz", + "title": "Jazz Violin Solo" + }, + "independent_inspection": "Publisher descriptions and original FFprobe tags checked before inference. Audio perception not claimed as human listening.", + "bytes": 198070, + "sha256": "9ce9c5d07f941c07cab575e58016d6ad1b09219a36477652118d8e85c3401dd8" + }, + { + "id": "sound01", + "path": "../media/assets/sound01.wav", + "kind": "sound", + "source_url": "https://opengameart.org/sites/default/files/dog_barking_mono.wav", + "source_page": "https://opengameart.org/content/dog-barking-mono", + "license": "CC0-1.0", + "license_url": "https://creativecommons.org/publicdomain/zero/1.0/", + "author": "Brandon Morris (uploaded by HaelDB)", + "changes": "None; original waveform and embedded metadata retained.", + "observation": "Dog bark recording from independently described source; real waveform retained.", + "duration_s": 2.038594, + "tags": {}, + "independent_inspection": "Publisher descriptions and original FFprobe tags checked before inference. Audio perception not claimed as human listening.", + "bytes": 179848, + "sha256": "bbd0f908b3514dd3bd7d2bc04dcf64f8d360a161e7f43cac5d6761e7add79451" + }, + { + "id": "sound02", + "path": "../media/assets/sound02.mp3", + "kind": "sound", + "source_url": "https://interactive-examples.mdn.mozilla.net/media/cc0-audio/t-rex-roar.mp3", + "source_page": "https://github.com/mdn/interactive-examples", + "license": "CC0-1.0", + "license_url": "https://creativecommons.org/publicdomain/zero/1.0/", + "author": "MDN contributors", + "changes": "None; original waveform and embedded metadata retained.", + "observation": "Roaring creature sound effect from independently described CC0 source; real waveform retained.", + "duration_s": 2.074218, + "tags": { + "encoder": "LAME3.98r" + }, + "independent_inspection": "Publisher descriptions and original FFprobe tags checked before inference. Audio perception not claimed as human listening.", + "bytes": 39868, + "sha256": "41191d0727073bf848bcc8f0bd851d71a0b0058e901abb1c1b236ad327bda52e" + }, + { + "id": "llama-unicode", + "path": "sources/llama__unicode.cpp", + "kind": "code", + "source_path": "src/unicode.cpp", + "source_repository": "https://github.com/ggml-org/llama.cpp", + "source_commit": "4d756bc72bf00a4aacf410ae15a2d315f3db400d", + "sha256": "2818ca5657cff7b2b4f50bc976bb9bdc4e75e92fd595a5a75555b22946e149da", + "bytes": 61285, + "license": "MIT; fixtures/mixed/LLAMA-LICENSE.txt", + "changes": "Exact committed upstream bytes retrieved with git show; local runtime patches excluded" + }, + { + "id": "llama-allocator", + "path": "sources/llama__ggml-alloc.c", + "kind": "code", + "source_path": "ggml/src/ggml-alloc.c", + "source_repository": "https://github.com/ggml-org/llama.cpp", + "source_commit": "4d756bc72bf00a4aacf410ae15a2d315f3db400d", + "sha256": "9664bda3e9c4223f5c94ebfeb3c0381f0065a5091ddae707c09a800f7e419a39", + "bytes": 45068, + "license": "MIT; fixtures/mixed/LLAMA-LICENSE.txt", + "changes": "Exact committed upstream bytes retrieved with git show; local runtime patches excluded" + } + ], + "queries": [ + { + "id": "code-budget-en", + "text": "share a CPU thread budget between foreground search and background indexing", + "language": "en", + "category": "code", + "expected": [ + "source01" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "code-budget-de", + "text": "CPU-Threads zwischen Suche und Hintergrundindexierung aufteilen", + "language": "de", + "category": "code", + "expected": [ + "source01" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "code-audio-en", + "text": "code that chooses audio search when a query mentions speech", + "language": "en", + "category": "code", + "expected": [ + "source02" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "code-audio-de", + "text": "Code der Audiosuche auswählt wenn die Anfrage Sprache erwähnt", + "language": "de", + "category": "code", + "expected": [ + "source02" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "code-extract-en", + "text": "read a document once and hash the exact bytes consumed by the parser", + "language": "en", + "category": "code", + "expected": [ + "source03" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "code-extract-de", + "text": "Dokument einmal lesen und die vom Parser verwendeten Bytes hashen", + "language": "de", + "category": "code", + "expected": [ + "source03" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "code-selection-en", + "text": "restore a selected result after it temporarily disappears from search results", + "language": "en", + "category": "code", + "expected": [ + "source04" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "code-selection-de", + "text": "Ausgewähltes Ergebnis wiederherstellen nachdem es kurz aus Suchergebnissen verschwindet", + "language": "de", + "category": "code", + "expected": [ + "source04" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "code-timing-en", + "text": "measure queue delay separately from native search and rendering time", + "language": "en", + "category": "code", + "expected": [ + "source05" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "code-timing-de", + "text": "Wartezeit getrennt von nativer Suche und Darstellung messen", + "language": "de", + "category": "code", + "expected": [ + "source05" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "doc-timing-en", + "text": "documentation explaining why nested trace durations must not be added together", + "language": "en", + "category": "document", + "expected": [ + "source07" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "doc-timing-de", + "text": "Dokumentation warum verschachtelte Trace-Zeiten nicht addiert werden dürfen", + "language": "de", + "category": "document", + "expected": [ + "source07" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "doc-incremental-en", + "text": "how completed embedding work is reused after an interrupted index build", + "language": "en", + "category": "document", + "expected": [ + "source08" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "doc-incremental-de", + "text": "Wie fertige Embeddings nach unterbrochener Indexierung wiederverwendet werden", + "language": "de", + "category": "document", + "expected": [ + "source08" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "doc-watch-en", + "text": "limitations of detecting media changes with unchanged file size and modification time", + "language": "en", + "category": "document", + "expected": [ + "source09" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "doc-watch-de", + "text": "Grenzen der Erkennung von Medienänderungen bei gleicher Größe und Änderungszeit", + "language": "de", + "category": "document", + "expected": [ + "source09" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "doc-code-en", + "text": "how to search source code using the code retrieval task prefix", + "language": "en", + "category": "document", + "expected": [ + "source10" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "doc-code-de", + "text": "Wie Quellcodesuche das Code-Retrieval-Präfix verwendet", + "language": "de", + "category": "document", + "expected": [ + "source10" + ], + "split": "frozen-diagnostic", + "hard_negative_note": "Corresponding implementation and explanatory docs coexist; requested artifact type is intentional." + }, + { + "id": "media-q001", + "text": "the fluffy striped cat resting with its chin on the floor", + "language": "en", + "expected": [ + "image01" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q002", + "text": "die flauschige gestreifte Katze liegt mit dem Kinn auf dem Boden", + "language": "de", + "expected": [ + "image01" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q003", + "text": "the overhead breakfast bowl with berries and diced mango", + "language": "en", + "expected": [ + "image02" + ], + "category": "image-memory", + "hard_negatives": [ + "image09" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q004", + "text": "die Frühstücksschüssel mit Beeren und gewürfelter Mango von oben", + "language": "de", + "expected": [ + "image02" + ], + "category": "image-memory", + "hard_negatives": [ + "image09" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q005", + "text": "the daytime lake reflecting jagged mountains and pine trees", + "language": "en", + "expected": [ + "image03" + ], + "category": "image-memory", + "hard_negatives": [ + "image05", + "image06", + "image04" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q006", + "text": "der See bei Tageslicht spiegelt schroffe Berge und Tannen", + "language": "de", + "expected": [ + "image03" + ], + "category": "image-memory", + "hard_negatives": [ + "image05", + "image06", + "image04" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q007", + "text": "the Milky Way over a dark mountain valley", + "language": "en", + "expected": [ + "image04" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q008", + "text": "die Milchstraße über einem dunklen Bergtal", + "language": "de", + "expected": [ + "image04" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q009", + "text": "the wooden jetty pointing into a lake under moonlight", + "language": "en", + "expected": [ + "image05" + ], + "category": "image-memory", + "hard_negatives": [ + "image03", + "image04" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q010", + "text": "der Holzsteg führt bei Mondlicht in einen See", + "language": "de", + "expected": [ + "image05" + ], + "category": "image-memory", + "hard_negatives": [ + "image03", + "image04" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q011", + "text": "the pink layered mountain ridges beneath a golden sunset", + "language": "en", + "expected": [ + "image06" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q012", + "text": "die rosa Bergketten unter einem goldenen Sonnenuntergang", + "language": "de", + "expected": [ + "image06" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q013", + "text": "the herd of cows on green hills", + "language": "en", + "expected": [ + "image07" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q014", + "text": "die Kuhherde auf grünen Hügeln", + "language": "de", + "expected": [ + "image07" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q015", + "text": "the red sports car driving through woods", + "language": "en", + "expected": [ + "image08" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q016", + "text": "das rote Sportauto fährt durch den Wald", + "language": "de", + "expected": [ + "image08" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q017", + "text": "the vegetable and grain bowl beside carrot slices", + "language": "en", + "expected": [ + "image09" + ], + "category": "image-memory", + "hard_negatives": [ + "image02" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q018", + "text": "die Gemüse-Getreide-Schüssel neben Karottenscheiben", + "language": "de", + "expected": [ + "image09" + ], + "category": "image-memory", + "hard_negatives": [ + "image02" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q019", + "text": "milk being poured into coffee to make a pattern", + "language": "en", + "expected": [ + "image10" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q020", + "text": "Milch wird für ein Muster in den Kaffee gegossen", + "language": "de", + "expected": [ + "image10" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q021", + "text": "the empty tropical beach with turquoise water", + "language": "en", + "expected": [ + "image11" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q022", + "text": "der leere tropische Strand mit türkisfarbenem Wasser", + "language": "de", + "expected": [ + "image11" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q023", + "text": "the black and white photo with a hand reaching towards us", + "language": "en", + "expected": [ + "image12" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q024", + "text": "das Schwarzweißfoto mit einer Hand, die nach uns greift", + "language": "de", + "expected": [ + "image12" + ], + "category": "image-memory", + "hard_negatives": [], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q025", + "text": "the cat jumping up beside a door handle", + "language": "en", + "expected": [ + "clip01" + ], + "category": "video-action", + "intervals": { + "clip01": [ + [ + 2, + 4.9 + ], + [ + 15.5, + 17.5 + ] + ] + }, + "hard_negatives": [ + "image01" + ], + "negative_intervals": { + "clip01": [ + [ + 7, + 14 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q026", + "text": "die Katze springt neben dem Türgriff hoch", + "language": "de", + "expected": [ + "clip01" + ], + "category": "video-action", + "intervals": { + "clip01": [ + [ + 2, + 4.9 + ], + [ + 15.5, + 17.5 + ] + ] + }, + "hard_negatives": [ + "image01" + ], + "negative_intervals": { + "clip01": [ + [ + 7, + 14 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q027", + "text": "the cat sitting low by the door after trying to open it", + "language": "en", + "expected": [ + "clip01" + ], + "category": "video-action", + "intervals": { + "clip01": [ + [ + 7, + 14 + ] + ] + }, + "negative_intervals": { + "clip01": [ + [ + 2, + 4.9 + ], + [ + 15.5, + 17.5 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q028", + "text": "die Katze sitzt nach dem Öffnungsversuch unten an der Tür", + "language": "de", + "expected": [ + "clip01" + ], + "category": "video-action", + "intervals": { + "clip01": [ + [ + 7, + 14 + ] + ] + }, + "negative_intervals": { + "clip01": [ + [ + 2, + 4.9 + ], + [ + 15.5, + 17.5 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q029", + "text": "the red flower bud before its petals open", + "language": "en", + "expected": [ + "clip03" + ], + "category": "video-temporal", + "intervals": { + "clip03": [ + [ + 0, + 1.0 + ] + ] + }, + "negative_intervals": { + "clip03": [ + [ + 3.5, + 5.05 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q030", + "text": "die rote Blütenknospe bevor sich die Blätter öffnen", + "language": "de", + "expected": [ + "clip03" + ], + "category": "video-temporal", + "intervals": { + "clip03": [ + [ + 0, + 1.0 + ] + ] + }, + "negative_intervals": { + "clip03": [ + [ + 3.5, + 5.05 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q031", + "text": "the red flower with its petals open wide", + "language": "en", + "expected": [ + "clip03" + ], + "category": "video-temporal", + "intervals": { + "clip03": [ + [ + 3.5, + 5.05 + ] + ] + }, + "negative_intervals": { + "clip03": [ + [ + 0, + 1.0 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q032", + "text": "die rote Blume mit weit geöffneten Blütenblättern", + "language": "de", + "expected": [ + "clip03" + ], + "category": "video-temporal", + "intervals": { + "clip03": [ + [ + 3.5, + 5.05 + ] + ] + }, + "negative_intervals": { + "clip03": [ + [ + 0, + 1.0 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q033", + "text": "the tall waterfall on a foggy rocky mountain", + "language": "en", + "expected": [ + "clip02" + ], + "category": "video-scene", + "intervals": { + "clip02": [ + [ + 0, + 13.81 + ] + ] + }, + "hard_negatives": [ + "image03", + "image05", + "image11" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q034", + "text": "der hohe Wasserfall an einem nebligen felsigen Berg", + "language": "de", + "expected": [ + "clip02" + ], + "category": "video-scene", + "intervals": { + "clip02": [ + [ + 0, + 13.81 + ] + ] + }, + "hard_negatives": [ + "image03", + "image05", + "image11" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q035", + "text": "the astronaut standing on the Moon next to landing equipment", + "language": "en", + "expected": [ + "clip04" + ], + "category": "video-scene", + "intervals": { + "clip04": [ + [ + 10, + 20 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q036", + "text": "der Astronaut auf dem Mond neben der Landeausrüstung", + "language": "de", + "expected": [ + "clip04" + ], + "category": "video-scene", + "intervals": { + "clip04": [ + [ + 10, + 20 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q037", + "text": "the lunar rover driving over grey Moon terrain", + "language": "en", + "expected": [ + "clip04" + ], + "category": "video-scene", + "intervals": { + "clip04": [ + [ + 24, + 33 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q038", + "text": "der Mondrover fährt über graues Mondgelände", + "language": "de", + "expected": [ + "clip04" + ], + "category": "video-scene", + "intervals": { + "clip04": [ + [ + 24, + 33 + ] + ] + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q039", + "text": "someone saying that we choose to go to the Moon", + "language": "en", + "expected": [ + "clip04" + ], + "category": "video-speech", + "intervals": { + "clip04": [ + [ + 1.551, + 3.603 + ] + ] + }, + "required_modality": "audio", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q040", + "text": "jemand sagt, dass wir uns entscheiden, zum Mond zu fliegen", + "language": "de", + "expected": [ + "clip04" + ], + "category": "video-speech", + "intervals": { + "clip04": [ + [ + 1.551, + 3.603 + ] + ] + }, + "required_modality": "audio", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q041", + "text": "someone finding orange soil on the Moon", + "language": "en", + "expected": [ + "clip04" + ], + "category": "video-speech", + "intervals": { + "clip04": [ + [ + 13.129, + 16.316 + ] + ] + }, + "required_modality": "audio", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q042", + "text": "jemand findet orangefarbenen Boden auf dem Mond", + "language": "de", + "expected": [ + "clip04" + ], + "category": "video-speech", + "intervals": { + "clip04": [ + [ + 13.129, + 16.316 + ] + ] + }, + "required_modality": "audio", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q043", + "text": "a request to fund the Artemis program for the first woman on the Moon", + "language": "en", + "expected": [ + "clip04" + ], + "category": "video-speech", + "intervals": { + "clip04": [ + [ + 43.777, + 52.419 + ] + ] + }, + "required_modality": "audio", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q044", + "text": "eine Bitte um Finanzierung von Artemis für die erste Frau auf dem Mond", + "language": "de", + "expected": [ + "clip04" + ], + "category": "video-speech", + "intervals": { + "clip04": [ + [ + 43.777, + 52.419 + ] + ] + }, + "required_modality": "audio", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q045", + "text": "someone talking about planting a flag on Mars", + "language": "en", + "expected": [ + "clip04" + ], + "category": "video-speech", + "intervals": { + "clip04": [ + [ + 56.539, + 63.496 + ] + ] + }, + "required_modality": "audio", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q046", + "text": "jemand spricht davon, eine Flagge auf dem Mars aufzustellen", + "language": "de", + "expected": [ + "clip04" + ], + "category": "video-speech", + "intervals": { + "clip04": [ + [ + 56.539, + 63.496 + ] + ] + }, + "required_modality": "audio", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q047", + "text": "gentle solo piano music without anyone singing", + "language": "en", + "expected": [ + "music01" + ], + "category": "music-memory", + "intervals": { + "music01": [ + [ + 0, + 254.45 + ] + ] + }, + "required_modality": "audio", + "hard_negatives": [ + "music02" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q048", + "text": "sanfte Klaviermusik ohne Gesang", + "language": "de", + "expected": [ + "music01" + ], + "category": "music-memory", + "intervals": { + "music01": [ + [ + 0, + 254.45 + ] + ] + }, + "required_modality": "audio", + "hard_negatives": [ + "music02" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q049", + "text": "a jazzy violin melody", + "language": "en", + "expected": [ + "music02" + ], + "category": "music-memory", + "intervals": { + "music02": [ + [ + 0, + 30 + ] + ] + }, + "required_modality": "audio", + "hard_negatives": [ + "music01" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q050", + "text": "eine jazzige Geigenmelodie", + "language": "de", + "expected": [ + "music02" + ], + "category": "music-memory", + "intervals": { + "music02": [ + [ + 0, + 30 + ] + ] + }, + "required_modality": "audio", + "hard_negatives": [ + "music01" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q051", + "text": "a dog barking nearby", + "language": "en", + "expected": [ + "sound01" + ], + "category": "sound-memory", + "intervals": { + "sound01": [ + [ + 0, + 2.039 + ] + ] + }, + "required_modality": "audio", + "hard_negatives": [ + "sound02" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q052", + "text": "ein Hund bellt in der Nähe", + "language": "de", + "expected": [ + "sound01" + ], + "category": "sound-memory", + "intervals": { + "sound01": [ + [ + 0, + 2.039 + ] + ] + }, + "required_modality": "audio", + "hard_negatives": [ + "sound02" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q053", + "text": "a growling roaring creature sound effect", + "language": "en", + "expected": [ + "sound02" + ], + "category": "sound-memory", + "intervals": { + "sound02": [ + [ + 0, + 2.074 + ] + ] + }, + "required_modality": "audio", + "hard_negatives": [ + "sound01" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q054", + "text": "ein knurrender brüllender Kreatur-Soundeffekt", + "language": "de", + "expected": [ + "sound02" + ], + "category": "sound-memory", + "intervals": { + "sound02": [ + [ + 0, + 2.074 + ] + ] + }, + "required_modality": "audio", + "hard_negatives": [ + "sound01" + ], + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q055", + "text": "Kimiko Ishizaka", + "language": "en", + "expected": [ + "music01" + ], + "category": "exact-metadata", + "metadata": { + "artist": "Kimiko Ishizaka" + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q056", + "text": "Jazz Violin Solo", + "language": "en", + "expected": [ + "music02" + ], + "category": "exact-metadata", + "metadata": { + "title": "Jazz Violin Solo" + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q057", + "text": "Violinist WTD", + "language": "en", + "expected": [ + "music02" + ], + "category": "exact-metadata", + "metadata": { + "artist": "Violinist WTD" + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q058", + "text": "wooden pier under the moon", + "language": "en", + "expected": [ + "image05" + ], + "category": "exact-filter", + "filters": { + "extension": "jpg" + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q059", + "text": "the cat jumping beside the handle", + "language": "en", + "expected": [], + "category": "temporal-negative", + "filters": { + "asset": "clip01", + "time_range_s": [ + 7, + 14 + ] + }, + "required_modality": "image", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q060", + "text": "someone mentioning Mars", + "language": "en", + "expected": [], + "category": "filter-no-answer", + "filters": { + "extension": "ogg" + }, + "required_modality": "audio", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q061", + "text": "a purple elephant playing a violin on the beach", + "language": "en", + "expected": [], + "category": "content-no-answer", + "split": "held-out-no-answer", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q062", + "text": "ein violetter Elefant spielt Geige am Strand", + "language": "de", + "expected": [], + "category": "content-no-answer", + "split": "held-out-no-answer", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q063", + "text": "someone explains quantum entanglement", + "language": "en", + "expected": [], + "category": "speech-no-answer", + "required_modality": "audio", + "split": "held-out-no-answer", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q064", + "text": "jemand erklärt Quantenverschränkung", + "language": "de", + "expected": [], + "category": "speech-no-answer", + "required_modality": "audio", + "split": "held-out-no-answer", + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "media-q065", + "text": "Adele", + "language": "en", + "expected": [], + "category": "metadata-no-answer", + "metadata": { + "artist": "Adele" + }, + "label_source": "fixtures/media/fixture.json" + }, + { + "id": "absent-83", + "text": "implementation of a chess engine using alpha beta pruning", + "language": "en", + "category": "mixed-no-answer", + "expected": [], + "split": "frozen-diagnostic" + }, + { + "id": "absent-84", + "text": "Implementierung einer Schachengine mit Alpha-Beta-Suche", + "language": "de", + "category": "mixed-no-answer", + "expected": [], + "split": "frozen-diagnostic" + }, + { + "id": "absent-85", + "text": "signed employment agreement with annual salary and holiday allowance", + "language": "en", + "category": "mixed-no-answer", + "expected": [], + "split": "frozen-diagnostic" + }, + { + "id": "absent-86", + "text": "Unterschriebener Arbeitsvertrag mit Jahresgehalt und Urlaubsanspruch", + "language": "de", + "category": "mixed-no-answer", + "expected": [], + "split": "frozen-diagnostic" + }, + { + "id": "upstream-unicode-en", + "text": "decode a UTF-8 sequence into a Unicode code point and advance the byte offset", + "language": "en", + "category": "code", + "expected": [ + "llama-unicode" + ], + "split": "frozen-diagnostic" + }, + { + "id": "upstream-unicode-de", + "text": "UTF-8-Sequenz in einen Unicode-Codepoint dekodieren und Byteposition erhöhen", + "language": "de", + "category": "code", + "expected": [ + "llama-unicode" + ], + "split": "frozen-diagnostic" + }, + { + "id": "upstream-allocator-en", + "text": "reuse memory buffers for tensors in a computational graph", + "language": "en", + "category": "code", + "expected": [ + "llama-allocator" + ], + "split": "frozen-diagnostic" + }, + { + "id": "upstream-allocator-de", + "text": "Speicherpuffer für Tensoren in einem Rechengraphen wiederverwenden", + "language": "de", + "category": "code", + "expected": [ + "llama-allocator" + ], + "split": "frozen-diagnostic" + } + ] +} diff --git a/fixtures/mixed/labels.lock.json b/fixtures/mixed/labels.lock.json new file mode 100644 index 0000000..e1680c3 --- /dev/null +++ b/fixtures/mixed/labels.lock.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "fixture_sha256": "f83d27227582f614159ff153654d5bb5a47bf74998141e389557f3950ea0b924", + "model_runs": 0 +} diff --git a/fixtures/mixed/sources/apps__desktop__src__search-performance.ts b/fixtures/mixed/sources/apps__desktop__src__search-performance.ts new file mode 100644 index 0000000..7fc3219 --- /dev/null +++ b/fixtures/mixed/sources/apps__desktop__src__search-performance.ts @@ -0,0 +1,230 @@ +/** UI times use performance.now(); native spans use their own command clock. */ +export interface NativeSpan { + name: string; + start_ms: number; + duration_ms: number; + parent: number | null; + counters: Record; +} +export interface NativeMetrics { + total_ms: number; + result_count: number; + trace: NativeSpan[] | null; + dropped_spans: number; +} +export interface MeasuredResponse { + results: T[]; + metrics: NativeMetrics; +} +export type SearchLane = "lexical" | "hybrid"; +export interface AttemptStart { + lane: SearchLane; + startedAt: number; + submittedAt: number; + submission: number; +} +interface Attempt { + lane: SearchLane; + submission: number; + attempt: number; + queued_ms: number; + retry_wait_ms: number; + invoke_ms: number; + native: NativeMetrics | null; + transport_estimate_ms: number | null; + failed: boolean; + ended_at: number; +} +interface InputTimingSummary { + debounce_ms: number; + first_results_ms: number | null; + latest_displayed_ms: number | null; + hybrid_final_results_ms: number | null; +} + +export class SearchMeasurement { + private originalInput: Readonly | null = null; + debug: boolean; + inputAt: number; + scheduledAt: number; + context: Record; + submissions = new Map(); + latestSubmission = new Map(); + submissionSequence = new Map(); + laneAttempts = new Map(); + attempts: Attempt[] = []; + droppedAttempts = 0; + droppedDisplays = 0; + firstDisplayMs: number | null = null; + displays: { + lane: SearchLane; + elapsed_ms: number; + dom_update_ms: number; + count: number; + }[] = []; + + constructor( + inputAt: number, + scheduledAt: number, + debug: boolean, + context: Record = {}, + ) { + this.inputAt = inputAt; + this.scheduledAt = scheduledAt; + this.debug = debug; + this.context = { ...context }; + } + /** Filesystem updates are independent requests, not elapsed user idle time. */ + freshForRefresh(now: number): SearchMeasurement { + const refreshed = new SearchMeasurement(now, now, this.debug, this.context); + // Preserve one fixed-size numeric snapshot, never an ancestor trace/history. + // Pending responses on the old request cannot rewrite this snapshot. + refreshed.originalInput = Object.freeze({ + ...(this.originalInput ?? { + debounce_ms: Math.max(0, this.scheduledAt - this.inputAt), + first_results_ms: this.firstDisplayMs, + latest_displayed_ms: this.displays.at(-1)?.elapsed_ms ?? null, + hybrid_final_results_ms: + [...this.displays] + .reverse() + .find((display) => display.lane === "hybrid")?.elapsed_ms ?? null, + }), + }); + return refreshed; + } + submitted(lane: SearchLane, at: number) { + if (!this.submissions.has(lane)) this.submissions.set(lane, at); + this.latestSubmission.set(lane, at); + this.submissionSequence.set( + lane, + (this.submissionSequence.get(lane) ?? 0) + 1, + ); + } + begin(lane: SearchLane, startedAt: number): AttemptStart { + return { + lane, + startedAt, + submittedAt: this.latestSubmission.get(lane) ?? startedAt, + submission: this.submissionSequence.get(lane) ?? 0, + }; + } + completed( + token: AttemptStart, + end: number, + native: NativeMetrics | null, + failed: boolean, + ) { + const { lane, startedAt: start, submittedAt, submission } = token; + const prior = this.attempts.filter((attempt) => attempt.lane === lane); + const previous = prior.at(-1); + const retry = previous?.failed && previous.submission === submission; + const attemptNumber = (this.laneAttempts.get(lane) ?? 0) + 1; + this.laneAttempts.set(lane, attemptNumber); + const elapsed = Math.max(0, end - start); + this.attempts.push({ + lane, + submission, + attempt: attemptNumber, + queued_ms: retry ? 0 : Math.max(0, start - submittedAt), + retry_wait_ms: retry ? Math.max(0, start - previous.ended_at) : 0, + invoke_ms: elapsed, + native, + transport_estimate_ms: native + ? Math.max(0, elapsed - native.total_ms) + : null, + failed, + ended_at: end, + }); + if (this.attempts.length > (this.debug ? 32 : 4)) { + this.attempts.shift(); + this.droppedAttempts++; + } + } + displayed(lane: SearchLane, start: number, end: number, count: number) { + if (this.firstDisplayMs === null) + this.firstDisplayMs = Math.max(0, end - this.inputAt); + this.displays.push({ + lane, + elapsed_ms: Math.max(0, end - this.inputAt), + dom_update_ms: Math.max(0, end - start), + count, + }); + if (this.displays.length > (this.debug ? 16 : 2)) { + this.displays.shift(); + this.droppedDisplays++; + } + } + latestNative(lane: SearchLane) { + return ( + this.attempts + .filter((attempt) => attempt.lane === lane && !attempt.failed) + .at(-1)?.native ?? null + ); + } + /** Deliberately exclude query text, filenames, paths, snippets and errors. */ + export() { + return { + version: 1, + trigger: this.originalInput ? "filesystem_refresh" : "user_search", + original_input: this.originalInput ? { ...this.originalInput } : null, + context: this.context, + clock_note: + "UI monotonic elapsed times; native command-relative spans. Nested spans and parallel lanes must not be summed.", + debounce_ms: Math.max(0, this.scheduledAt - this.inputAt), + lane_schedule_ms: Object.fromEntries( + [...this.submissions].map(([lane, at]) => [ + lane, + Math.max(0, at - this.scheduledAt), + ]), + ), + attempts: this.attempts.map(({ ended_at: _end, ...attempt }) => attempt), + displays: this.displays, + first_results_ms: this.firstDisplayMs, + latest_displayed_ms: this.displays.at(-1)?.elapsed_ms ?? null, + hybrid_final_results_ms: + [...this.displays] + .reverse() + .find((display) => display.lane === "hybrid")?.elapsed_ms ?? null, + dropped_attempts: this.droppedAttempts, + dropped_displays: this.droppedDisplays, + }; + } +} + +export function measuredFailure( + error: unknown, +): { message: string; metrics: NativeMetrics } | null { + if (!error || typeof error !== "object") return null; + const value = error as { message?: unknown; metrics?: NativeMetrics }; + return typeof value.message === "string" && + value.metrics && + Number.isFinite(value.metrics.total_ms) + ? { message: value.message, metrics: value.metrics } + : null; +} + +/** Malformed/cyclic parent chains cannot trap the debug renderer. */ +export function spanDepth(spans: NativeSpan[], index: number) { + let depth = 0; + const visited = new Set([index]); + let parent = spans[index]?.parent; + while ( + parent != null && + spans[parent] && + !visited.has(parent) && + depth < 16 + ) { + visited.add(parent); + depth++; + parent = spans[parent].parent; + } + return depth; +} + +export function briefMs(value: number) { + return value < 1 + ? "<1" + : value < 10 + ? value.toFixed(1) + : String(Math.round(value)); +} diff --git a/fixtures/mixed/sources/apps__desktop__src__selection-state.ts b/fixtures/mixed/sources/apps__desktop__src__selection-state.ts new file mode 100644 index 0000000..d79d403 --- /dev/null +++ b/fixtures/mixed/sources/apps__desktop__src__selection-state.ts @@ -0,0 +1,24 @@ +/** Preserve the user's result choice while progressive indexing hides it briefly. */ +export class ResultSelection { + private scope: string | null = null; + private intendedPath: string | null = null; + + begin(scope: string) { + if (scope !== this.scope) { + this.scope = scope; + this.intendedPath = null; + } + } + + choose(path: string) { + this.intendedPath = path; + } + + resolve(paths: readonly string[]): number { + if (!paths.length) return -1; + if (this.intendedPath === null) this.intendedPath = paths[0]; + const intended = paths.indexOf(this.intendedPath); + // A temporary fallback is presentation, not a new user choice. + return intended >= 0 ? intended : 0; + } +} diff --git a/fixtures/mixed/sources/crates__searchi-core__src__extraction.rs b/fixtures/mixed/sources/crates__searchi-core__src__extraction.rs new file mode 100644 index 0000000..6000014 --- /dev/null +++ b/fixtures/mixed/sources/crates__searchi-core__src__extraction.rs @@ -0,0 +1,320 @@ +//! Bounded inputs and outputs; parser allocations and CPU are not process isolated. +use crate::SourceLocation; +use anyhow::{bail, Context, Result}; +use quick_xml::{events::Event, name::ResolveResult, NsReader, XmlVersion}; +use sha2::{Digest, Sha256}; +use std::{ + fs::File, + io::{Cursor, Read}, + path::Path, +}; + +const MAX_TEXT: usize = 2 * 1024 * 1024; +const MAX_DOCUMENT: usize = 32 * 1024 * 1024; +const MAX_XML: usize = 8 * 1024 * 1024; +const MAX_PDF_STREAM: usize = 4 * 1024 * 1024; +const MAX_PASSAGES: usize = 10_000; + +pub(crate) struct Passage { + pub text: String, + pub source: SourceLocation, +} + +pub(crate) struct SourceInput { + bytes: Vec, + pub hash: String, +} + +/// Read once: the fingerprint and parser consume precisely the same bytes. +pub(crate) fn read_source(path: &Path) -> Result> { + let extension = path + .extension() + .and_then(|value| value.to_str()) + .unwrap_or("") + .to_ascii_lowercase(); + let document = matches!(extension.as_str(), "pdf" | "docx"); + if !document + && !crate::is_code_path(path) + && !matches!( + extension.as_str(), + "txt" + | "md" + | "json" + | "toml" + | "yaml" + | "yml" + | "csv" + | "html" + | "css" + | "xml" + | "log" + | "sh" + ) + { + return Ok(None); + } + if !document && File::open(path)?.metadata()?.len() > MAX_TEXT as u64 { + return Ok(None); + } + let bytes = read_bounded(path, if document { MAX_DOCUMENT } else { MAX_TEXT })?; + let hash = format!("{:x}", Sha256::digest(&bytes)); + Ok(Some(SourceInput { bytes, hash })) +} + +pub(crate) fn extract(path: &Path, input: SourceInput) -> Result>> { + let extension = path + .extension() + .and_then(|value| value.to_str()) + .unwrap_or("") + .to_ascii_lowercase(); + if extension == "pdf" { + return pdf(&input.bytes).map(Some); + } + if extension == "docx" { + return docx(&input.bytes).map(Some); + } + let code = crate::is_code_path(path); + if input.bytes.contains(&0) { + return Ok(None); + } + let Ok(text) = String::from_utf8(input.bytes) else { + return Ok(None); + }; + let lines = text.split_inclusive('\n').collect::>(); + if code { + let mut passages = Vec::new(); + for start in (0..lines.len()).step_by(96) { + if passages.len() >= MAX_PASSAGES { + return Ok(None); + } + let end = (start + 128).min(lines.len()); + passages.push(Passage { + text: lines[start..end].concat(), + source: SourceLocation { + kind: "line".into(), + number: (start + 1) as u32, + }, + }); + if end == lines.len() { + break; + } + } + return Ok(Some(passages)); + } + Ok(Some( + lines + .chunks(64) + .enumerate() + .map(|(index, lines)| Passage { + text: lines.concat(), + source: SourceLocation { + kind: "line".into(), + number: (index * 64 + 1) as u32, + }, + }) + .collect(), + )) +} + +fn read_bounded(path: &Path, maximum: usize) -> Result> { + let file = File::open(path)?; + if file.metadata()?.len() > maximum as u64 { + bail!("document input exceeds {maximum} bytes"); + } + let mut bytes = Vec::new(); + file.take(maximum as u64 + 1).read_to_end(&mut bytes)?; + if bytes.len() > maximum { + bail!("document input exceeds {maximum} bytes"); + } + Ok(bytes) +} + +fn pdf(bytes: &[u8]) -> Result> { + let document = lopdf::Document::load_mem_with_options( + bytes, + lopdf::LoadOptions { + strict: true, + max_decompressed_size: Some(MAX_PDF_STREAM), + ..Default::default() + }, + ) + .context("invalid, encrypted, or over-limit PDF")?; + if document.was_encrypted() || document.is_encrypted() { + bail!("encrypted PDF is unsupported"); + } + let pages = document.get_pages(); + if pages.is_empty() { + bail!("PDF has no pages"); + } + if pages.len() > 1000 { + bail!("PDF exceeds 1000 pages"); + } + let mut passages = Vec::new(); + let mut total = 0; + for number in pages.keys() { + let text = document + .extract_text_with_limit(&[*number], MAX_PDF_STREAM) + .with_context(|| format!("PDF text extraction failed on page {number}"))?; + total += text.len(); + if total > MAX_TEXT { + bail!("PDF text exceeds {MAX_TEXT} bytes"); + } + if !text.trim().is_empty() { + passages.push(Passage { + text, + source: SourceLocation { + kind: "page".into(), + number: *number, + }, + }); + } + } + Ok(passages) +} + +fn docx(bytes: &[u8]) -> Result> { + let mut archive = + zip::ZipArchive::new(Cursor::new(bytes)).context("invalid DOCX ZIP container")?; + if archive.len() > 10_000 { + bail!("DOCX exceeds 10000 archive entries"); + } + let mut xml = String::new(); + let part = archive + .by_name("word/document.xml") + .context("DOCX lacks word/document.xml")?; + if part.size() > MAX_XML as u64 { + bail!("DOCX document XML exceeds {MAX_XML} bytes"); + } + part.take(MAX_XML as u64 + 1) + .read_to_string(&mut xml) + .context("DOCX XML must be valid UTF-8")?; + if xml.len() > MAX_XML { + bail!("DOCX document XML exceeds {MAX_XML} bytes"); + } + let mut reader = NsReader::from_str(&xml); + let mut passages = Vec::new(); + let mut paragraph = String::new(); + let mut number = 0; + let mut in_paragraph = false; + let mut in_text = false; + let mut total = 0; + let mut depth = 0usize; + let mut saw_document = false; + loop { + let (namespace, event) = reader.read_resolved_event().context("invalid DOCX XML")?; + let word_namespace = matches!(namespace, ResolveResult::Bound(value) if matches!(value.as_ref(), + "http://schemas.openxmlformats.org/wordprocessingml/2006/main" | "http://purl.oclc.org/ooxml/wordprocessingml/main")); + match &event { + Event::Start(start) => { + if depth == 0 { + if saw_document || !word_namespace || start.local_name().as_ref() != "document" + { + bail!("invalid DOCX document root"); + } + saw_document = true; + } + depth += 1; + if depth > 128 { + bail!("DOCX XML exceeds 128 nesting levels"); + } + } + Event::End(_) => depth = depth.checked_sub(1).context("invalid DOCX XML depth")?, + Event::Empty(_) if depth == 0 => bail!("invalid DOCX document root"), + _ => {} + } + match event { + Event::Start(event) if word_namespace => match event.local_name().as_ref() { + "p" => { + if in_paragraph { + bail!("nested DOCX paragraphs are unsupported"); + } + number += 1; + if number > MAX_PASSAGES { + bail!("DOCX exceeds 10000 paragraphs"); + } + in_paragraph = true; + paragraph.clear(); + } + "t" => in_text = in_paragraph, + _ => {} + }, + Event::End(event) if word_namespace => match event.local_name().as_ref() { + "t" => in_text = false, + "p" => { + in_paragraph = false; + in_text = false; + total += paragraph.len(); + if total > MAX_TEXT { + bail!("DOCX text exceeds {MAX_TEXT} bytes"); + } + if !paragraph.trim().is_empty() { + passages.push(Passage { + text: std::mem::take(&mut paragraph), + source: SourceLocation { + kind: "paragraph".into(), + number: number as u32, + }, + }); + } + } + _ => {} + }, + Event::Empty(event) if word_namespace => match event.local_name().as_ref() { + "tab" if in_paragraph => paragraph.push('\t'), + "br" | "cr" if in_paragraph => paragraph.push('\n'), + "p" => { + number += 1; + if number > MAX_PASSAGES { + bail!("DOCX exceeds 10000 paragraphs"); + } + } + _ => {} + }, + Event::Text(event) if in_text => { + paragraph.push_str(&event.xml_content(XmlVersion::Implicit1_0)) + } + Event::CData(event) if in_text => { + paragraph.push_str(&event.xml_content(XmlVersion::Implicit1_0)) + } + Event::GeneralRef(event) if in_text => { + let reference = format!("&{};", event.as_ref()); + paragraph.push_str( + &quick_xml::escape::unescape(&reference).context("unsupported DOCX entity")?, + ); + } + Event::DocType(_) => bail!("DOCX XML document types are unsupported"), + Event::Eof => { + if in_paragraph || depth != 0 || !saw_document { + bail!("incomplete DOCX document"); + } + break; + } + _ => {} + } + if total + paragraph.len() > MAX_TEXT { + bail!("DOCX text exceeds {MAX_TEXT} bytes"); + } + } + Ok(passages) +} + +#[cfg(test)] +mod provenance_tests { + use super::*; + + #[test] + fn parsing_and_fingerprint_share_one_snapshot_even_when_path_changes() -> Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("notes.txt"); + std::fs::write(&path, "original parser input")?; + let input = read_source(&path)?.unwrap(); + assert_eq!( + input.hash, + format!("{:x}", Sha256::digest(b"original parser input")) + ); + std::fs::write(&path, "replacement on disk")?; + let passages = extract(&path, input)?.unwrap(); + assert_eq!(passages[0].text, "original parser input"); + Ok(()) + } +} diff --git a/fixtures/mixed/sources/crates__searchi-service__src__coordination.rs b/fixtures/mixed/sources/crates__searchi-service__src__coordination.rs new file mode 100644 index 0000000..44b4ffd --- /dev/null +++ b/fixtures/mixed/sources/crates__searchi-service__src__coordination.rs @@ -0,0 +1,417 @@ +//! Shared admission for persistent inference sessions. This bounds admitted +//! native worker threads, not all extraction/ANN/OS work. Already executing +//! inference cannot be preempted; long audio windows remain a latency limit. +use crate::EmbeddingProvider; +use anyhow::{bail, Result}; +use searchi_core::metrics::{SpanToken, TraceCollector}; +use searchi_semantic::TextWindow; +use std::{ + collections::BTreeMap, + path::Path, + sync::{Arc, Condvar, Mutex}, + time::{Duration, Instant}, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InferenceRole { + Foreground, + Background, +} +#[derive(Default)] +struct State { + limit: usize, + used: usize, + foreground: usize, +} +struct Shared { + state: Mutex, + changed: Condvar, +} +/// Clone this handle across query, setup and indexing sessions. +#[derive(Clone)] +pub struct InferenceBudget(Arc); +impl InferenceBudget { + pub fn new(requested: usize) -> Self { + Self(Arc::new(Shared { + state: Mutex::new(State { + limit: clamp(requested), + ..State::default() + }), + changed: Condvar::new(), + })) + } + pub fn threads(&self) -> usize { + self.0.state.lock().unwrap_or_else(|e| e.into_inner()).limit + } + /// Leave one admitted worker thread for background indexing. On a + /// single-thread budget, already executing inference still cannot yield. + pub fn worker_threads(&self, role: InferenceRole) -> usize { + match role { + InferenceRole::Foreground => self.threads().saturating_sub(1).max(1), + InferenceRole::Background => 1, + } + } + /// Update admission immediately; already admitted work is not cancelled. + /// Recreate sessions whose worker thread count exceeds a reduced limit. + pub fn configure(&self, requested: usize) -> usize { + let limit = clamp(requested); + self.0.state.lock().unwrap_or_else(|e| e.into_inner()).limit = limit; + self.0.changed.notify_all(); + limit + } + /// Register before waiting for a service mutex. This reserves priority, not + /// threads, so the foreground does not deadlock with its indexing session. + pub fn pending_foreground(&self) -> PendingForeground { + self.0 + .state + .lock() + .unwrap_or_else(|e| e.into_inner()) + .foreground += 1; + self.0.changed.notify_all(); + PendingForeground(self.clone()) + } + pub fn acquire( + &self, + role: InferenceRole, + actual_worker_threads: usize, + ) -> Result { + let weight = actual_worker_threads.max(1); + let _pending = (role == InferenceRole::Foreground).then(|| self.pending_foreground()); + let deadline = Instant::now() + Duration::from_secs(30); + let mut state = self.0.state.lock().unwrap_or_else(|e| e.into_inner()); + loop { + if weight > state.limit { + bail!("Inference session uses {weight} threads but shared budget is {}; recreate the session after changing its thread count", state.limit); + } + if state.used <= state.limit - weight + && (role == InferenceRole::Foreground || state.foreground == 0) + { + state.used += weight; + return Ok(InferencePermit { + budget: self.clone(), + weight, + }); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + bail!("Timed out waiting for shared inference budget"); + } + state = self + .0 + .changed + .wait_timeout(state, remaining) + .unwrap_or_else(|e| e.into_inner()) + .0; + } + } +} +fn clamp(requested: usize) -> usize { + requested + .clamp(1, 8) + .min(std::thread::available_parallelism().map_or(1, usize::from)) +} +pub struct PendingForeground(InferenceBudget); +impl Drop for PendingForeground { + fn drop(&mut self) { + let mut s = self.0 .0.state.lock().unwrap_or_else(|e| e.into_inner()); + s.foreground -= 1; + self.0 .0.changed.notify_all(); + } +} +pub struct InferencePermit { + budget: InferenceBudget, + weight: usize, +} +impl Drop for InferencePermit { + fn drop(&mut self) { + let mut s = self + .budget + .0 + .state + .lock() + .unwrap_or_else(|e| e.into_inner()); + s.used -= self.weight; + self.budget.0.changed.notify_all(); + } +} + +/// Delegates genuine inference unchanged. Background documents yield between +/// individual encodes even when the caller supplies a larger batch. +pub struct BudgetedProvider { + inner: Box, + budget: InferenceBudget, + role: InferenceRole, + threads: usize, + query_started: bool, +} +impl BudgetedProvider { + pub fn new( + inner: Box, + budget: &InferenceBudget, + role: InferenceRole, + actual_worker_threads: usize, + ) -> Self { + Self { + inner, + budget: budget.clone(), + role, + threads: actual_worker_threads.max(1), + query_started: false, + } + } + fn permit(&self) -> Result { + self.budget.acquire(self.role, self.threads) + } +} +impl EmbeddingProvider for BudgetedProvider { + fn document_batch_limit(&self) -> usize { + if self.role == InferenceRole::Background { + 1 + } else { + self.inner.document_batch_limit() + } + } + fn fingerprint(&self) -> &str { + self.inner.fingerprint() + } + fn dimensions(&self) -> usize { + self.inner.dimensions() + } + fn split_windows(&mut self, text: &str, title: &str) -> Result> { + let _p = self.permit()?; + self.inner.split_windows(text, title) + } + fn embed_documents(&mut self, texts: &[&str], title: &str) -> Result>> { + if self.role == InferenceRole::Background { + let mut output = Vec::with_capacity(texts.len()); + for text in texts { + let _p = self.permit()?; + output.extend(self.inner.embed_documents(&[*text], title)?); + } + Ok(output) + } else { + let _p = self.permit()?; + self.inner.embed_documents(texts, title) + } + } + fn embed_query(&mut self, query: &str) -> Result> { + self.query_started = false; + let _p = self.permit()?; + self.query_started = true; + self.inner.embed_query(query) + } + fn embed_query_debug(&mut self, query: &str) -> Result> { + self.query_started = false; + let _p = self.permit()?; + self.query_started = true; + self.inner.embed_query_debug(query) + } + fn embed_code_query(&mut self, query: &str) -> Result> { + self.query_started = false; + let _p = self.permit()?; + self.query_started = true; + self.inner.embed_code_query(query) + } + fn embed_code_query_debug(&mut self, query: &str) -> Result> { + self.query_started = false; + let _p = self.permit()?; + self.query_started = true; + self.inner.embed_code_query_debug(query) + } + fn query_diagnostics(&self) -> BTreeMap { + if self.query_started { + self.inner.query_diagnostics() + } else { + Default::default() + } + } + fn embed_query_traced( + &mut self, + query: &str, + debug: bool, + trace: &mut TraceCollector, + parent: Option, + ) -> Result> { + self.query_started = false; + let span = trace.begin("semantic.inference.admission_wait", parent); + let permit = self.permit(); + trace.end( + span, + if trace.is_debug() { + BTreeMap::from([ + ("worker_threads".into(), self.threads as u64), + ("failed".into(), u64::from(permit.is_err())), + ]) + } else { + Default::default() + }, + ); + let _permit = permit?; + self.query_started = true; + self.inner.embed_query_traced(query, debug, trace, parent) + } + fn embed_code_query_traced( + &mut self, + query: &str, + debug: bool, + trace: &mut TraceCollector, + parent: Option, + ) -> Result> { + self.query_started = false; + let span = trace.begin("semantic.inference.admission_wait", parent); + let permit = self.permit(); + trace.end( + span, + if trace.is_debug() { + BTreeMap::from([ + ("worker_threads".into(), self.threads as u64), + ("failed".into(), u64::from(permit.is_err())), + ]) + } else { + Default::default() + }, + ); + let _permit = permit?; + self.query_started = true; + self.inner + .embed_code_query_traced(query, debug, trace, parent) + } + fn embed_images(&mut self, paths: &[&Path], budget: u32) -> Result>> { + if self.role == InferenceRole::Background { + let mut output = Vec::with_capacity(paths.len()); + for path in paths { + let _p = self.permit()?; + output.extend(self.inner.embed_images(&[*path], budget)?); + } + return Ok(output); + } + let _p = self.permit()?; + self.inner.embed_images(paths, budget) + } + fn embed_audio(&mut self, pcm: &[f32]) -> Result> { + let _p = self.permit()?; + self.inner.embed_audio(pcm) + } + fn worker_info(&self) -> Option { + self.inner.worker_info() + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn failed_admission_clears_prior_worker_diagnostics_without_calling_worker() { + use searchi_core::metrics::MetricsMode; + use std::sync::atomic::{AtomicUsize, Ordering}; + struct ObservedInvocation(Arc); + impl EmbeddingProvider for ObservedInvocation { + fn fingerprint(&self) -> &str { + "scheduler-test-only" + } + fn dimensions(&self) -> usize { + 0 + } + fn split_windows(&mut self, _: &str, _: &str) -> Result> { + unreachable!() + } + fn embed_documents(&mut self, _: &[&str], _: &str) -> Result>> { + unreachable!() + } + fn embed_query(&mut self, _: &str) -> Result> { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(Vec::new()) + } + fn embed_code_query(&mut self, query: &str) -> Result> { + self.embed_query(query) + } + fn query_diagnostics(&self) -> BTreeMap { + BTreeMap::from([("prior_worker_call".into(), 1)]) + } + } + let budget = InferenceBudget::new(1); + // Test worker weight two without requiring particular test host hardware. + budget.0.state.lock().unwrap().limit = 2; + let calls = Arc::new(AtomicUsize::new(0)); + let mut provider = BudgetedProvider::new( + Box::new(ObservedInvocation(calls.clone())), + &budget, + InferenceRole::Foreground, + 2, + ); + provider + .embed_query_debug("first admitted invocation") + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(provider.query_diagnostics()["prior_worker_call"], 1); + assert_eq!(budget.configure(1), 1); + let mut trace = TraceCollector::new(MetricsMode::Debug); + assert!(provider + .embed_query_traced("rejected invocation", true, &mut trace, None) + .is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!(provider.query_diagnostics().is_empty()); + let metrics = trace.finish(0); + let stages = metrics.trace.unwrap(); + assert_eq!(stages.len(), 1); + assert_eq!(stages[0].name, "semantic.inference.admission_wait"); + assert_eq!(stages[0].counters["failed"], 1); + assert_eq!(stages[0].counters["worker_threads"], 2); + assert!(stages[0].duration_ms >= 0.0); + assert!(provider.embed_code_query_debug("also rejected").is_err()); + assert!(provider.query_diagnostics().is_empty()); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(budget.0.state.lock().unwrap().foreground, 0); + assert_eq!(budget.0.state.lock().unwrap().used, 0); + } + #[test] + fn reserved_query_capacity_admits_foreground_during_background() { + let budget = InferenceBudget::new(1); + // Deliberately exercise a two-thread scheduler independent of CI hardware. + budget.0.state.lock().unwrap().limit = 2; + assert_eq!(budget.worker_threads(InferenceRole::Foreground), 1); + assert_eq!(budget.worker_threads(InferenceRole::Background), 1); + let background = budget.acquire(InferenceRole::Background, 1).unwrap(); + let foreground = budget.acquire(InferenceRole::Foreground, 1).unwrap(); + assert_eq!(budget.0.state.lock().unwrap().used, 2); + drop(foreground); + drop(background); + } + #[test] + fn pending_query_blocks_background_until_released() { + let budget = InferenceBudget::new(1); + let pending = budget.pending_foreground(); + let other = budget.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let _permit = other.acquire(InferenceRole::Background, 1).unwrap(); + tx.send(()).unwrap(); + }); + assert!(rx.recv_timeout(Duration::from_millis(50)).is_err()); + let foreground = budget.acquire(InferenceRole::Foreground, 1).unwrap(); + drop(pending); + assert!(rx.recv_timeout(Duration::from_millis(50)).is_err()); + drop(foreground); + rx.recv_timeout(Duration::from_secs(2)).unwrap(); + worker.join().unwrap(); + } + #[test] + fn panic_releases_threads_and_priority() { + let budget = InferenceBudget::new(1); + let shared = budget.clone(); + assert!(std::panic::catch_unwind(move || { + let _pending = shared.pending_foreground(); + let _permit = shared.acquire(InferenceRole::Foreground, 1).unwrap(); + panic!("owned inference failure"); + }) + .is_err()); + let _permit = budget.acquire(InferenceRole::Background, 1).unwrap(); + } + #[test] + fn invalid_session_weight_fails_instead_of_oversubscribing() { + let budget = InferenceBudget::new(1); + assert!(budget.acquire(InferenceRole::Foreground, 2).is_err()); + assert_eq!(budget.configure(0), 1); + assert!(InferenceBudget::new(usize::MAX).threads() <= 8); + } +} diff --git a/fixtures/mixed/sources/crates__searchi-service__src__routing.rs b/fixtures/mixed/sources/crates__searchi-service__src__routing.rs new file mode 100644 index 0000000..2173b5c --- /dev/null +++ b/fixtures/mixed/sources/crates__searchi-service__src__routing.rs @@ -0,0 +1,218 @@ +use super::*; + +/// Explicit semantic source channel, independent of filename/type filters. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SemanticOrigin { + All, + Text, + Image, + Video, + Audio, +} +impl SemanticOrigin { + fn source_type(self) -> Option<&'static str> { + match self { + Self::All => None, + Self::Text => Some("text"), + Self::Image => Some("image"), + Self::Video => Some("video"), + Self::Audio => Some("audio"), + } + } +} + +/// A conservative intent hint, not a language understanding model. Explicit +/// document/visual cues veto it; ordinary topic and instrument queries do not +/// silently restrict modality. Word boundaries avoid filename substrings. +fn preferred_origin(query: &str) -> Option { + if query.split_whitespace().count() == 1 && Path::new(query).extension().is_some() { + return None; + } + let lower = query.to_lowercase(); + let words: Vec<_> = lower + .split(|c: char| !c.is_alphanumeric()) + .filter(|s| !s.is_empty()) + .collect(); + let has = |values: &[&str]| words.iter().any(|word| values.contains(word)); + if has(&[ + "image", + "photo", + "picture", + "screenshot", + "diagram", + "document", + "pdf", + "text", + "paper", + "bild", + "foto", + "screenshot", + "dokument", + "textdatei", + ]) { + return None; + } + if has(&[ + "audio", + "spoken", + "speech", + "recording", + "conversation", + "interview", + "podcast", + "sound", + "music", + "gesprochen", + "gespräch", + "gespraech", + "geräusch", + "geraeusch", + "musik", + "hörbar", + "hörbarer", + "hörbare", + "hoerbar", + "hoerbarer", + "hoerbare", + "tonaufnahme", + ]) { + return Some(SemanticOrigin::Audio); + } + if (has(&["someone", "speaker"]) + && has(&[ + "mentions", + "mentioned", + "says", + "said", + "speaks", + "speaking", + ])) + || (has(&["jemand", "sprecher", "sprecherin"]) + && has(&["erwähnt", "erwaehnt", "sagt", "gesagt", "spricht"])) + { + return Some(SemanticOrigin::Audio); + } + None +} + +impl HybridSearch { + pub(super) fn routed_semantic( + &mut self, + query: &str, + vector: &[f32], + limit: usize, + options: &SearchOptions, + ) -> Result> { + let mut trace = TraceCollector::new(MetricsMode::Basic); + self.routed_semantic_traced(query, vector, limit, options, &mut trace, None) + } + pub(super) fn routed_semantic_traced( + &mut self, + query: &str, + vector: &[f32], + limit: usize, + options: &SearchOptions, + trace: &mut TraceCollector, + parent: Option, + ) -> Result> { + trace.timed("semantic.routing", parent, |trace, parent| { + if let Some(origin) = if options.code_only { + None + } else { + preferred_origin(query) + } { + trace.counter(parent, "audio_preference", 1); + let preferred = self.store.search_origin_traced( + vector, + &self.generation_fingerprint, + limit, + options, + origin.source_type(), + trace, + parent, + )?; + if !preferred.is_empty() { + return Ok(preferred); + } + } + trace.counter(parent, "general_lane", 1); + self.store.search_origin_traced( + vector, + &self.generation_fingerprint, + limit, + options, + None, + trace, + parent, + ) + }) + } + + /// Query one actual source channel, or All to bypass automatic spoken/audio + /// preference. This never embeds files again or changes an active policy. + pub fn search_semantic_with_origin( + &mut self, + query: &str, + limit: usize, + options: &SearchOptions, + origin: SemanticOrigin, + ) -> Result> { + let _ = self.lexical.search_with_options(query, 0, options)?; + if query.trim().is_empty() || limit == 0 { + return Ok(Vec::new()); + } + let mut trace = TraceCollector::new(MetricsMode::Basic); + let vector = self.query_vector(query, options.code_only, &mut trace, None)?; + self.store.search_origin( + &vector, + &self.generation_fingerprint, + limit.min(1000), + options, + origin.source_type(), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn persisted_custom_limits_require_identical_model_and_preprocessing() { + let expected = "pinned-model:pinned-media:preprocessing-v1"; + assert!(reusable_policy( + expected, + &format!("{expected}:custom-240-2000-120-10000-2000-1800-14400000-17179869184") + )); + for active in ["other-model:pinned-media:preprocessing-v1:custom-240-2000-120-10000-2000-1800-14400000-17179869184","pinned-model:pinned-media:preprocessing-v2:custom-240-2000-120-10000-2000-1800-14400000-17179869184","pinned-model:pinned-media:preprocessing-v1:custom-unknown"] {assert!(!reusable_policy(expected,active));} + } + #[test] + fn explicit_spoken_intent_is_conservative_and_bilingual() { + for query in [ + "where someone mentions a contract", + "a speaker says the date", + "spoken discussion of ecology", + "audio of a waterfall", + "jemand erwähnt das Budget", + "Wo spricht jemand über Kündigung?", + "Gespräch über die neue Wohnung", + "hörbarer Regen", + ] { + assert_eq!( + preferred_origin(query), + Some(SemanticOrigin::Audio), + "{query}" + ); + } + for query in [ + "contract deadline", + "violin solo", + "photo of someone speaking", + "Bild von einem Sprecher", + "a document that someone mentioned", + "speech.png", + "a screenshot of an audio waveform", + ] { + assert_eq!(preferred_origin(query), None, "{query}"); + } + } +} diff --git a/fixtures/mixed/sources/crates__searchi-service__src__store__incremental.rs b/fixtures/mixed/sources/crates__searchi-service__src__store__incremental.rs new file mode 100644 index 0000000..9f4df1b --- /dev/null +++ b/fixtures/mixed/sources/crates__searchi-service__src__store__incremental.rs @@ -0,0 +1,967 @@ +//! Completed-source manifests and resumable staging. Legacy generations stay +//! searchable but cannot supply vectors without explicit completion evidence. +use super::*; + +#[derive(Debug, Clone)] +pub(crate) struct SourceTask { + pub key: String, + pub path: String, + /// `text` for one extracted passage, `media` for the entire media file. + pub source_type: String, + pub source_hash: String, + pub title_hash: String, + pub policy: String, + pub passage_id: i64, +} + +pub(super) fn migrate(connection: &Connection) -> Result<()> { + for (table, name, declaration) in [ + ( + "semantic_generations", + "incremental", + "INTEGER NOT NULL DEFAULT 0", + ), + ("semantic_chunks", "task_key", "TEXT"), + ( + "semantic_chunks", + "file_change_revision", + "INTEGER NOT NULL DEFAULT 0", + ), + ] { + let exists: bool = connection.query_row( + "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name=?2)", + params![table, name], + |r| r.get(0), + )?; + if !exists { + connection.execute_batch(&format!( + "ALTER TABLE {table} ADD COLUMN {name} {declaration}" + ))?; + } + } + connection.execute_batch("CREATE TABLE IF NOT EXISTS semantic_tasks( + generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE, + task_key TEXT NOT NULL,path TEXT NOT NULL,source_type TEXT NOT NULL, + source_hash TEXT NOT NULL,title_hash TEXT NOT NULL,policy TEXT NOT NULL, + passage_id INTEGER NOT NULL,state TEXT NOT NULL CHECK(state IN ('pending','running','done','failed')), + attempts INTEGER NOT NULL DEFAULT 0,expected_chunks INTEGER,message TEXT, + PRIMARY KEY(generation,task_key)); + CREATE INDEX IF NOT EXISTS semantic_task_reuse ON semantic_tasks(source_hash,title_hash,policy,source_type,state); + CREATE INDEX IF NOT EXISTS semantic_chunk_task ON semantic_chunks(generation,task_key); + UPDATE semantic_meta SET schema_version=8 WHERE singleton=1;")?; + Ok(()) +} + +pub(super) fn validate_completion(connection: &Connection, generation: i64) -> Result<()> { + let incomplete: bool = connection.query_row("SELECT EXISTS(SELECT 1 FROM semantic_tasks t WHERE generation=?1 AND + (state NOT IN ('done','failed') OR (state='done' AND (expected_chunks IS NULL OR expected_chunks!=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key))) OR + (state='failed' AND EXISTS(SELECT 1 FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key))))",[generation], |r|r.get(0))?; + ensure!( + !incomplete, + "Semantic generation contains incomplete source tasks" + ); + let unowned: bool = connection.query_row("SELECT EXISTS(SELECT 1 FROM semantic_chunks c LEFT JOIN semantic_tasks t ON t.generation=c.generation AND t.task_key=c.task_key WHERE c.generation=?1 AND (t.state IS NULL OR t.state!='done'))",[generation],|r|r.get(0))?; + ensure!( + !unowned, + "Semantic generation contains chunks without completed sources" + ); + Ok(()) +} + +impl Store { + pub(crate) fn revision(&self) -> Result { + Ok(self.connection.query_row( + "SELECT corpus_revision FROM semantic_meta WHERE singleton=1", + [], + |r| r.get(0), + )?) + } + + /// Caller must retain the OS lock through reconciliation and publication. + pub(crate) fn start_or_resume( + &mut self, + fingerprint: &str, + lock: &BuildLock, + ) -> Result<(i64, bool)> { + ensure!( + lock.1 == self.lock_path, + "Build lock belongs to a different index" + ); + ensure!( + !fingerprint.is_empty() && fingerprint.len() <= 4096, + "Invalid generation fingerprint" + ); + let tx = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let generation: Option=tx.query_row("SELECT id FROM semantic_generations WHERE id!=COALESCE((SELECT active_generation FROM semantic_meta WHERE singleton=1),-1) AND fingerprint=?1 AND dimensions=?2 AND incremental=1 ORDER BY id DESC LIMIT 1",params![fingerprint,self.dimensions as i64],|r|r.get(0)).optional()?; + let resumed = generation.is_some(); + let id = if let Some(id) = generation { + id + } else { + tx.execute("INSERT INTO semantic_generations(fingerprint,dimensions,corpus_revision,incremental) SELECT ?1,?2,corpus_revision,1 FROM semantic_meta WHERE singleton=1",params![fingerprint,self.dimensions as i64])?; + tx.last_insert_rowid() + }; + tx.execute("DELETE FROM semantic_generations WHERE id!=?1 AND id!=COALESCE((SELECT active_generation FROM semantic_meta WHERE singleton=1),-1)",[id])?; + // Untagged chunks are writes interrupted before the source transaction. + tx.execute("UPDATE semantic_tasks SET state='pending',expected_chunks=NULL WHERE generation=?1 AND state='done' AND + (expected_chunks IS NULL OR expected_chunks!=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=semantic_tasks.generation AND c.task_key=semantic_tasks.task_key))",[id])?; + tx.execute("DELETE FROM semantic_chunks WHERE generation=?1 AND NOT EXISTS(SELECT 1 FROM semantic_tasks t WHERE t.generation=semantic_chunks.generation AND t.task_key=semantic_chunks.task_key AND t.state='done')",[id])?; + tx.execute("UPDATE semantic_tasks SET state='pending',expected_chunks=NULL,message=NULL WHERE generation=?1 AND state!='done'",[id])?; + tx.commit()?; + Ok((id, resumed)) + } + + pub(crate) fn stage_task(&mut self, generation: i64, task: &SourceTask) -> Result { + validate_task(task)?; + let tx = self.connection.transaction()?; + let same:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM semantic_tasks WHERE generation=?1 AND task_key=?2 AND source_hash=?3 AND title_hash=?4 AND policy=?5 AND source_type=?6 AND state='done')",params![generation,task.key,task.source_hash,task.title_hash,task.policy,task.source_type],|r|r.get(0))?; + if !same { + tx.execute( + "DELETE FROM semantic_chunks WHERE generation=?1 AND task_key=?2", + params![generation, task.key], + )?; + tx.execute("INSERT INTO semantic_tasks(generation,task_key,path,source_type,source_hash,title_hash,policy,passage_id,state,attempts) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,'running',1) + ON CONFLICT(generation,task_key) DO UPDATE SET path=excluded.path,source_type=excluded.source_type,source_hash=excluded.source_hash,title_hash=excluded.title_hash,policy=excluded.policy,passage_id=excluded.passage_id,state='running',attempts=semantic_tasks.attempts+1,expected_chunks=NULL,message=NULL",params![generation,task.key,task.path,task.source_type,task.source_hash,task.title_hash,task.policy,task.passage_id])?; + } + tx.commit()?; + Ok(same) + } + + pub(crate) fn complete_source( + &mut self, + generation: i64, + task: &SourceTask, + expected_chunks: usize, + ) -> Result<()> { + ensure!( + !task.source_hash.is_empty(), + "Cannot complete a source without verified identity" + ); + ensure!( + expected_chunks <= MAX_SCAN_CHUNKS, + "Source exceeds semantic chunk budget" + ); + let tx = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let count:i64=tx.query_row("SELECT count(*) FROM semantic_chunks WHERE generation=?1 AND path=?2 AND ((?3='text' AND source_type='text' AND passage_id=?4) OR (?3='media' AND source_type!='text'))",params![generation,task.path,task.source_type,task.passage_id],|r|r.get(0))?; + ensure!( + usize::try_from(count)? == expected_chunks, + "Incomplete source chunk membership" + ); + let total: usize = tx.query_row( + "SELECT count(*) FROM semantic_chunks WHERE generation=?1", + [generation], + |r| r.get(0), + )?; + ensure!( + total <= MAX_SCAN_CHUNKS, + "Generation exceeds semantic chunk budget" + ); + tx.execute("UPDATE semantic_chunks SET task_key=?5 WHERE generation=?1 AND path=?2 AND ((?3='text' AND source_type='text' AND passage_id=?4) OR (?3='media' AND source_type!='text'))",params![generation,task.path,task.source_type,task.passage_id,task.key])?; + let changed=tx.execute("UPDATE semantic_tasks SET state='done',expected_chunks=?3,path=?4,passage_id=?5,message=NULL WHERE generation=?1 AND task_key=?2 AND source_hash=?6 AND title_hash=?7 AND policy=?8 AND source_type=?9",params![generation,task.key,count,task.path,task.passage_id,task.source_hash,task.title_hash,task.policy,task.source_type])?; + ensure!(changed == 1, "Missing matching source task"); + if task.source_type == "media" { + tx.execute( + "DELETE FROM semantic_media_errors WHERE generation=?1 AND path=?2", + params![generation, task.path], + )?; + } + tx.commit()?; + Ok(()) + } + + pub(crate) fn fail_source( + &mut self, + generation: i64, + task: &SourceTask, + message: &str, + ) -> Result<()> { + let tx = self.connection.transaction()?; + tx.execute("DELETE FROM semantic_chunks WHERE generation=?1 AND (task_key=?2 OR (path=?3 AND ((?4='text' AND source_type='text' AND passage_id=?5) OR (?4='media' AND source_type!='text'))))",params![generation,task.key,task.path,task.source_type,task.passage_id])?; + tx.execute("UPDATE semantic_tasks SET state='failed',expected_chunks=NULL,message=?3 WHERE generation=?1 AND task_key=?2",params![generation,task.key,message.chars().take(2048).collect::()])?; + tx.commit()?; + Ok(()) + } + + /// Reconcile completed staging groups before any encoder call. A bounded + /// SQL snapshot preserves renamed donors (including swaps and duplicates) + /// while rebuilding only current compatible memberships. Active vectors + /// are never touched. Full range/vector validation still happens on reuse. + pub(crate) fn prune_before_inference( + &mut self, + generation: i64, + tasks: &[SourceTask], + revision: i64, + ) -> Result<()> { + ensure!( + tasks.len() <= MAX_SCAN_CHUNKS, + "Generation exceeds source task budget" + ); + for task in tasks { + validate_task(task)?; + } + // The bounded snapshot can still contain 100K x 768d vectors. Keep it + // SQLite-backed on disk rather than turning that into a Rust/heap copy. + self.connection.pragma_update(None, "temp_store", "FILE")?; + let tx = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let current: i64 = tx.query_row( + "SELECT corpus_revision FROM semantic_meta WHERE singleton=1", + [], + |r| r.get(0), + )?; + ensure!( + current == revision, + "Indexed corpus changed during semantic discovery; retry required" + ); + let active: bool = tx.query_row( + "SELECT active_generation=?1 FROM semantic_meta WHERE singleton=1", + [generation], + |r| r.get::<_, Option>(0).map(|v| v.unwrap_or(false)), + )?; + ensure!(!active, "Cannot prune the active semantic generation"); + let existing: usize = tx.query_row( + "SELECT count(*) FROM semantic_chunks WHERE generation=?1", + [generation], + |r| r.get(0), + )?; + ensure!( + existing <= MAX_SCAN_CHUNKS, + "Staging generation exceeds semantic chunk budget" + ); + tx.execute_batch("CREATE TEMP TABLE IF NOT EXISTS semantic_discovery_tasks(task_key TEXT PRIMARY KEY,path TEXT NOT NULL,source_type TEXT NOT NULL,source_hash TEXT NOT NULL,title_hash TEXT NOT NULL,policy TEXT NOT NULL,passage_id INTEGER NOT NULL); + DELETE FROM semantic_discovery_tasks; + CREATE TEMP TABLE IF NOT EXISTS semantic_prune_tasks AS SELECT * FROM semantic_tasks WHERE 0; + CREATE TEMP TABLE IF NOT EXISTS semantic_prune_chunks AS SELECT * FROM semantic_chunks WHERE 0; + DELETE FROM semantic_prune_tasks; DELETE FROM semantic_prune_chunks;")?; + for task in tasks { + tx.execute( + "INSERT INTO semantic_discovery_tasks VALUES(?1,?2,?3,?4,?5,?6,?7)", + params![ + task.key, + task.path, + task.source_type, + task.source_hash, + task.title_hash, + task.policy, + task.passage_id + ], + )?; + } + tx.execute("INSERT INTO semantic_prune_tasks SELECT * FROM semantic_tasks t WHERE generation=?1 AND state='done' AND source_hash!='' AND expected_chunks IS NOT NULL AND expected_chunks=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key)",[generation])?; + tx.execute( + "INSERT INTO semantic_prune_chunks SELECT * FROM semantic_chunks WHERE generation=?1", + [generation], + )?; + tx.execute_batch("CREATE UNIQUE INDEX IF NOT EXISTS temp.semantic_prune_task_key ON semantic_prune_tasks(task_key); + CREATE INDEX IF NOT EXISTS temp.semantic_prune_reuse ON semantic_prune_tasks(source_hash,title_hash,policy,source_type); + CREATE INDEX IF NOT EXISTS temp.semantic_prune_membership ON semantic_prune_chunks(task_key);")?; + let mut mappings = Vec::new(); + { + let mut statement = tx.prepare("SELECT d.task_key,COALESCE((SELECT old.task_key FROM semantic_prune_tasks old WHERE old.task_key=d.task_key AND old.source_hash=d.source_hash AND old.title_hash=d.title_hash AND old.policy=d.policy AND old.source_type=d.source_type),(SELECT old.task_key FROM semantic_prune_tasks old WHERE old.source_hash=d.source_hash AND old.title_hash=d.title_hash AND old.policy=d.policy AND old.source_type=d.source_type ORDER BY old.task_key LIMIT 1)) FROM semantic_discovery_tasks d")?; + let rows = statement.query_map([], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?)) + })?; + for row in rows { + let (key, donor) = row?; + if let Some(donor) = donor { + mappings.push((key, donor)); + } + } + } + let mut total = 0usize; + for (_, donor) in &mappings { + let count: usize = tx.query_row( + "SELECT expected_chunks FROM semantic_prune_tasks WHERE task_key=?1", + [donor], + |r| r.get(0), + )?; + total = total + .checked_add(count) + .context("Semantic chunk count overflow")?; + ensure!( + total <= MAX_SCAN_CHUNKS, + "Current completed sources exceed semantic chunk budget" + ); + } + tx.execute( + "DELETE FROM semantic_chunks WHERE generation=?1", + [generation], + )?; + tx.execute( + "DELETE FROM semantic_tasks WHERE generation=?1", + [generation], + )?; + for (key, donor) in mappings { + tx.execute("INSERT INTO semantic_tasks(generation,task_key,path,source_type,source_hash,title_hash,policy,passage_id,state,attempts,expected_chunks,message) + SELECT ?1,d.task_key,d.path,d.source_type,d.source_hash,d.title_hash,d.policy,d.passage_id,'done',old.attempts,old.expected_chunks,NULL FROM semantic_discovery_tasks d JOIN semantic_prune_tasks old ON old.task_key=?3 WHERE d.task_key=?2",params![generation,key,donor])?; + tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision) + SELECT ?1,d.passage_id,c.ordinal,d.path,c.kind,c.number,c.content_hash,c.start_byte,c.end_byte,c.snippet,c.vector,c.source_type,c.file_size,c.file_mtime,d.task_key,COALESCE((SELECT change_revision FROM files WHERE path=d.path),c.file_change_revision) FROM semantic_prune_chunks c JOIN semantic_discovery_tasks d ON d.task_key=?2 WHERE c.task_key=?3",params![generation,key,donor])?; + } + tx.execute( + "DELETE FROM semantic_media_errors WHERE generation=?1", + [generation], + )?; + tx.execute_batch("DELETE FROM semantic_prune_tasks; DELETE FROM semantic_prune_chunks; DELETE FROM semantic_discovery_tasks;")?; + tx.commit()?; + Ok(()) + } + + pub(crate) fn reconcile_tasks( + &mut self, + generation: i64, + keys: &[String], + revision: i64, + ) -> Result<()> { + ensure!( + keys.len() <= MAX_SCAN_CHUNKS, + "Generation exceeds source task budget" + ); + let tx = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let current: i64 = tx.query_row( + "SELECT corpus_revision FROM semantic_meta WHERE singleton=1", + [], + |r| r.get(0), + )?; + ensure!( + current == revision, + "Indexed corpus changed during semantic reconciliation; retry required" + ); + tx.execute_batch("CREATE TEMP TABLE IF NOT EXISTS semantic_current_tasks(task_key TEXT PRIMARY KEY); DELETE FROM semantic_current_tasks;")?; + for key in keys { + tx.execute( + "INSERT OR IGNORE INTO semantic_current_tasks VALUES(?1)", + [key], + )?; + } + tx.execute("DELETE FROM semantic_chunks WHERE generation=?1 AND (task_key IS NULL OR task_key NOT IN (SELECT task_key FROM semantic_current_tasks))",[generation])?; + tx.execute("DELETE FROM semantic_tasks WHERE generation=?1 AND task_key NOT IN (SELECT task_key FROM semantic_current_tasks)",[generation])?; + tx.execute("DELETE FROM semantic_media_errors WHERE generation=?1 AND path NOT IN (SELECT path FROM semantic_tasks WHERE generation=?1)",[generation])?; + tx.execute( + "UPDATE semantic_generations SET corpus_revision=?2 WHERE id=?1", + params![generation, revision], + )?; + tx.commit()?; + Ok(()) + } + + #[cfg(test)] + pub(crate) fn activate_checked(&mut self, generation: i64, revision: i64) -> Result<()> { + self.activate_checked_with_validation(generation, revision, || Ok(())) + } + + /// Validate live sources after graph preparation, outside the SQLite write + /// transaction. Rejection preserves both the active reader and staging. + pub(crate) fn activate_checked_with_validation( + &mut self, + generation: i64, + revision: i64, + validate_sources: impl FnOnce() -> Result<()>, + ) -> Result<()> { + validate_completion(&self.connection, generation)?; + self.activate_revision(generation, Some(revision), validate_sources) + } +} + +fn validate_task(task: &SourceTask) -> Result<()> { + ensure!( + !task.key.is_empty() + && task.key.len() <= 256 + && !task.path.is_empty() + && task.path.len() <= 32768, + "Invalid source task identity" + ); + ensure!( + matches!(task.source_type.as_str(), "text" | "media"), + "Invalid source task type" + ); + ensure!( + (!task.source_hash.is_empty() || task.source_type == "media") + && task.source_hash.len() <= 256 + && task.title_hash.len() <= 256 + && !task.policy.is_empty() + && task.policy.len() <= 4096, + "Invalid source completion policy" + ); + Ok(()) +} + +impl Store { + fn reusable_task( + &self, + generation: i64, + task: &SourceTask, + ) -> Result> { + Ok(self.connection.query_row("SELECT t.generation,t.task_key,t.expected_chunks FROM semantic_tasks t JOIN semantic_generations g ON g.id=t.generation JOIN semantic_generations destination ON destination.id=?1 + WHERE t.source_hash=?2 AND t.title_hash=?3 AND t.policy=?4 AND t.source_type=?5 AND t.state='done' AND t.expected_chunks IS NOT NULL + AND g.fingerprint=destination.fingerprint AND g.dimensions=destination.dimensions + AND t.expected_chunks=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key) + ORDER BY (t.generation=?1) DESC,t.generation DESC LIMIT 1",params![generation,task.source_hash,task.title_hash,task.policy,task.source_type],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?) + } + + /// No tokenizer or neural call: only a completed, policy-compatible group + /// whose persisted ranges still equal the current extracted bytes qualifies. + pub(crate) fn reuse_text_source( + &mut self, + generation: i64, + task: &SourceTask, + passage: &IndexedPassage, + ) -> Result> { + ensure!( + task.source_type == "text" + && task.passage_id == passage.id + && task.path == passage.path, + "Text task mapping differs from current source" + ); + ensure!( + task.source_hash == content_hash(&passage.text), + "Text task hash differs from source bytes" + ); + let Some((source, key, count)) = self.reusable_task(generation, task)? else { + return Ok(None); + }; + ensure!( + count <= MAX_SCAN_CHUNKS, + "Reusable source exceeds chunk budget" + ); + { + let mut stmt=self.connection.prepare("SELECT start_byte,end_byte,snippet,vector,source_type,ordinal,content_hash FROM semantic_chunks WHERE generation=?1 AND task_key=?2 ORDER BY ordinal")?; + let mut rows = stmt.query(params![source, key])?; + let mut seen = 0usize; + while let Some(row) = rows.next()? { + let start: usize = row.get(0)?; + let end: usize = row.get(1)?; + let snippet: String = row.get(2)?; + ensure!( + passage.text.get(start..end) == Some(snippet.as_str()), + "Reusable text range differs from current source" + ); + ensure!( + row.get::<_, String>(4)? == "text" + && row.get::<_, usize>(5)? == seen + && row.get::<_, String>(6)? == task.source_hash, + "Corrupt completed text source membership" + ); + validate_vector(&row.get::<_, Vec>(3)?, self.dimensions)?; + seen += 1; + } + ensure!(seen == count, "Reusable text completion count differs"); + } + self.copy_completed( + generation, + task, + source, + &key, + count, + Some(&passage.source), + None, + )?; + Ok(Some(count)) + } + + pub(crate) fn reuse_media_source( + &mut self, + generation: i64, + task: &SourceTask, + file_id: i64, + identity: &searchi_media::MediaIdentity, + ) -> Result> { + ensure!( + task.source_type == "media" + && task.passage_id == -file_id + && task.path == identity.path.to_string_lossy() + && task.source_hash == identity.sha256, + "Media task mapping differs from current source" + ); + let Some((source, key, count)) = self.reusable_task(generation, task)? else { + return Ok(None); + }; + ensure!( + count <= MAX_SCAN_CHUNKS, + "Reusable source exceeds chunk budget" + ); + { + let mut stmt=self.connection.prepare("SELECT vector,source_type,ordinal,content_hash FROM semantic_chunks WHERE generation=?1 AND task_key=?2 ORDER BY ordinal")?; + let mut rows = stmt.query(params![source, key])?; + let mut seen = 0usize; + while let Some(row) = rows.next()? { + ensure!( + matches!( + row.get::<_, String>(1)?.as_str(), + "image" | "audio" | "video" + ) && row.get::<_, usize>(2)? == seen + && row.get::<_, String>(3)? == task.source_hash, + "Corrupt completed media source membership" + ); + validate_vector(&row.get::<_, Vec>(0)?, self.dimensions)?; + seen += 1; + } + ensure!(seen == count, "Reusable media completion count differs"); + } + self.copy_completed(generation, task, source, &key, count, None, Some(identity))?; + Ok(Some(count)) + } + + #[allow(clippy::too_many_arguments)] + fn copy_completed( + &mut self, + generation: i64, + task: &SourceTask, + source: i64, + key: &str, + count: usize, + location: Option<&SourceLocation>, + identity: Option<&searchi_media::MediaIdentity>, + ) -> Result<()> { + let tx = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let total:usize=tx.query_row("SELECT count(*) FROM semantic_chunks WHERE generation=?1 AND (task_key IS NULL OR task_key!=?2)",params![generation,task.key],|r|r.get(0))?; + ensure!( + total.saturating_add(count) <= MAX_SCAN_CHUNKS, + "Generation exceeds semantic chunk budget" + ); + // A temporary SQL table allows remapping a completed staging task onto + // itself without retaining its old citations or allocating all vectors. + tx.execute_batch("CREATE TEMP TABLE IF NOT EXISTS semantic_reuse_chunks AS SELECT * FROM semantic_chunks WHERE 0; DELETE FROM semantic_reuse_chunks;")?; + tx.execute("INSERT INTO semantic_reuse_chunks SELECT * FROM semantic_chunks WHERE generation=?1 AND task_key=?2",params![source,key])?; + tx.execute( + "DELETE FROM semantic_chunks WHERE generation=?1 AND task_key=?2", + params![generation, task.key], + )?; + tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision) + SELECT ?1,?2,ordinal,?3,COALESCE(?4,kind),COALESCE(?5,number),content_hash,start_byte,end_byte,snippet,vector,source_type,COALESCE(?6,file_size),COALESCE(?7,file_mtime),?8,COALESCE((SELECT change_revision FROM files WHERE path=?3),file_change_revision) FROM semantic_reuse_chunks",params![generation,task.passage_id,task.path,location.map(|s|s.kind.as_str()),location.map(|s|s.number),identity.map(|i|i.bytes as i64),identity.map(|i|i.modified_ns),task.key])?; + let changed=tx.execute("UPDATE semantic_tasks SET state='done',expected_chunks=?3,path=?4,passage_id=?5,message=NULL WHERE generation=?1 AND task_key=?2 AND source_hash=?6 AND title_hash=?7 AND policy=?8 AND source_type=?9",params![generation,task.key,count as i64,task.path,task.passage_id,task.source_hash,task.title_hash,task.policy,task.source_type])?; + ensure!(changed == 1, "Missing source task during reuse"); + if task.source_type == "media" { + tx.execute( + "DELETE FROM semantic_media_errors WHERE generation=?1 AND path=?2", + params![generation, task.path], + )?; + } + tx.execute_batch("DELETE FROM semantic_reuse_chunks;")?; + tx.commit()?; + Ok(()) + } +} + +fn validate_vector(bytes: &[u8], dimensions: usize) -> Result<()> { + ensure!( + bytes.len() == dimensions * 4, + "Corrupt reusable vector length" + ); + let mut norm = 0.0f32; + for value in bytes.as_chunks::<4>().0 { + let value = f32::from_le_bytes(*value); + ensure!(value.is_finite(), "Corrupt reusable vector value"); + norm += value * value; + } + ensure!( + norm.is_finite() && (norm.sqrt() - 1.0).abs() < 1e-3, + "Reusable vector is not normalized" + ); + Ok(()) +} +#[cfg(test)] +mod tests { + use super::*; + use searchi_core::SearchIndex; + + // Storage-only vectors exercise durability and membership. They are not + // simulated production inference or evidence of semantic relevance. + fn vector() -> Vec { + vec![1.0, 0.0] + } + fn task(p: &IndexedPassage) -> SourceTask { + SourceTask { + key: format!("text-{}-{}", p.path, p.source.number), + path: p.path.clone(), + source_type: "text".into(), + source_hash: content_hash(&p.text), + title_hash: content_hash(&p.filename), + policy: "storage-test-window-policy".into(), + passage_id: p.id, + } + } + fn insert_complete( + store: &mut Store, + g: i64, + p: &IndexedPassage, + t: &SourceTask, + ) -> Result<()> { + store.stage_task(g, t)?; + store.insert( + g, + p, + 0, + &[TextWindow { + text: p.text.clone(), + start_byte: 0, + end_byte: p.text.len(), + }], + &[vector()], + )?; + store.complete_source(g, t, 1) + } + + #[test] + fn pre_inference_pruning_preserves_renamed_donors_at_the_chunk_ceiling() -> Result<()> { + // Actual 100K membership boundary; vectors are storage-only fixtures. + let dir = tempfile::tempdir()?; + let root = dir.path().join("docs"); + std::fs::create_dir(&root)?; + std::fs::write(root.join("notes.txt"), "complete source")?; + let db = dir.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let p = lexical.passage_batch(0, 1)?.remove(0); + let mut store = Store::open(&db, 2)?; + let lock = store.build_lock()?; + let active = store.start_or_resume("prune-policy", &lock)?.0; + insert_complete(&mut store, active, &p, &task(&p))?; + store.activate_checked(active, store.revision()?)?; + let staging = store.start_or_resume("prune-policy", &lock)?.0; + let original = task(&p); + insert_complete(&mut store, staging, &p, &original)?; + store.connection.execute("WITH RECURSIVE ord(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM ord WHERE n+1 Result<()> { + let dir = tempfile::tempdir()?; + let root = dir.path().join("docs"); + std::fs::create_dir(&root)?; + std::fs::write(root.join("first.txt"), "complete source")?; + std::fs::write(root.join("second.txt"), "interrupted source")?; + let db = dir.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let passages = lexical.passage_batch(0, 64)?; + let first = task(&passages[0]); + let second = task(&passages[1]); + let generation; + { + let mut store = Store::open(&db, 2)?; + let lock = store.build_lock()?; + generation = store.start_or_resume("storage-only-model", &lock)?.0; + insert_complete(&mut store, generation, &passages[0], &first)?; + store.stage_task(generation, &second)?; + store.insert( + generation, + &passages[1], + 0, + &[TextWindow { + text: passages[1].text.clone(), + start_byte: 0, + end_byte: passages[1].text.len(), + }], + &[vector()], + )?; + assert!(store + .activate_checked(generation, store.revision()?) + .is_err()); + } + let mut store = Store::open(&db, 2)?; + let lock = store.build_lock()?; + assert_eq!( + store.start_or_resume("storage-only-model", &lock)?, + (generation, true) + ); + assert_eq!( + store.connection.query_row( + "SELECT count(*) FROM semantic_chunks WHERE generation=?1", + [generation], + |r| r.get::<_, usize>(0) + )?, + 1 + ); + store.stage_task(generation, &first)?; + assert_eq!( + store.reuse_text_source(generation, &first, &passages[0])?, + Some(1) + ); + assert!(store + .reuse_text_source(generation, &second, &passages[1])? + .is_none()); + assert!(store + .activate_checked(generation, store.revision()?) + .is_err()); + store.reconcile_tasks(generation, &[first.key], store.revision()?)?; + store.activate_checked(generation, store.revision()?)?; + Ok(()) + } + + #[test] + fn legacy_vectors_remain_searchable_but_do_not_qualify_for_reuse() -> Result<()> { + let dir = tempfile::tempdir()?; + let root = dir.path().join("docs"); + std::fs::create_dir(&root)?; + std::fs::write(root.join("notes.txt"), "legacy searchable source")?; + let db = dir.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let p = lexical.passage_batch(0, 1)?.remove(0); + let mut store = Store::open(&db, 2)?; + let legacy = store.begin("legacy-policy")?; + store.insert( + legacy, + &p, + 0, + &[TextWindow { + text: p.text.clone(), + start_byte: 0, + end_byte: p.text.len(), + }], + &[vector()], + )?; + store.activate(legacy)?; + // Reopen a schema6 DB with legacy active chunks and no manifests. + store.connection.execute_batch( + "DROP INDEX semantic_chunk_task; ALTER TABLE semantic_chunks DROP COLUMN task_key; + DROP TABLE semantic_tasks; ALTER TABLE semantic_generations DROP COLUMN incremental; + UPDATE semantic_meta SET schema_version=6;", + )?; + drop(store); + let mut store = Store::open(&db, 2)?; + assert_eq!( + store + .search(&vector(), "legacy-policy", 1, &SearchOptions::default())? + .len(), + 1 + ); + let lock = store.build_lock()?; + let (g, _) = store.start_or_resume("legacy-policy", &lock)?; + let t = task(&p); + store.stage_task(g, &t)?; + assert_eq!(store.reuse_text_source(g, &t, &p)?, None); + assert_eq!( + store.active_fingerprint()?.as_deref(), + Some("legacy-policy") + ); + Ok(()) + } + + #[test] + fn late_source_validation_rejects_publication_without_holding_write_lock() -> Result<()> { + let dir = tempfile::tempdir()?; + let root = dir.path().join("docs"); + std::fs::create_dir(&root)?; + std::fs::write(root.join("notes.txt"), "source identity before graph build")?; + let db = dir.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let passage = lexical.passage_batch(0, 1)?.remove(0); + let source = task(&passage); + let mut store = Store::open(&db, 2)?; + let lock = store.build_lock()?; + let (active, _) = store.start_or_resume("old-policy", &lock)?; + insert_complete(&mut store, active, &passage, &source)?; + store.activate_checked(active, store.revision()?)?; + let (staging, _) = store.start_or_resume("new-policy", &lock)?; + insert_complete(&mut store, staging, &passage, &source)?; + let revision = store.revision()?; + let mut called = false; + let error = store + .activate_checked_with_validation(staging, revision, || { + called = true; + // An independent write succeeds only when validation runs outside + // the publishing connection's Immediate transaction. + let other = rusqlite::Connection::open(&db)?; + other.busy_timeout(std::time::Duration::ZERO)?; + other.execute_batch("BEGIN IMMEDIATE; COMMIT;")?; + std::fs::write(root.join("notes.txt"), "direct filesystem mutation")?; + anyhow::bail!("source changed after graph preparation") + }) + .unwrap_err(); + assert!(called); + assert!(error + .to_string() + .contains("source changed after graph preparation")); + assert_eq!(store.active_fingerprint()?.as_deref(), Some("old-policy")); + assert_eq!( + store.ann_cache.as_ref().map(|(generation, _)| *generation), + Some(active) + ); + assert_eq!(store.start_or_resume("new-policy", &lock)?, (staging, true)); + assert_eq!( + store.connection.query_row( + "SELECT count(*) FROM semantic_chunks WHERE generation=?1", + [staging], + |row| row.get::<_, usize>(0), + )?, + 1 + ); + Ok(()) + } + + #[test] + fn revision_race_cannot_publish_and_failed_activation_keeps_staging() -> Result<()> { + let dir = tempfile::tempdir()?; + let root = dir.path().join("docs"); + std::fs::create_dir(&root)?; + std::fs::write(root.join("notes.txt"), "source before change")?; + let db = dir.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let p = lexical.passage_batch(0, 1)?.remove(0); + let mut store = Store::open(&db, 2)?; + let lock = store.build_lock()?; + let (g, _) = store.start_or_resume("checked-policy", &lock)?; + let t = task(&p); + insert_complete(&mut store, g, &p, &t)?; + let old_revision = store.revision()?; + std::fs::write(root.join("notes.txt"), "source after change longer")?; + lexical.index_folder(&root)?; + assert!(store.activate_checked(g, old_revision).is_err()); + assert_eq!(store.active_fingerprint()?, None); + assert_eq!(store.start_or_resume("checked-policy", &lock)?, (g, true)); + assert_eq!( + store.connection.query_row( + "SELECT count(*) FROM semantic_chunks WHERE generation=?1", + [g], + |r| r.get::<_, usize>(0) + )?, + 1 + ); + Ok(()) + } +} + +#[cfg(test)] +mod completion_failure_tests { + use super::*; + use searchi_core::SearchIndex; + + #[test] + fn failed_completion_rolls_back_chunk_membership_and_malformed_ranges_never_reuse() -> Result<()> + { + // SQLite durability fixture, not neural inference evidence. + let dir = tempfile::tempdir()?; + let root = dir.path().join("docs"); + std::fs::create_dir(&root)?; + std::fs::write(root.join("notes.txt"), "UTF-8 source with 中文 preserved")?; + let db = dir.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let p = lexical.passage_batch(0, 1)?.remove(0); + let t = SourceTask { + key: "durability-task".into(), + path: p.path.clone(), + source_type: "text".into(), + source_hash: content_hash(&p.text), + title_hash: content_hash(&p.filename), + policy: "exact-window-test".into(), + passage_id: p.id, + }; + let mut store = Store::open(&db, 2)?; + let lock = store.build_lock()?; + let (g, _) = store.start_or_resume("storage-geometry", &lock)?; + store.stage_task(g, &t)?; + store.insert( + g, + &p, + 0, + &[TextWindow { + text: p.text.clone(), + start_byte: 0, + end_byte: p.text.len(), + }], + &[vec![1.0, 0.0]], + )?; + assert!(store.complete_source(g, &t, 2).is_err()); + let (state,owned):(String,usize)=store.connection.query_row("SELECT state,(SELECT count(*) FROM semantic_chunks WHERE generation=?1 AND task_key IS NOT NULL) FROM semantic_tasks WHERE generation=?1 AND task_key=?2",params![g,t.key],|r|Ok((r.get(0)?,r.get(1)?)))?; + assert_eq!(state, "running"); + assert_eq!(owned, 0); + assert!(store.activate_checked(g, store.revision()?).is_err()); + store.complete_source(g, &t, 1)?; + store.connection.execute("UPDATE semantic_chunks SET start_byte=1,snippet='different snippet' WHERE generation=?1",[g])?; + assert!(store.reuse_text_source(g, &t, &p).is_err()); + assert!(store.active_fingerprint()?.is_none()); + Ok(()) + } +} diff --git a/fixtures/mixed/sources/docs__AUTOMATIC-INDEXING.md b/fixtures/mixed/sources/docs__AUTOMATIC-INDEXING.md new file mode 100644 index 0000000..a8068c6 --- /dev/null +++ b/fixtures/mixed/sources/docs__AUTOMATIC-INDEXING.md @@ -0,0 +1,100 @@ +# Automatic folder updates + +The desktop now registers saved folders at startup and reconciles them in the +background. Native notifications use pinned `notify` 8.2.0. They are hints: +[notify documents](https://docs.rs/notify/8.2.0/notify/) missed events, network +filesystem limitations and platform-specific editor behavior. Periodic scans +remain necessary; a native watch failure keeps polling and reports the issue. + +## Scheduling and bounds + +- At most 64 roots and 128 retained paths per root; overflow becomes a full + reconciliation rather than an unbounded event queue. +- A 750 ms quiet period coalesces bursts, with a five-second maximum delay. +- Startup reconciles saved folders; ordinary verification runs every five minutes. +- Busy manual indexing retains queued work. Failed scans retry with backoff from + five seconds to one minute. Events arriving during a scan remain pending. +- Read/access-time events are ignored; native rescan flags still take priority. +- Missing roots keep existing records and retry attachment. Successful + reattachment after a prior watch requests verification of missed media events. +- Callback panics become a retryable failure. Shutdown signals cancellation, + wakes the owned thread and joins it without holding its state mutex. + +Scans own separate SQLite connections. They do not hold the foreground search +mutex during traversal or extraction. A changed scan prepares its reader and +briefly swaps it into the desktop. UI events refresh counts and the active query, +reject results from before the change, and request normal semantic indexing. +Refresh timing starts at the change event rather than include idle time since +the original input. A bounded numeric snapshot retains the original input +timings separately. Intended selection survives temporary result absence; +explicit navigation changes that intent, and a new query or filter clears it. + +## Targeted reconciliation + +`SearchIndex::index_paths_with_options` handles bounded absolute normalized paths. +Files update individually; directory hints reconcile only that subtree. Missing +paths remove only records within their root and path boundary. Parent/child hints +are coalesced. Outside paths, traversal and symlink ancestors are rejected. +Read/traversal failure or cancellation rolls back the complete batch. + +The cancellable folder and path variants check between entries and before commit. +An individual bounded parser input can finish before cancellation is observed. +Parser process isolation and bounded shutdown time still require further work. + +Absolute exclusions complement directory-name exclusions. The desktop excludes +its data, model and worker directories from both scans and notifications. +Decoded video frames use owned scratch directories under application data, +so temporary frames cannot feed back into the index when a home folder is watched. + +## Media freshness + +Core schema 13 adds `files.change_revision`, initially zero for existing records. +Targeted media hints increment it even when file size and timestamp are unchanged, +and remove old media tags. Full native-rescan/overflow hints take the targeted +root path; ordinary periodic scans retain the metadata fast path for media. + +Semantic schema 8 records the matching `file_change_revision` on media chunks. +An old media vector is ineligible after a hint until genuine byte-verified indexing +completes. Unchanged media can reuse completed vectors with the current revision; +changed bytes require new Google inference. Metadata probing captures the revision +before work and writes only if it still matches, preventing a stale probe from +restoring tags after a concurrent event. Generation activation remains guarded +by the original corpus revision and actual source SHA validation. + +Periodic scans hash supported bounded document inputs, but do not hash every +large media file. A same-stat media overwrite with no observed native hint remains +a verification limitation; explicit semantic rebuilds verify actual media bytes. + +## Executed validation and remaining gates + +Windows native watcher tests observed real create/update/rename/delete events. +The final Windows workspace run passed 123 tests (19 explicitly ignored), and +the desktop suite passed 28 tests. Frontend build, 39 tests, formatting and strict +workspace/desktop Clippy passed. The real CPU FFmpeg decoder regression passed +with frame paths under the configured owned scratch parent. +Coordinator tests cover overflow, rescan, busy retention, concurrent events, +periodic scheduling, panic recovery and shutdown. Core tests cover same-stat +documents/media, subtree and boundary deletion, exclusions, migrations, rollback, +concurrent readers and cancellation after a write transaction starts. + +The real Google mixed-media CPU regression passed in 83.70 seconds. It verified +that a same-stat image hint hides old vectors, changed image bytes cause one new +image encode, unchanged hinted image/audio sources reuse vectors, and current +revision tokens restore eligibility. This is correctness evidence on a tiny +fixture, not relevance quality or search/indexing throughput evidence. + +Current-source native Linux checks also passed: 124 workspace tests, 28 desktop +Release tests including actual inotify create/update/rename/delete, and the +updated real Google mixed-media regression in 67.95 seconds. Native Linux +interaction evidence is in +[LINUX-DESKTOP-VALIDATION.md](LINUX-DESKTOP-VALIDATION.md). Actual watcher-driven +UI change/create/rename/delete passed on `657de3d`; timing and selection issues +found during those interactions require separate fixes and retesting. + +macOS native watcher execution, root disappearance/reappearance on +real removable/network storage, native UI refresh interaction, representative +large-corpus contention and power-loss behavior remain open. Full reconciliation +still holds a write transaction during extraction and can contend with semantic +writes; targeted batches reduce that work but do not establish a universal +latency bound. Resource admission currently covers neural inference, not all +filesystem, decoder, parser or ANN work. diff --git a/fixtures/mixed/sources/docs__CODE-SEARCH.md b/fixtures/mixed/sources/docs__CODE-SEARCH.md new file mode 100644 index 0000000..1683bd3 --- /dev/null +++ b/fixtures/mixed/sources/docs__CODE-SEARCH.md @@ -0,0 +1,80 @@ +# Search source code + +searchi searches source files already included in your selected indexed folders. +In the desktop, open **All files** and select **Code** in the scope selector. +Search an exact symbol or describe the implementation you want. The preview +preserves whitespace and reports the passage's starting source line. +Use `--code-only` in the CLI, or `code_only: true` in MCP, to restrict every +retrieval lane to source-code files before candidate ranking. This is a file-type +filter, not an access-control boundary or an instruction to index new folders. +Other folder, extension, size and date filters still apply. + +Source support includes Rust, Python, JS/TS, C/C++, Go, Java/Kotlin, C#, Swift, +Ruby, PHP, SQL, shells, PowerShell and additional common languages. Named +`Makefile`, `GNUmakefile`, `Dockerfile`, `CMakeLists.txt` and `Justfile` are +included. Ordinary JSON/TOML/YAML configuration remains searchable in All files. +UTF-8 source passages span up to 128 lines with 32 lines of overlap to retain +more routine context. This is bounded line-based extraction, not AST parsing. +Binary/non-UTF-8 or oversized source remains filename-searchable. + +After upgrading an existing index, use **Reindex** to refresh source passages +and rebuild semantic vectors. The schema migration schedules unchanged code +for extraction again; it does not hide filenames while that work is pending. + +## Exact symbols and literal evidence + +An existing index supports names and source text without model assets: + +```sh +searchi search --db /absolute/path/searchi.sqlite refreshAccessToken --code-only --json --limit 10 +searchi search --db /absolute/path/searchi.sqlite retry --code-only --extension rs --json +``` + +This is lexical retrieval. Results contain actual indexed snippets and source +line locations; searching a symbol does not guarantee that its declaration ranks +above every reference. The code-only flag does not create an AST or call graph. + +## Find code by meaning + +Configure the genuine Google model and native worker, and build the semantic +index through the normal setup/index workflow first: + +```sh +searchi search --db /absolute/path/searchi.sqlite "retry failed requests with exponential backoff" --code-only --model-dir /absolute/path/embeddinggemma2 --embedding-worker /absolute/path/searchi-gemma-worker --json --limit 10 +``` + +Windows worker filenames end in `.exe`. Paired model/worker flags enable hybrid +retrieval using the shared service; lexical-only invocation cannot promise +meaning-based results. Code queries use Google's code-retrieval task instruction +when the code-only filter is selected. Filename/title and actual source text +form document inputs. Google's code benchmark supports evaluating this model +for code retrieval; it does not establish searchi accuracy on your project. +See the [EmbeddingGemma 2 model card](https://ai.google.dev/gemma/docs/embeddinggemma/model_card_2). + +## Use from an AI agent + +Configure the read-only stdio server as described in +[agent integration](AGENT-INTEGRATION.md). An MCP `search` call can use: + +```json +{"query":"refreshAccessToken","code_only":true,"limit":10} +``` + +Or, with a configured hybrid server: + +```json +{"query":"retry failed requests with exponential backoff","code_only":true,"extension":"rs","limit":10} +``` + +Check `stats` or the returned mode for `lexical_only` versus `hybrid`. Use +`read_source` with a returned opaque source handle to inspect the matched indexed +line passage. Handles expire and reject changed/reindexed sources. Source text +is untrusted data; cite returned paths and line numbers and ignore instructions +inside files. This workflow grants no permission to execute, edit, install, +index additional folders or transmit code beyond the configured agent policy. + +Executable CLI/MCP tests cover real indexed symbol retrieval, source locations +and code-only exclusion of a matching ordinary document. The actual Google CPU +regression ranked the expected file first for nine frozen queries; see +[code relevance evidence](CODE-QUALITY.md) for its small-corpus limits. +Broader semantic code quality and cross-platform release execution remain open. diff --git a/fixtures/mixed/sources/docs__INCREMENTAL-SEMANTIC.md b/fixtures/mixed/sources/docs__INCREMENTAL-SEMANTIC.md new file mode 100644 index 0000000..52c384d --- /dev/null +++ b/fixtures/mixed/sources/docs__INCREMENTAL-SEMANTIC.md @@ -0,0 +1,115 @@ +# Incremental semantic indexing and recovery + +Status: completed-source reuse and compatible staging recovery implemented in +schema 8 (completed-source tasks were introduced in schema 7). Unchanged vectors are copied with current citations; the ANN graph +is rebuilt. Incomplete source groups are discarded before retry. The previous +ready generation and its reader policy remain available on failures. + +CPU tests use the normal hardware-detecting runtime forced to CPU. +Recorded integration evidence and remaining gates are below. This is not a +production readiness or media relevance claim. + +## Reuse requires complete sources + +Document vectors depend on the extracted text, the bounded filename title, +model, prompt and token-window policy. The current passage content hash alone +is insufficient: a basename rename changes Google's document prompt. Passage +IDs are recreated by core reconciliation, so reuse must remap current IDs, +paths and page/line locations rather than retain old citations. + +Media reuse requires a verified hash of the actual file bytes and matching +encoder/preprocessing/sampling policy. Size and modification time alone are +insufficient. Existing media chunks are written progressively; their presence +does not prove that a file completed. A crash can leave an incomplete group. + +The versioned completion manifest records per source group: generation, task key, +source type, encoder/policy key, source hash, title hash where applicable, +expected chunk count and completion state. Associate chunks with their group. +Only completed groups can supply reusable vectors. Historical generations +without completion evidence remain searchable but require one rebuild before +their vectors qualify for reuse. + +Start with matching complete-generation policies. Component encoder keys can +later let unchanged document vectors survive an audio-sampling policy change. +Do not silently omit image, audio or video work to make a rebuild cheaper. + +## Durable work and atomic publication + +Persist bounded pending/running/done/failed tasks in SQLite. Tasks retain their +source identity, attempts and a bounded error. Obtain the existing OS build +lock before starting or resuming a compatible staging generation; only then +can interrupted running tasks be reset. Keep inference outside write +transactions. Commit source completion, verified chunk membership and task +state atomically. Incomplete groups must be discarded or resumed without +qualifying as completed reuse sources. + +Discover work in bounded keyset batches. Before publication, reconcile staging +against current indexed sources: remove deleted sources, invalidate changed +identities and enqueue additions. Verify exact document window slices and +remap citations. For media, confirm the file is unchanged after extraction and +inference. Retain the 100,000-chunk budget and explicit per-file media failures. + +Build the ANN graph only from eligible completed groups. Publish under a +transaction that checks the reconciled corpus revision; a concurrent source +change must trigger reconciliation/retry. Storage or activation failure must +preserve both the previous active generation and the same reader's policy. + +## Acceptance gates + +- [x] Schema-6 storage migration preserves active-generation search; legacy + chunks without completion evidence cannot supply reusable vectors. +- [x] Rebuild mixed text/code/image/audio/video fixtures unchanged with zero + document/media inference calls and identical vectors and source locations. +- [x] Change one document and one image; re-encode only affected groups. + Image regression preserves file size and exact mtime to exercise SHA identity. +- [x] Re-encode text after a basename rename; reuse a folder-only move with + current citations. +- [ ] Verify media bytes and current citations after a media-file rename. +- [ ] Regenerate affected media when sampling policy changes; retain eligible + text groups when component encoder-key reuse is implemented. +- [ ] Terminate after a committed batch and resume without repeating completed + inference. Never reuse or publish an unfinished source group. +- [x] Same-size/mtime text mutation during genuine inference rejects publication; + a mutation before rebuilding fails before text encoding and ordinary rescans + reextract it. Unaffected vectors survive resumed retry. +- [ ] Concurrent deletion/media mutation and comprehensive queued/running source + search freshness tests. +- [x] Inject activation failure, preserve existing results and resume completed + staging with zero document inference; cleanup-failure unit regression retains + the primary error and old reader policy. +- [x] Use genuine Google CPU inference for reuse integration checks. Storage + fixtures alone do not establish retrieval quality or reuse equivalence. +- [ ] Measure preparation, the first foreground query and warm p95 while + resumed indexing runs; record actual inference/reuse counts and resource use. + +Foreground query priority and bounded persistent workers remain part of the +first-use requirement. Faster unchanged rebuilds alone do not prove responsive +search while indexing. + +## Current limits + +Reuse requires the complete generation policy to match. Component-level reuse +across media sampling changes remains open. Durable staging resumes at completed +source boundaries; a real process-kill test during a multi-batch source remains +open. Storage tests cover discarded partial groups and corpus revision races. + +Full source hashes are checked after ANN construction and serialization, before +opening the publication write transaction. Indexed changes are also rejected by +the transaction's revision guard. This narrows the filesystem race; it does not +make file reads and SQLite publication one atomic filesystem operation. + +Core schema 12 records the SHA-256 of the exact bounded bytes consumed by each +parser. Semantic builds verify that provenance before reusing or encoding text. +Same-size/mtime changes are detected by normal folder rescanning. Legacy records +retain lexical search but need a rescan before semantic rebuilding or serving +agent excerpts. Excerpts reject missing or mismatched source provenance. + +Bounded discovery now prunes obsolete resumed groups before inference. Compatible +renamed staging groups survive with current memberships through a transactional, +disk-backed SQLite snapshot; active vectors are untouched. Genuine duplicate +current groups still count toward the 100,000-chunk ceiling. Hashing, discovery, +media metadata probes, snapshot copying and graph rebuilding have costs: zero +encoder calls does not mean zero work or prove fast first input. + +Synthetic colored-image and sine-wave fixtures prove actual vector reuse and +citation equality; they do not establish useful media relevance. diff --git a/fixtures/mixed/sources/docs__PERFORMANCE.md b/fixtures/mixed/sources/docs__PERFORMANCE.md new file mode 100644 index 0000000..44aa974 --- /dev/null +++ b/fixtures/mixed/sources/docs__PERFORMANCE.md @@ -0,0 +1,219 @@ +# Search performance diagnostics + +Search performance must distinguish total request latency from the work inside +it. The shared Rust schema exposes basic metrics for routine searches and a +bounded nested trace for deliberate debugging. It uses monotonic `Instant` +timing, not wall-clock timestamps. No query text, source passages or file paths +belong in stage names/counters. + +## Shared response + +`SearchResponse` contains `results: Vec` and `metrics: SearchMetrics`. + +| Field | Meaning | +| --- | --- | +| total_ms | Elapsed time inside the collector's measured request boundary | +| result_count | Number of final returned results | +| trace | null in basic mode; debug spans in begin order | +| dropped_spans | Spans omitted when the 512-span ceiling or invalid parent is encountered | + +A `SearchSpan` has a static `name`, relative `start_ms`, `duration_ms`, optional +parent index and integer `counters`. Parent indices reference earlier spans in +the same trace; child timings lie within their parent. Nested durations overlap, +so summing every span would double-count work. Counters are capped at 32 keys per +span, with 96-character labels. Missing spans due to the cap are visible through +`dropped_spans`, never presented as complete instrumentation. + +Basic mode retains only the request start Instant and result count; there is no +per-stage span Vec or diagnostic string allocation. Debug mode captures up to +512 spans. Scope timings preserve normal closure return values and errors. +Already-ended or foreign tokens are harmless, and unfinished scopes close at +collector finish. Closing a parent closes its open descendants for containment. + +## Instrumenting the shared engine + +```rust +use searchi_core::metrics::{MetricsMode, TraceCollector}; +let mut trace = TraceCollector::new(MetricsMode::Debug); +let results = trace.timed("search", None, |trace, parent| { + trace.timed("candidate_selection", parent, |trace, token| { + trace.counter(token, "candidate_count", candidates.len() as u64); + retrieve_candidates(candidates) + }) +}); +let metrics = trace.finish(results.len()); +``` + +This is an API illustration, not a measured benchmark. `begin(name,parent)` / +`end(token,counters)` supports explicit scopes spanning locks or separate calls; +`SpanToken` is Copy and exposes its index. `counter` records bounded integer +counts without source data. `elapsed_ms` reports elapsed request time. Capture +queue/lock boundaries in the caller, then pass that same collector into the +shared core/service so stages refer to one origin. + +Useful actual boundaries include request admission/reader lock, filter snapshot, +filename/FTS retrieval, tokenizer/query inference, active ANN load, eligibility +refresh, filtered HNSW traversal, per-file deduplication, ranking/fusion and +result assembly. Name only work that is truly measured. Do not label the whole +worker round trip as pure neural execution or fabricate stage splits from a +single elapsed number. Model load/warmup and initial eligibility work should be +reported separately from a retained worker's first foreground and warm queries. + +## CLI and desktop integration + +The CLI exposes these actual modes: + +```sh +searchi search --db searchi.sqlite contract --json +searchi search --db searchi.sqlite contract --performance --json +searchi search --db searchi.sqlite contract --debug-performance --json +``` + +Plain `--json` preserves the legacy result array. `--performance --json` emits +`SearchResponse` with basic metrics; `--debug-performance --json` emits the +same envelope with detailed spans. Normal terminal output prints results and +reports elapsed time/count on stderr, including CLI index/session initialization. +Debug terminal mode also prints the detailed metrics on stderr. Legacy Vec +interfaces remain wrappers for compatibility. The desktop should use a compact ordinary summary and an optional +diagnostic view, without confusing score/similarity with elapsed time or confidence. + +Both measured GUI commands start the native clock at command entry, include instrumented queue/lock waits, and finish before response serialization. UI timing separately captures input/debounce, lane/retry waits, invoke/round trip and synchronous DOM updates. Display latency ends at the DOM update; actual compositor paint and deferred media preview loading are not measured. Their overlapping boundaries must not be summed. Native metrics exclude frontend delays and post-return paint. A fast Rust request does not +prove a responsive first foreground interaction. Conversely, pending setup or +model loading must not be silently included in a warm-query claim. + +## Reproducible performance validation + +Automatic filesystem refreshes start a new frontend measurement at the refresh +event. `trigger: "filesystem_refresh"` distinguishes them from user searches; +`SHOWN` and `latest_displayed_ms` exclude time spent idle before that event. +`original_input` retains a fixed-size snapshot of the original debounce and +display timings. Refreshes do not replay previous attempts or append ancestor +traces. A refresh before the first response leaves that original response timing +null rather than inventing a first-input measurement. + +Record build profile, CPU/device and thread budget, VM/host resources, corpus and +active generation, filters, cold/warm state, concurrent indexing and setup costs. +Use multiple queries and p50/p95 distributions; retain raw reports. Compare basic +and debug mode on the same corpus to quantify instrumentation overhead. Debug +traces guide investigation but are not a substitute for representative benchmarks. + +Current recorded filename, actual Google seed and synthetic ANN evidence is in +[VALIDATION](VALIDATION.md) and [SEMANTIC](SEMANTIC.md). Media quality/timing is in +[MEDIA-QUALITY](MEDIA-QUALITY.md). Synthetic vectors measure index behavior only; +real Google inference remains required for product semantic search. Platform, +large-corpus, concurrent-indexing and GPU performance gates remain independent. +## Worker round-trip diagnostics + +The retained hybrid service now keeps a bounded 32-query vector LRU. Debug +`semantic.query_embedding` spans report `query_cache_hit` and `query_cache_miss`. +A hit reuses an actual model vector and performs no neural inference; it must +not carry the previous worker's tokenization/inference counters. Filters, current +source eligibility, ANN retrieval and fusion still execute. Warmup performs an +actual neural probe and does not seed an unseen foreground-query result. +Compare fresh queries, repeats and filter changes separately. The current +[EmbeddingGemma 2 audit](EMBEDDINGGEMMA2-AUDIT.md) includes paired real-CPU +measurements, startup boundaries and remaining indexing/media costs. + +Measured desktop failures return `{message, metrics}` and retain the trace for +the failed attempt. Busy retries are separate attempts. If an unexpected worker +task failure loses its collector, elapsed time is retained, the trace is null, +and a dropped-span marker makes the missing detail explicit. Error messages are +excluded from the exported timing report. + +Embedding IPC spans expose named `*_us` integer counters for request encoding, +writing, worker wait/response decode, validation, and native tokenization, batch +preparation, inference and pooling/normalization when genuinely reported. They +also expose `worker_token_count` and the GPU-to-CPU restart count. Microsecond +counters are durations, not occurrence counts; their suffix records units. + +The native worker clock is not aligned with Rust's collector origin. Native +stage values remain counters on the IPC span rather than invented positioned +child spans. Worker wait includes response-reader JSON parsing and scheduling; +it is not pure inference time. The encompassing round trip and its component +counters overlap. A fallback/restart may include setup outside the last worker's +reported neural pass; retain that distinction when diagnosing slow requests. + +## Small instrumentation-overhead diagnostic + +The deterministic example creates 1,000 local text files, builds/prewarms SQLite, +then alternates basic/debug order over 24 exact filename queries and 20 repeats. +Both modes must retrieve the correct file. It records 480 samples per mode, +outer-call p50/p95/mean and actual debug span counts, excluding fixture setup +and JSON serialization. It runs no model, media decoder or network. + +```powershell +cargo run --release -p searchi-core --example trace_overhead -- .searchi/trace-overhead.json +``` + +This tiny warm lexical corpus measures diagnostic overhead under the current +VM conditions. It does not certify production search latency or model-query +tracing overhead. Keep the raw report and profile/device/concurrency caveats; +repeat on representative corpora and compare instrumentation policies before +claiming low overhead across every retrieval mode. + +Windows CPU release diagnostic, 2026-10-12 (480 samples per mode): + +| Mode | Mean | p50 | p95 | +| --- | --- | --- | --- | +| Basic | 0.556 ms | 0.440 ms | 0.934 ms | +| Detailed | 0.968 ms | 0.819 ms | 1.615 ms | + +The detailed searches retained 14 spans each. These measurements include the +search itself and metrics assembly, and exclude fixture setup and JSON encoding. +Other work in the VM could contend for CPU. [Raw samples and measurement scope](evidence/windows-cpu-trace-overhead.json). +The recorded Windows x86-64 release run retained 480 raw samples per mode and +14 actual debug spans per request: + +| Mode | Mean | p50 | p95 | +| --- | ---: | ---: | ---: | +| Basic | 0.556 ms | 0.440 ms | 0.934 ms | +| Debug | 0.968 ms | 0.819 ms | 1.615 ms | + +[Raw samples and measurement contract](evidence/windows-cpu-trace-overhead.json) +include the active VM-contention caveat: native photo inference and other work +may overlap. This is an absolute small-corpus comparison, not an old-baseline +regression claim or proof of model/media tracing overhead. The extra cost +supports keeping detailed mode optional rather than claiming it is free. + +## Prepared-reader publication + +The desktop shares a bounded process-local prepared ANN between indexing and +its retained query service. The graph is registered before SQL activation and +matched by canonical database, dimensions, generation, policy and checksum. +`semantic.prepared_graph_lookup` measures that handoff; a successful handoff +sets `semantic.ann_load` cache_hit to one, without blob read/checksum/deserialization. +The query model and real-vector LRU remain warm. Eligibility refresh and query +inference still have costs and are measured independently. + +The actual CPU regression checks both a repeated cached query and an unseen +query after publication; the unseen input must record query_cache_hit zero. +See [RECOVERY-AND-HANDOFF.md](RECOVERY-AND-HANDOFF.md). Its single-document +request times do not establish representative-corpus p95, native first input +or foreground priority during indexing. + +## Shared inference admission + +The desktop now shares one CPU thread budget across its persistent query and +indexing workers. The default is two threads, clamped to available logical CPUs +and a maximum of eight. Indexing uses one thread; the query worker uses the +remaining threads (at least one). Pending searches have priority over new +background encodes, and document indexing yields between individual encodes. +Cached query vectors do not acquire an inference permit. + +Settings version two stores the total budget. Version-one settings counted +threads per worker; migration adds one background thread, capped at eight. +An explicit version-two budget of one remains one. Resource changes are rejected +during active semantic work; a failed settings save preserves the warm session. + +Debug traces record `semantic.inference.admission_wait` separately from native +inference. Admission waits are bounded at 30 seconds. Already executing native +inference cannot be preempted: a one-thread budget or single-core machine can +still delay a search during a long media encode. This budget covers admitted +inference worker threads, not extraction, decoding, ANN construction, rendering +or total operating-system CPU usage. + +The real Google CPU regression `budget_google_cpu` confirmed overlapping query +and background inference with a two-thread budget: one query thread, one indexing +thread, six background document calls, an unseen query taking 103.407 ms including +0.034 ms admission in the final scoped-cleanup run. These are Windows VM debug measurements on a small fixture, +not representative-corpus p95, native UI latency, GPU or scale evidence. diff --git a/fixtures/mixed/sources/llama__ggml-alloc.c b/fixtures/mixed/sources/llama__ggml-alloc.c new file mode 100644 index 0000000..b0c067b --- /dev/null +++ b/fixtures/mixed/sources/llama__ggml-alloc.c @@ -0,0 +1,1173 @@ +#include "ggml-alloc.h" +#include "ggml-backend-impl.h" +#include "ggml.h" +#include "ggml-impl.h" + +#include +#include +#include +#include +#include +#include + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MAX_FREE_BLOCKS 256 + +//#define GGML_ALLOCATOR_DEBUG + +//#define AT_PRINTF(...) GGML_LOG_DEBUG(__VA_ARGS__) +#define AT_PRINTF(...) + +// ops that return true for this function must not use restrict pointers for their backend implementations +bool ggml_op_can_inplace(enum ggml_op op) { + switch (op) { + case GGML_OP_FILL: + case GGML_OP_SCALE: + case GGML_OP_DIAG_MASK_ZERO: + case GGML_OP_DIAG_MASK_INF: + case GGML_OP_ADD: + case GGML_OP_ADD_ID: + case GGML_OP_ADD1: + case GGML_OP_SUB: + case GGML_OP_MUL: + case GGML_OP_DIV: + case GGML_OP_SQR: + case GGML_OP_SQRT: + case GGML_OP_LOG: + case GGML_OP_UNARY: + case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: + case GGML_OP_SILU_BACK: + case GGML_OP_RMS_NORM: + case GGML_OP_RMS_NORM_BACK: + case GGML_OP_CLAMP: + case GGML_OP_SOFT_MAX: + case GGML_OP_SOFT_MAX_BACK: + return true; + + default: + return false; + } +} + +static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) { + assert(alignment && !(alignment & (alignment - 1))); // power of 2 + size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment; + return offset + align; +} + +// tallocr + +struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer) { + void * base = ggml_backend_buffer_get_base(buffer); + size_t align = ggml_backend_buffer_get_alignment(buffer); + + assert(align && !(align & (align - 1))); // power of 2 + + struct ggml_tallocr talloc = (struct ggml_tallocr) { + /*.buffer = */ buffer, + /*.base = */ base, + /*.alignment = */ align, + /*.offset = */ aligned_offset(base, 0, align), + }; + return talloc; +} + +enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) { + size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor); + size = GGML_PAD(size, talloc->alignment); + + if (talloc->offset + size > ggml_backend_buffer_get_size(talloc->buffer)) { + GGML_LOG_ERROR("%s: not enough space in the buffer to allocate %s (needed %zu, available %zu)\n", + __func__, tensor->name, size, ggml_backend_buffer_get_size(talloc->buffer) - talloc->offset); + GGML_ABORT("not enough space in the buffer"); + } + + void * addr = (char *)ggml_backend_buffer_get_base(talloc->buffer) + talloc->offset; + talloc->offset += size; + + assert(((uintptr_t)addr % talloc->alignment) == 0); + + return ggml_backend_tensor_alloc(talloc->buffer, tensor, addr); +} + +// dynamic tensor allocator + +#define GGML_VBUFFER_MAX_CHUNKS 16 + +// relative memory address within an allocation that can be split into multiple buffers (chunks) +struct buffer_address { + int chunk; // index of a backend buffer + size_t offset; // local memory offset within the buffer +}; + +static const struct buffer_address GGML_BUFFER_ADDRESS_INVALID = { -1, SIZE_MAX }; + +static bool ggml_buffer_address_less(struct buffer_address a, struct buffer_address b) { + return a.chunk != b.chunk ? a.chunk < b.chunk : a.offset < b.offset; +} + +struct free_block { + size_t offset; + size_t size; +}; + +struct tallocr_chunk { + struct free_block free_blocks[MAX_FREE_BLOCKS]; + int n_free_blocks; + size_t max_size; +}; + +struct ggml_dyn_tallocr { + size_t alignment; + size_t max_chunk_size; + struct tallocr_chunk * chunks[GGML_VBUFFER_MAX_CHUNKS]; + int n_chunks; + +#ifdef GGML_ALLOCATOR_DEBUG + struct { + const struct ggml_tensor * tensor; + struct buffer_address addr; + } allocated_tensors[1024]; +#endif +}; + +static void ggml_dyn_tallocr_insert_block(struct tallocr_chunk * chunk, size_t offset, size_t size) { + GGML_ASSERT(chunk->n_free_blocks < MAX_FREE_BLOCKS && "out of free blocks"); + // insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster) + int insert_pos = 0; + while (insert_pos < chunk->n_free_blocks && chunk->free_blocks[insert_pos].offset < offset) { + insert_pos++; + } + // shift all blocks from insert_pos onward to make room for the new block + for (int i = chunk->n_free_blocks; i > insert_pos; i--) { + chunk->free_blocks[i] = chunk->free_blocks[i-1]; + } + // insert the new block + chunk->free_blocks[insert_pos].offset = offset; + chunk->free_blocks[insert_pos].size = size; + chunk->n_free_blocks++; +} + +static void ggml_dyn_tallocr_remove_block(struct tallocr_chunk * chunk, int idx) { + // shift all elements after idx by 1 to the left, overwriting the element at idx + for (int i = idx; i < chunk->n_free_blocks - 1; i++) { + chunk->free_blocks[i] = chunk->free_blocks[i+1]; + } + chunk->n_free_blocks--; +} + +static int ggml_dyn_tallocr_new_chunk(struct ggml_dyn_tallocr * alloc, size_t min_size) { + if (alloc->n_chunks >= GGML_VBUFFER_MAX_CHUNKS) { + return -1; + } + struct tallocr_chunk * chunk = calloc(1, sizeof(struct tallocr_chunk)); + chunk->n_free_blocks = 1; + chunk->free_blocks[0].offset = 0; + // available space in a chunk is limited to max_chunk_size, but can be higher if: + // 1. a single tensor exceeds the maximum, and cannot fit any other way + // 2. we are running out of chunks + // backends will either manage to allocate the larger size, or report an error. + chunk->free_blocks[0].size = MAX(min_size, alloc->max_chunk_size); + if (alloc->n_chunks == GGML_VBUFFER_MAX_CHUNKS - 1) { + chunk->free_blocks[0].size = SIZE_MAX/2; + } + alloc->chunks[alloc->n_chunks] = chunk; + alloc->n_chunks++; + return alloc->n_chunks - 1; +} + +#ifdef GGML_ALLOCATOR_DEBUG +static void add_allocated_tensor(struct ggml_dyn_tallocr * alloc, struct buffer_address addr, const struct ggml_tensor * tensor) { + for (int i = 0; i < 1024; i++) { + if (alloc->allocated_tensors[i].tensor == NULL) { + alloc->allocated_tensors[i].tensor = tensor; + alloc->allocated_tensors[i].addr = addr; + return; + } + } + GGML_ABORT("out of allocated_tensors"); +} +static void remove_allocated_tensor(struct ggml_dyn_tallocr * alloc, struct buffer_address addr, const struct ggml_tensor * tensor) { + for (int i = 0; i < 1024; i++) { + if (alloc->allocated_tensors[i].addr.chunk == addr.chunk && alloc->allocated_tensors[i].addr.offset == addr.offset) { + alloc->allocated_tensors[i].tensor = NULL; + return; + } + } + GGML_ABORT("tried to free tensor %s not found\n", tensor->name); +} +#endif + +static struct buffer_address ggml_dyn_tallocr_alloc(struct ggml_dyn_tallocr * alloc, size_t size, const struct ggml_tensor * tensor) { + size = aligned_offset(NULL, size, alloc->alignment); + + AT_PRINTF("%s: allocating %s (%zu bytes) - ", __func__, tensor->name, size); + + int best_fit_chunk = -1; + int best_fit_block = -1; + size_t max_avail = 0; + + // find the best fitting free block besides the last block, within any chunk + for (int c = 0; c < alloc->n_chunks; ++c) { + struct tallocr_chunk * chunk = alloc->chunks[c]; + size_t best_fit_size = SIZE_MAX; + for (int i = 0; i < chunk->n_free_blocks - 1; i++) { + struct free_block * block = &chunk->free_blocks[i]; + max_avail = MAX(max_avail, block->size); + if (block->size >= size && block->size <= best_fit_size) { + best_fit_chunk = c; + best_fit_block = i; + best_fit_size = block->size; + } + } + } + + if (best_fit_block == -1) { + // no suitable block found, try the last block (this may grow a chunks size) + int64_t best_reuse = INT64_MIN; + for (int c = 0; c < alloc->n_chunks; ++c) { + struct tallocr_chunk * chunk = alloc->chunks[c]; + if (chunk->n_free_blocks > 0) { + struct free_block * block = &chunk->free_blocks[chunk->n_free_blocks - 1]; + max_avail = MAX(max_avail, block->size); + int64_t reuse_factor = chunk->max_size - block->offset - size; + // reuse_factor < 0 : amount of extra memory that needs to be allocated + // reuse_factor = 0 : allocated free space exactly matches tensor size + // reuse_factor > 0 : superfluous memory that will remain unused + bool better_reuse = best_reuse < 0 && reuse_factor > best_reuse; + bool better_fit = reuse_factor >= 0 && reuse_factor < best_reuse; + if (block->size >= size && (better_reuse || better_fit)) { + best_fit_chunk = c; + best_fit_block = chunk->n_free_blocks - 1; + best_reuse = reuse_factor; + } + } + } + } + + if (best_fit_block == -1) { + // none of the existing chunks have enough space left + best_fit_chunk = ggml_dyn_tallocr_new_chunk(alloc, size); + best_fit_block = 0; + } + if (best_fit_chunk == -1) { + // since the last chunk always has virtually endless memory, this should never happen + GGML_LOG_ERROR("%s: not enough space in the buffer to allocate %zu bytes, largest block available %zu bytes\n", + __func__, size, max_avail); + GGML_ABORT("graph allocation: failed to reserve memory"); + } + + struct tallocr_chunk * chunk = alloc->chunks[best_fit_chunk]; + struct free_block * block = &chunk->free_blocks[best_fit_block]; + struct buffer_address addr = {.chunk = best_fit_chunk, .offset = block->offset }; + block->offset += size; + block->size -= size; + if (block->size == 0) { + // remove block if empty + ggml_dyn_tallocr_remove_block(chunk, best_fit_block); + } + + AT_PRINTF("block %d, offset %zu, chunk %d\n", best_fit_block, addr.offset, addr.chunk); + +#ifdef GGML_ALLOCATOR_DEBUG + add_allocated_tensor(alloc, addr, tensor); + size_t cur_max = addr.offset + size; + if (cur_max > chunk->max_size) { + // sort allocated_tensors by chunk/offset + for (int i = 0; i < 1024; i++) { + for (int j = i + 1; j < 1024; j++) { + if (ggml_buffer_address_less(alloc->allocated_tensors[j].addr, alloc->allocated_tensors[i].addr)) { + const struct ggml_tensor * tmp_tensor = alloc->allocated_tensors[i].tensor; + struct buffer_address tmp_addr = alloc->allocated_tensors[i].addr; + alloc->allocated_tensors[i].tensor = alloc->allocated_tensors[j].tensor; + alloc->allocated_tensors[i].addr = alloc->allocated_tensors[j].addr; + alloc->allocated_tensors[j].tensor = tmp_tensor; + alloc->allocated_tensors[j].addr = tmp_addr; + } + } + } + GGML_LOG_DEBUG("max_size[%d] = %.2f MB: tensors: ", addr.chunk, cur_max / 1024.0 / 1024.0); + for (int i = 0; i < 1024; i++) { + if (alloc->allocated_tensors[i].tensor) { + GGML_LOG_DEBUG("%s [%d: %zx-%zx] (%.2f MB) ", alloc->allocated_tensors[i].tensor->name, + alloc->allocated_tensors[i].addr.chunk, + alloc->allocated_tensors[i].addr.offset, + alloc->allocated_tensors[i].addr.offset + ggml_nbytes(alloc->allocated_tensors[i].tensor), + ggml_nbytes(alloc->allocated_tensors[i].tensor) / 1024.0 / 1024.0); + } + } + GGML_LOG_DEBUG("\n"); + } +#endif + + chunk->max_size = MAX(chunk->max_size, addr.offset + size); + + return addr; + + GGML_UNUSED(tensor); +} + +// this is a very naive implementation, but for our case the number of free blocks should be very small +static void ggml_dyn_tallocr_free_bytes(struct ggml_dyn_tallocr * alloc, struct buffer_address addr, size_t size) { + size = aligned_offset(NULL, size, alloc->alignment); + + struct tallocr_chunk * chunk = alloc->chunks[addr.chunk]; + + // see if we can merge with an existing block + for (int i = 0; i < chunk->n_free_blocks; i++) { + struct free_block * block = &chunk->free_blocks[i]; + // check if ptr is at the end of the block + if (block->offset + block->size == addr.offset) { + block->size += size; + // check if we can merge with the next block + if (i < chunk->n_free_blocks - 1) { + struct free_block * next = &chunk->free_blocks[i+1]; + if (block->offset + block->size == next->offset) { + block->size += next->size; + ggml_dyn_tallocr_remove_block(chunk, i+1); + } + } + return; + } + // check if ptr is at the beginning of the block + if (addr.offset + size == block->offset) { + block->offset = addr.offset; + block->size += size; + // check if we can merge with the previous block + if (i > 0) { + struct free_block * prev = &chunk->free_blocks[i-1]; + if (prev->offset + prev->size == block->offset) { + prev->size += block->size; + ggml_dyn_tallocr_remove_block(chunk, i); + } + } + return; + } + } + // otherwise, add a new block + ggml_dyn_tallocr_insert_block(chunk, addr.offset, size); +} + +static void ggml_dyn_tallocr_reset(struct ggml_dyn_tallocr * alloc) { + for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS; i++) { + free(alloc->chunks[i]); + alloc->chunks[i] = NULL; + } + alloc->n_chunks = 0; + +#ifdef GGML_ALLOCATOR_DEBUG + for (int i = 0; i < 1024; i++) { + alloc->allocated_tensors[i].tensor = NULL; + } +#endif +} + +static struct ggml_dyn_tallocr * ggml_dyn_tallocr_new(size_t alignment, size_t max_buffer_size) { + struct ggml_dyn_tallocr * alloc = (struct ggml_dyn_tallocr *)malloc(sizeof(struct ggml_dyn_tallocr)); + + *alloc = (struct ggml_dyn_tallocr) { + /*.alignment = */ alignment, + /*.max_chunk_size = */ MIN(max_buffer_size, SIZE_MAX/2), // clamp to avoid overflows + /*.chunks = */ {NULL}, + /*.n_chunks = */ 0, +#ifdef GGML_ALLOCATOR_DEBUG + /*.allocated_tensors = */ {{0}}, +#endif + }; + + ggml_dyn_tallocr_reset(alloc); + + return alloc; +} + +static void ggml_dyn_tallocr_free(struct ggml_dyn_tallocr * alloc) { + for (int i = 0; i < alloc->n_chunks; ++i) { + free(alloc->chunks[i]); + } + free(alloc); +} + +static size_t ggml_dyn_tallocr_max_size(struct ggml_dyn_tallocr * alloc, int chunk) { + return chunk < alloc->n_chunks ? alloc->chunks[chunk]->max_size : 0; +} + + +// virtual buffer with contiguous memory range, split into multiple backend buffers (chunks) + +struct vbuffer { + ggml_backend_buffer_t chunks[GGML_VBUFFER_MAX_CHUNKS]; +}; + +static void ggml_vbuffer_free(struct vbuffer * buf) { + if (buf == NULL) { + return; + } + for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS; ++i) { + ggml_backend_buffer_free(buf->chunks[i]); + } + free(buf); +} + +static size_t ggml_vbuffer_chunk_size(struct vbuffer * buf, int chunk) { + return buf->chunks[chunk] ? ggml_backend_buffer_get_size(buf->chunks[chunk]) : 0; +} + +static size_t ggml_vbuffer_size(struct vbuffer * buf) { + size_t size = 0; + for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS && buf->chunks[i]; ++i) { + size += ggml_backend_buffer_get_size(buf->chunks[i]); + } + return size; +} + +static struct vbuffer * ggml_vbuffer_alloc(ggml_backend_buffer_type_t buft, const struct ggml_dyn_tallocr * talloc, enum ggml_backend_buffer_usage usage) { + struct vbuffer * buf = (struct vbuffer *)calloc(1, sizeof(struct vbuffer)); + if (buf == NULL) { + return NULL; + } + + for (int n = 0; n < talloc->n_chunks; n++) { + size_t chunk_size = talloc->chunks[n]->max_size; + buf->chunks[n] = ggml_backend_buft_alloc_buffer(buft, chunk_size); + if (buf->chunks[n] == NULL) { + ggml_vbuffer_free(buf); + return NULL; + } + ggml_backend_buffer_set_usage(buf->chunks[n], usage); + } + return buf; +} + +static void ggml_vbuffer_tensor_alloc(struct vbuffer * buf, struct ggml_tensor * tensor, struct buffer_address buf_addr) { + void * base = ggml_backend_buffer_get_base(buf->chunks[buf_addr.chunk]); + void * addr = (char *)base + buf_addr.offset; + ggml_backend_tensor_alloc(buf->chunks[buf_addr.chunk], tensor, addr); +} + +static void ggml_vbuffer_reset(struct vbuffer * buf) { + for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS && buf->chunks[i]; ++i) { + ggml_backend_buffer_reset(buf->chunks[i]); + } +} + + +///////////////////////////////////// + +// graph allocator + +struct hash_node { + int n_children; + int n_views; + int buffer_id; + struct buffer_address addr; + bool allocated; +}; + +struct tensor_alloc { + int buffer_id; + struct buffer_address addr; + size_t size_max; // 0 = pre-allocated, unused, or view +}; + +struct leaf_alloc { + struct tensor_alloc leaf; +}; + +struct node_alloc { + struct tensor_alloc dst; + struct tensor_alloc src[GGML_MAX_SRC]; +}; + +struct ggml_gallocr { + ggml_backend_buffer_type_t * bufts; // [n_buffers] + struct vbuffer ** buffers; // [n_buffers] + struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers] + int n_buffers; + + struct ggml_hash_set hash_set; + struct hash_node * hash_values; // [hash_set.size] + + struct node_alloc * node_allocs; // [n_nodes] + int n_nodes; + + struct leaf_alloc * leaf_allocs; // [n_leafs] + int n_leafs; +}; + +ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs) { + ggml_gallocr_t galloc = (ggml_gallocr_t)calloc(1, sizeof(struct ggml_gallocr)); + GGML_ASSERT(galloc != NULL); + + galloc->bufts = calloc(n_bufs, sizeof(ggml_backend_buffer_type_t)); + GGML_ASSERT(galloc->bufts != NULL); + + galloc->buffers = calloc(n_bufs, sizeof(struct vbuffer *)); + GGML_ASSERT(galloc->buffers != NULL); + + galloc->buf_tallocs = calloc(n_bufs, sizeof(struct ggml_dyn_tallocr *)); + GGML_ASSERT(galloc->buf_tallocs != NULL); + + for (int i = 0; i < n_bufs; i++) { + galloc->bufts[i] = bufts[i]; + galloc->buffers[i] = NULL; + + // check if the same buffer type is used multiple times and reuse the same allocator + for (int j = 0; j < i; j++) { + if (bufts[i] == bufts[j]) { + galloc->buf_tallocs[i] = galloc->buf_tallocs[j]; + break; + } + } + + if (galloc->buf_tallocs[i] == NULL) { + size_t alignment = ggml_backend_buft_get_alignment(bufts[i]); + size_t max_size = ggml_backend_buft_get_max_size(bufts[i]); + galloc->buf_tallocs[i] = ggml_dyn_tallocr_new(alignment, max_size); + } + } + galloc->n_buffers = n_bufs; + + return galloc; +} + +ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft) { + return ggml_gallocr_new_n(&buft, 1); +} + +void ggml_gallocr_free(ggml_gallocr_t galloc) { + if (galloc == NULL) { + return; + } + + for (int i = 0; i < galloc->n_buffers; i++) { + if (galloc->buffers != NULL) { + // skip if already freed + bool freed = false; + for (int j = 0; j < i; j++) { + if (galloc->buffers[j] == galloc->buffers[i]) { + freed = true; + break; + } + } + if (!freed) { + ggml_vbuffer_free(galloc->buffers[i]); + } + } + if (galloc->buf_tallocs != NULL) { + // skip if already freed + bool freed = false; + for (int j = 0; j < i; j++) { + if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) { + freed = true; + break; + } + } + if (!freed) { + ggml_dyn_tallocr_free(galloc->buf_tallocs[i]); + } + } + } + + ggml_hash_set_free(&galloc->hash_set); + free(galloc->hash_values); + free(galloc->bufts); + free(galloc->buffers); + free(galloc->buf_tallocs); + free(galloc->node_allocs); + free(galloc->leaf_allocs); + free(galloc); +} + +typedef struct ggml_gallocr * ggml_gallocr_t; + +static struct hash_node * ggml_gallocr_hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) { + size_t i = ggml_hash_find_or_insert(&galloc->hash_set, t); + return &galloc->hash_values[i]; +} + +static bool ggml_gallocr_is_own(ggml_gallocr_t galloc, struct ggml_tensor * t) { + return ggml_gallocr_hash_get(galloc, t)->allocated; +} + +static bool ggml_gallocr_is_allocated(ggml_gallocr_t galloc, struct ggml_tensor * t) { + return t->data != NULL // tensor data already set externally + || t->buffer // tensor on external buffer (but not yet allocated) + || ggml_gallocr_is_own(galloc, t); // tensor will be allocated by galloc +} + +// free the extra space at the end if the new tensor is smaller +static void ggml_gallocr_free_extra_space(ggml_gallocr_t galloc, struct ggml_tensor * node, struct ggml_tensor * parent) { + struct hash_node * hn = ggml_gallocr_hash_get(galloc, node); + struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent); + + size_t parent_size = ggml_backend_buft_get_alloc_size(galloc->bufts[p_hn->buffer_id], parent); + size_t node_size = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node); + + GGML_ASSERT(parent_size >= node_size); + + // note: we want after the freeing the chunks to continue to be aligned + struct ggml_dyn_tallocr * p_alloc = galloc->buf_tallocs[p_hn->buffer_id]; + parent_size = aligned_offset(NULL, parent_size, p_alloc->alignment); + node_size = aligned_offset(NULL, node_size, p_alloc->alignment); + + if (parent_size > node_size) { + struct buffer_address p_addr = p_hn->addr; + p_addr.offset += node_size; + size_t extra_size = parent_size - node_size; + AT_PRINTF("freeing extra %zu bytes from parent %s for %s\n", extra_size, parent->name, node->name); + ggml_dyn_tallocr_free_bytes(p_alloc, p_addr, extra_size); + } +} + +static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) { + GGML_ASSERT(buffer_id >= 0); + struct hash_node * hn = ggml_gallocr_hash_get(galloc, node); + + if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_impl_is_view(node)) { + hn->allocated = true; + assert(hn->addr.offset == 0); + + // try to reuse a parent's buffer (inplace) + if (ggml_op_can_inplace(node->op)) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + struct ggml_tensor * parent = node->src[i]; + if (parent == NULL) { + continue; + } + + // if the node's data is external, then we cannot re-use it + if (!ggml_gallocr_is_own(galloc, parent)) { + AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data); + continue; + } + + // outputs cannot be reused + if (parent->flags & GGML_TENSOR_FLAG_OUTPUT || (parent->view_src != NULL && parent->view_src->flags & GGML_TENSOR_FLAG_OUTPUT)) { + AT_PRINTF("not reusing parent %s for %s as it is an output\n", parent->name, node->name); + continue; + } + + if (!ggml_are_same_layout(node, parent)) { + AT_PRINTF("not reusing parent %s for %s as layouts are different\n", parent->name, node->name); + continue; + } + + struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent); + if (p_hn->n_children == 1 && p_hn->n_views == 0) { + if (ggml_impl_is_view(parent)) { + struct ggml_tensor * view_src = parent->view_src; + struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src); + if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) { + AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name); + assert(view_src_hn->addr.chunk == p_hn->addr.chunk && view_src_hn->addr.offset == p_hn->addr.offset); + hn->buffer_id = p_hn->buffer_id; + hn->addr = p_hn->addr; + p_hn->allocated = false; // avoid freeing the parent + view_src_hn->allocated = false; + ggml_gallocr_free_extra_space(galloc, node, view_src); + return; + } + } else { + AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name); + hn->buffer_id = p_hn->buffer_id; + hn->addr = p_hn->addr; + p_hn->allocated = false; // avoid freeing the parent + ggml_gallocr_free_extra_space(galloc, node, parent); + return; + } + } + } + } + // allocate tensor from the buffer + struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id]; + ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id]; + size_t size = ggml_backend_buft_get_alloc_size(buft, node); + hn->buffer_id = buffer_id; + hn->addr = ggml_dyn_tallocr_alloc(alloc, size, node); + } +} + +static void ggml_gallocr_free_node(ggml_gallocr_t galloc, struct ggml_tensor * node) { + // graph outputs are never freed + if (node->flags & GGML_TENSOR_FLAG_OUTPUT) { + AT_PRINTF("not freeing output %s\n", node->name); + return; + } + + struct hash_node * hn = ggml_gallocr_hash_get(galloc, node); + int buffer_id = hn->buffer_id; + struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id]; + ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id]; + size_t size = ggml_backend_buft_get_alloc_size(buft, node); + + AT_PRINTF("%s: freeing %s at {chunk=%d, offset=%zu} (%zu bytes) - n_free_blocks = %d\n", + __func__, node->name, hn->addr.chunk, hn->addr.offset, size, alloc->chunks[hn->addr.chunk]->n_free_blocks); +#ifdef GGML_ALLOCATOR_DEBUG + remove_allocated_tensor(alloc, hn->addr, node); +#endif + + ggml_dyn_tallocr_free_bytes(alloc, hn->addr, size); + hn->allocated = false; +} + +static int get_node_buffer_id(const int * node_buffer_ids, int i) { + return node_buffer_ids ? node_buffer_ids[i] : 0; +} + +static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) { + // clear hash tables + ggml_hash_set_reset(&galloc->hash_set); + memset(galloc->hash_values, 0, sizeof(struct hash_node) * galloc->hash_set.size); + + // allocate leafs + // these may be tensors that the application is not using in the graph, but may still want to allocate for other purposes + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + ggml_gallocr_allocate_node(galloc, leaf, get_node_buffer_id(leaf_buffer_ids, i)); + } + + // count number of children and views + // allocate other graph inputs and leafs first to avoid overwriting them + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + + // TODO: better way to add external dependencies + // GGML_OP_NONE does not appear normally in the graph nodes, but is used by ggml-backend to add dependencies to + // control when some tensors are allocated and freed. in this case, the dependencies are in `src`, but the node + // itself is never used and should not be considered a dependency + if (ggml_impl_is_view(node) && node->op != GGML_OP_NONE) { + struct ggml_tensor * view_src = node->view_src; + ggml_gallocr_hash_get(galloc, view_src)->n_views += 1; + } + + if (node->flags & GGML_TENSOR_FLAG_INPUT) { + ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i)); + } + + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (src == NULL) { + continue; + } + + ggml_gallocr_hash_get(galloc, src)->n_children += 1; + + // allocate explicit inputs + if (src->flags & GGML_TENSOR_FLAG_INPUT) { + ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i)); + } + } + } + + // allocate tensors + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + int buffer_id = get_node_buffer_id(node_buffer_ids, i); + + // allocate parents (only leafs need to be allocated at this point) + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * parent = node->src[j]; + if (parent == NULL) { + continue; + } + ggml_gallocr_allocate_node(galloc, parent, buffer_id); + } + + // allocate node + ggml_gallocr_allocate_node(galloc, node, buffer_id); + + AT_PRINTF("exec: %s (%s) <= ", ggml_op_desc(node), node->name); + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * parent = node->src[j]; + if (parent == NULL) { + continue; + } + AT_PRINTF("%s", parent->name); + if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) { + AT_PRINTF(", "); + } + } + AT_PRINTF("\n"); + + // update parents + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * parent = node->src[j]; + if (parent == NULL) { + continue; + } + struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent); + p_hn->n_children -= 1; + + AT_PRINTF("parent %s: %d children, %d views, allocated: %d\n", + parent->name, p_hn->n_children, p_hn->n_views, p_hn->allocated); + + if (p_hn->n_children == 0 && p_hn->n_views == 0) { + if (ggml_impl_is_view(parent)) { + struct ggml_tensor * view_src = parent->view_src; + struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src); + view_src_hn->n_views -= 1; + AT_PRINTF("view_src %s: %d children, %d views\n", + view_src->name, view_src_hn->n_children, view_src_hn->n_views); + if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src_hn->allocated) { + ggml_gallocr_free_node(galloc, view_src); + } + } + else if (p_hn->allocated) { + ggml_gallocr_free_node(galloc, parent); + } + } + AT_PRINTF("\n"); + } + } +} + +static bool ggml_gallocr_reserve_n_impl( + ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids, bool no_alloc) { + size_t min_hash_size = graph->n_nodes + graph->n_leafs; + // add 25% margin to avoid hash collisions + min_hash_size += min_hash_size / 4; + + // initialize hash table + if (galloc->hash_set.size < min_hash_size) { + ggml_hash_set_free(&galloc->hash_set); + galloc->hash_set = ggml_hash_set_new(min_hash_size); + GGML_ASSERT(galloc->hash_set.keys != NULL); + + free(galloc->hash_values); + galloc->hash_values = malloc(sizeof(struct hash_node) * galloc->hash_set.size); + GGML_ASSERT(galloc->hash_values != NULL); + } + + // reset allocators + for (int i = 0; i < galloc->n_buffers; i++) { + ggml_dyn_tallocr_reset(galloc->buf_tallocs[i]); + } + + // allocate in hash table + ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids); + + // set the node_allocs from the hash table + if (galloc->n_nodes < graph->n_nodes) { + free(galloc->node_allocs); + galloc->node_allocs = calloc(graph->n_nodes, sizeof(struct node_alloc)); + GGML_ASSERT(galloc->node_allocs != NULL); + } + galloc->n_nodes = graph->n_nodes; + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + struct node_alloc * node_alloc = &galloc->node_allocs[i]; + if (node->view_src || node->data) { + node_alloc->dst.buffer_id = -1; + node_alloc->dst.addr = GGML_BUFFER_ADDRESS_INVALID; + node_alloc->dst.size_max = 0; + } else { + struct hash_node * hn = ggml_gallocr_hash_get(galloc, node); + node_alloc->dst.buffer_id = hn->buffer_id; + node_alloc->dst.addr = hn->addr; + node_alloc->dst.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node); + } + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (!src || src->view_src || src->data) { + node_alloc->src[j].buffer_id = -1; + node_alloc->src[j].addr = GGML_BUFFER_ADDRESS_INVALID; + node_alloc->src[j].size_max = 0; + } else { + struct hash_node * hn = ggml_gallocr_hash_get(galloc, src); + node_alloc->src[j].buffer_id = hn->buffer_id; + node_alloc->src[j].addr = hn->addr; + node_alloc->src[j].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], src); + } + } + } + if (galloc->n_leafs < graph->n_leafs) { + free(galloc->leaf_allocs); + galloc->leaf_allocs = calloc(graph->n_leafs, sizeof(galloc->leaf_allocs[0])); + GGML_ASSERT(galloc->leaf_allocs != NULL); + } + galloc->n_leafs = graph->n_leafs; + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf); + if (leaf->view_src || leaf->data) { + galloc->leaf_allocs[i].leaf.buffer_id = -1; + galloc->leaf_allocs[i].leaf.addr = GGML_BUFFER_ADDRESS_INVALID; + galloc->leaf_allocs[i].leaf.size_max = 0; + } else { + galloc->leaf_allocs[i].leaf.buffer_id = hn->buffer_id; + galloc->leaf_allocs[i].leaf.addr = hn->addr; + galloc->leaf_allocs[i].leaf.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf); + } + } + + // reallocate buffers if needed + for (int i = 0; i < galloc->n_buffers; i++) { + // if the buffer type is used multiple times, we reuse the same buffer + for (int j = 0; j < i; j++) { + if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) { + galloc->buffers[i] = galloc->buffers[j]; + break; + } + } + + // even if there are no tensors allocated in this buffer, we still need to allocate it to initialize views + bool realloc = galloc->buffers[i] == NULL; + size_t new_size = 0; + for (int c = 0; c < galloc->buf_tallocs[i]->n_chunks; c++) { + size_t cur_chunk_size = galloc->buffers[i] ? ggml_vbuffer_chunk_size(galloc->buffers[i], c) : 0; + size_t new_chunk_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i], c); + new_size += new_chunk_size; + if (new_chunk_size > cur_chunk_size) { + realloc = true; + } + } + if (realloc) { +#ifndef NDEBUG + { + size_t cur_size = galloc->buffers[i] ? ggml_vbuffer_size(galloc->buffers[i]) : 0; + if (cur_size > 0) { + GGML_LOG_DEBUG("%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\n", + __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0); + } + } +#endif + ggml_vbuffer_free(galloc->buffers[i]); + if (no_alloc) { + galloc->buffers[i] = NULL; + } else { + galloc->buffers[i] = ggml_vbuffer_alloc(galloc->bufts[i], galloc->buf_tallocs[i], GGML_BACKEND_BUFFER_USAGE_COMPUTE); + if (galloc->buffers[i] == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size); + return false; + } + } + } + } + + return true; +} + +void ggml_gallocr_reserve_n_size( + ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids, size_t * sizes) { + GGML_ASSERT(ggml_gallocr_reserve_n_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids, /*no_alloc =*/ true)); + for (int i = 0; i < galloc->n_buffers; i++) { + sizes[i] = 0; + for (int c = 0; c < galloc->buf_tallocs[i]->n_chunks; c++) { + sizes[i] += galloc->buf_tallocs[i]->chunks[c]->max_size; + } + } +} + +bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) { + return ggml_gallocr_reserve_n_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids, /*no_alloc =*/ false); +} + +bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) { + return ggml_gallocr_reserve_n(galloc, graph, NULL, NULL); +} + +static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * tensor, struct tensor_alloc * tensor_alloc) { + int buffer_id = tensor_alloc->buffer_id; + assert(tensor->data || tensor->view_src || ggml_backend_buft_get_alloc_size(galloc->bufts[buffer_id], tensor) <= tensor_alloc->size_max); + + if (tensor->view_src != NULL) { + if (tensor->buffer == NULL) { + assert(tensor_alloc->addr.offset == SIZE_MAX); + if (tensor->view_src->buffer == NULL) { + // this tensor was allocated without ggml-backend + return; + } + ggml_backend_view_init(tensor); + } + } else { + if (tensor->data == NULL) { + assert(tensor_alloc->addr.offset != SIZE_MAX); + assert(ggml_backend_buft_get_alloc_size(galloc->bufts[buffer_id], tensor) <= tensor_alloc->size_max); + ggml_vbuffer_tensor_alloc(galloc->buffers[buffer_id], tensor, tensor_alloc->addr); + } else { + if (tensor->buffer == NULL) { + // this tensor was allocated without ggml-backend + return; + } + } + } +} + +static bool ggml_gallocr_node_needs_realloc(ggml_gallocr_t galloc, struct ggml_tensor * node, struct tensor_alloc * talloc) { + size_t node_size = 0; + if (!node->data && !node->view_src) { + // If we previously had data but don't now then reallocate + if (talloc->buffer_id < 0) { + return false; + } + node_size = ggml_backend_buft_get_alloc_size(galloc->bufts[talloc->buffer_id], node); + } + return talloc->size_max >= node_size; +} + +static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph * graph) { + if (galloc->n_nodes != graph->n_nodes) { +#ifndef NDEBUG + GGML_LOG_DEBUG("%s: graph has different number of nodes\n", __func__); +#endif + return true; + } + + if (galloc->n_leafs != graph->n_leafs) { +#ifndef NDEBUG + GGML_LOG_DEBUG("%s: graph has different number of leafs\n", __func__); +#endif + return true; + } + + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + struct node_alloc * node_alloc = &galloc->node_allocs[i]; + + if (!ggml_gallocr_node_needs_realloc(galloc, node, &node_alloc->dst)) { +#ifndef NDEBUG + GGML_LOG_DEBUG("%s: node %s is not valid\n", __func__, node->name); +#endif + return true; + } + + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (src == NULL) { + continue; + } + if (!ggml_gallocr_node_needs_realloc(galloc, src, &node_alloc->src[j])) { +#ifndef NDEBUG + GGML_LOG_DEBUG("%s: src %d (%s) of node %s is not valid\n", __func__, j, src->name, node->name); +#endif + return true; + } + } + } + + return false; +} + +bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) { + if (ggml_gallocr_needs_realloc(galloc, graph)) { + if (galloc->n_buffers == 1) { +#ifndef NDEBUG + GGML_LOG_DEBUG("%s: reallocating buffers automatically\n", __func__); +#endif + if (!ggml_gallocr_reserve(galloc, graph)) { + return false; + } + } else { +#ifndef NDEBUG + GGML_LOG_DEBUG("%s: cannot reallocate multi buffer graph automatically, call reserve\n", __func__); +#endif + return false; + } + } + + // reset buffers + for (int i = 0; i < galloc->n_buffers; i++) { + if (galloc->buffers[i] != NULL) { + ggml_vbuffer_reset(galloc->buffers[i]); + } + } + + // allocate the graph tensors from the previous assignments + // leafs + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + struct leaf_alloc * leaf_alloc = &galloc->leaf_allocs[i]; + ggml_gallocr_init_tensor(galloc, leaf, &leaf_alloc->leaf); + } + // nodes + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + struct node_alloc * node_alloc = &galloc->node_allocs[i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + struct ggml_tensor * src = node->src[j]; + if (src == NULL) { + continue; + } + ggml_gallocr_init_tensor(galloc, src, &node_alloc->src[j]); + } + ggml_gallocr_init_tensor(galloc, node, &node_alloc->dst); + } + + return true; +} + +size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) { + GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers); + + if (galloc->buffers[buffer_id] == NULL) { + return 0; + } + + for (int i = 0; i < buffer_id; i++) { + if (galloc->buffers[i] == galloc->buffers[buffer_id]) { + // this buffer is the same as a previous one due to the same buffer type being used multiple times + // only return the buffer size the first time it appears to avoid double counting + return 0; + } + } + + return ggml_vbuffer_size(galloc->buffers[buffer_id]); +} + +// utils + +static struct ggml_tensor ** ggml_backend_alloc_ctx_tensors_from_buft_collect( + struct ggml_context * ctx, int * n_tensors) { + int n = 0; + for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + n++; + } + *n_tensors = n; + if (n == 0) { + return NULL; + } + + struct ggml_tensor ** tensors = (struct ggml_tensor **) malloc(n * sizeof(struct ggml_tensor *)); + if (tensors == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, n * sizeof(struct ggml_tensor *)); + return NULL; + } + int i = 0; + for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + tensors[i++] = t; + } + return tensors; +} + +ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) { + GGML_ASSERT(ggml_get_no_alloc(ctx) == true); + + int n_tensors = 0; + struct ggml_tensor ** tensors = ggml_backend_alloc_ctx_tensors_from_buft_collect(ctx, &n_tensors); + if (tensors == NULL) { + return NULL; + } + + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer_n(buft, tensors, n_tensors); + free(tensors); + return buffer; +} + +size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) { + GGML_ASSERT(ggml_get_no_alloc(ctx) == true); + + int n_tensors = 0; + struct ggml_tensor ** tensors = ggml_backend_alloc_ctx_tensors_from_buft_collect(ctx, &n_tensors); + if (tensors == NULL) { + return 0; + } + + size_t nbytes_total = ggml_backend_buft_get_alloc_size_n(buft, tensors, n_tensors); + free(tensors); + return nbytes_total; +} + +ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend) { + return ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_get_default_buffer_type(backend)); +} diff --git a/fixtures/mixed/sources/llama__unicode.cpp b/fixtures/mixed/sources/llama__unicode.cpp new file mode 100644 index 0000000..07b425f --- /dev/null +++ b/fixtures/mixed/sources/llama__unicode.cpp @@ -0,0 +1,1564 @@ +#include "unicode.h" +#include "unicode-data.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +size_t unicode_len_utf8(char src) { + const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; + uint8_t highbits = static_cast(src) >> 4; + return lookup[highbits]; +} + +static std::string unicode_cpts_to_utf8(const std::vector & cps) { + std::string result; + for (size_t i = 0; i < cps.size(); ++i) { + result.append(unicode_cpt_to_utf8(cps[i])); + } + return result; +} + +uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) { + assert(offset < utf8.size()); + if (!(utf8[offset + 0] & 0x80)) { + auto result = utf8[offset + 0]; + offset += 1; + return result; + } + if (!(utf8[offset + 0] & 0x40)) { + throw std::invalid_argument("invalid character"); + } + if (!(utf8[offset + 0] & 0x20)) { + if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) { + throw std::invalid_argument("invalid character"); + } + auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f); + offset += 2; + return result; + } + if (!(utf8[offset + 0] & 0x10)) { + if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) { + throw std::invalid_argument("invalid character"); + } + auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f); + offset += 3; + return result; + } + if (!(utf8[offset + 0] & 0x08)) { + if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) { + throw std::invalid_argument("invalid character"); + } + auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f); + offset += 4; + return result; + } + throw std::invalid_argument("failed to convert utf8 to codepoint"); +} + +//static std::vector unicode_cpt_to_utf16(uint32_t cpt) { +// std::vector result; +// if (/* 0x0000 <= cpt && */ cpt <= 0xffff) { +// result.emplace_back(cpt); +// return result; +// } +// if (0x10000 <= cpt && cpt <= 0x10ffff) { +// result.emplace_back(0xd800 | ((cpt - 0x10000) >> 10)); +// result.emplace_back(0xdc00 | ((cpt - 0x10000) & 0x03ff)); +// return result; +// } +// throw std::invalid_argument("failed to convert codepoint to utf16"); +//} + +//static std::vector unicode_cpts_to_utf16(const std::vector & cps) { +// std::vector result; +// for (size_t i = 0; i < cps.size(); ++i) { +// auto temp = unicode_cpt_to_utf16(cps[i]); +// result.insert(result.end(), temp.begin(), temp.end()); +// } +// return result; +//} + +//static uint32_t unicode_cpt_from_utf16(const std::vector & utf16, size_t & offset) { +// assert(offset < utf16.size()); +// if (((utf16[0] >> 10) << 10) != 0xd800) { +// auto result = utf16[offset + 0]; +// offset += 1; +// return result; +// } +// +// if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) { +// throw std::invalid_argument("invalid character"); +// } +// +// auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff)); +// offset += 2; +// return result; +//} + +//static std::vector unicode_cpts_from_utf16(const std::vector & utf16) { +// std::vector result; +// size_t offset = 0; +// while (offset < utf16.size()) { +// result.push_back(unicode_cpt_from_utf16(utf16, offset)); +// } +// return result; +//} + +static std::vector unicode_cpt_flags_array() { + std::vector cpt_flags(MAX_CODEPOINTS, unicode_cpt_flags::UNDEFINED); + + assert (unicode_ranges_flags.begin()[0].first == 0); + assert (unicode_ranges_flags.begin()[unicode_ranges_flags.size()-1].first == MAX_CODEPOINTS); + for (size_t i = 1; i < unicode_ranges_flags.size(); ++i) { + const auto range_ini = unicode_ranges_flags.begin()[i-1]; // codepoint_ini, flags + const auto range_end = unicode_ranges_flags.begin()[i]; // codepoint_end, flags + for (uint32_t cpt = range_ini.first; cpt < range_end.first; ++cpt) { + cpt_flags[cpt] = range_ini.second; + } + } + + for (auto cpt : unicode_set_whitespace) { + cpt_flags[cpt].is_whitespace = true; + } + + for (auto p : unicode_map_lowercase) { + cpt_flags[p.second].is_lowercase = true; + } + + for (auto p : unicode_map_uppercase) { + cpt_flags[p.second].is_uppercase = true; + } + + for (auto &range : unicode_ranges_nfd) { // start, last, nfd + cpt_flags[range.nfd].is_nfd = true; + } + + return cpt_flags; +} + +static std::unordered_map unicode_byte_to_utf8_map() { + std::unordered_map map; + for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~' + assert(0 <= ch && ch < 256); + map[ch] = unicode_cpt_to_utf8(ch); + } + for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬' + assert(0 <= ch && ch < 256); + map[ch] = unicode_cpt_to_utf8(ch); + } + for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ' + assert(0 <= ch && ch < 256); + map[ch] = unicode_cpt_to_utf8(ch); + } + auto n = 0; + for (int ch = 0; ch < 256; ++ch) { + if (map.find(ch) == map.end()) { + map[ch] = unicode_cpt_to_utf8(256 + n); + ++n; + } + } + return map; +} + +static std::unordered_map unicode_utf8_to_byte_map() { + std::unordered_map map; + for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~' + assert(0 <= ch && ch < 256); + map[unicode_cpt_to_utf8(ch)] = ch; + } + for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬' + assert(0 <= ch && ch < 256); + map[unicode_cpt_to_utf8(ch)] = ch; + } + for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ' + assert(0 <= ch && ch < 256); + map[unicode_cpt_to_utf8(ch)] = ch; + } + auto n = 0; + for (int ch = 0; ch < 256; ++ch) { + if (map.find(unicode_cpt_to_utf8(ch)) == map.end()) { + map[unicode_cpt_to_utf8(256 + n)] = ch; + ++n; + } + } + return map; +} + +static std::vector unicode_byte_encoding_process(const std::vector & bpe_words) { + std::vector bpe_encoded_words; + for (const auto & word : bpe_words) { + std::string text_utf; + auto utf_word = unicode_cpts_from_utf8(word); + for (size_t i = 0; i < utf_word.size(); ++i) { + text_utf += unicode_cpt_to_utf8(utf_word[i]); + } + + std::string encoded_token; + for (char & c : text_utf) { + encoded_token += unicode_byte_to_utf8(c); + } + bpe_encoded_words.emplace_back(encoded_token); + } + return bpe_encoded_words; +} + +// GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+ +static std::vector unicode_regex_split_custom_gpt2(const std::string & text, const std::vector & offsets) { + std::vector bpe_offsets; // store the offset of each word + bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size + + const auto cpts = unicode_cpts_from_utf8(text); + + size_t start = 0; + for (auto offset : offsets) { + const size_t offset_ini = start; + const size_t offset_end = start + offset; + assert(offset_end <= cpts.size()); + start = offset_end; + + static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; + auto _get_cpt = [&] (const size_t pos) -> uint32_t { + return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; + }; + + auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; + }; + + size_t _prev_end = offset_ini; + auto _add_token = [&] (const size_t end) -> size_t { + assert(_prev_end <= end && end <= offset_end); + size_t len = end - _prev_end; + if (len > 0) { + bpe_offsets.push_back(len); + } + _prev_end = end; + //if (len > 0) { + // std::string s = ""; + // for(size_t p = end-len; p < end; p++) + // s += unicode_cpt_to_utf8(cpts[p]); + // printf(">>> '%s'\n", s.c_str()); + //} + return len; + }; + + for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { + const uint32_t cpt = _get_cpt(pos); + const auto flags = _get_flags(pos); + + // regex: 's|'t|'re|'ve|'m|'ll|'d + if (cpt == '\'' && pos+1 < offset_end) { + uint32_t cpt_next = _get_cpt(pos+1); + if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { + pos += _add_token(pos+2); + continue; + } + if (pos+2 < offset_end) { + uint32_t cpt_next_next = _get_cpt(pos+2); + if ((cpt_next == 'r' && cpt_next_next == 'e') || + (cpt_next == 'v' && cpt_next_next == 'e') || + (cpt_next == 'l' && cpt_next_next == 'l')) { + pos += _add_token(pos+3); + continue; + } + } + } + + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); + // regex: ?\p{L}+ + if (flags2.is_letter) { + pos += (cpt == ' '); + while (flags2.is_letter) { + flags2 = _get_flags(++pos); + } + _add_token(pos); + continue; + } + // regex: ?\p{N}+ + if (flags2.is_number) { + pos += (cpt == ' '); + while (flags2.is_number) { + flags2 = _get_flags(++pos); + } + _add_token(pos); + continue; + } + // regex: ?[^\s\p{L}\p{N}]+ + if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) { + pos += (cpt == ' '); + while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) { + flags2 = _get_flags(++pos); + } + _add_token(pos); + continue; + } + + size_t num_whitespaces = 0; + while (_get_flags(pos+num_whitespaces).is_whitespace) { + num_whitespaces++; + } + + // regex: \s+(?!\S) + if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) { + pos += num_whitespaces - 1; + _add_token(pos); + continue; + } + + // regex: \s+ + if (num_whitespaces > 0) { + pos += num_whitespaces; + _add_token(pos); + continue; + } + + // no matches + _add_token(++pos); + } + } + + return bpe_offsets; +} + +// LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+" +static std::vector unicode_regex_split_custom_llama3(const std::string & text, const std::vector & offsets) { + std::vector bpe_offsets; // store the offset of each word + bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size + + const auto cpts = unicode_cpts_from_utf8(text); + + size_t start = 0; + for (auto offset : offsets) { + const size_t offset_ini = start; + const size_t offset_end = start + offset; + assert(offset_end <= cpts.size()); + start = offset_end; + + static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; + auto _get_cpt = [&] (const size_t pos) -> uint32_t { + return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; + }; + + auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; + }; + + size_t _prev_end = offset_ini; + auto _add_token = [&] (const size_t end) -> size_t { + assert(_prev_end <= end && end <= offset_end); + size_t len = end - _prev_end; + if (len > 0) { + bpe_offsets.push_back(len); + } + _prev_end = end; + //if (len > 0) { + // std::string s = ""; + // for(size_t p = end-len; p < end; p++) + // s += unicode_cpt_to_utf8(cpts[p]); + // printf(">>> '%s'\n", s.c_str()); + //} + return len; + }; + + for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { + const uint32_t cpt = _get_cpt(pos); + const auto flags = _get_flags(pos); + + // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive + if (cpt == '\'' && pos+1 < offset_end) { + uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); + if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { + pos += _add_token(pos+2); + continue; + } + if (pos+2 < offset_end) { + uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); + if ((cpt_next == 'r' && cpt_next_next == 'e') || + (cpt_next == 'v' && cpt_next_next == 'e') || + (cpt_next == 'l' && cpt_next_next == 'l')) { + pos += _add_token(pos+3); + continue; + } + } + } + + // regex: [^\r\n\p{L}\p{N}]?\p{L}+ + if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) { + if (flags.is_letter || _get_flags(pos+1).is_letter) { // one or more letters + pos++; + while (_get_flags(pos).is_letter) { + pos++; + } + _add_token(pos); + continue; + } + } + + // regex: \p{N}{1,3} + if (flags.is_number) { + size_t ini = pos; + while (_get_flags(pos).is_number) { + if (++pos - ini >= 3 ) { + _add_token(pos); + ini = pos; + } + } + _add_token(pos); + continue; + } + + // regex: ?[^\s\p{L}\p{N}]+[\r\n]* + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); + if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) { + pos += (cpt == ' '); + while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) { + flags2 = _get_flags(++pos); + } + uint32_t cpt2 = _get_cpt(pos); + while (cpt2 == '\r' || cpt2 == '\n') { + cpt2 = _get_cpt(++pos); + } + _add_token(pos); + continue; + } + + size_t num_whitespaces = 0; + size_t last_end_r_or_n = 0; + while (_get_flags(pos+num_whitespaces).is_whitespace) { + uint32_t cpt2 = _get_cpt(pos+num_whitespaces); + if (cpt2 == '\r' || cpt2 == '\n') { + last_end_r_or_n = pos + num_whitespaces + 1; + } + num_whitespaces++; + } + + // regex: \s*[\r\n]+ + if (last_end_r_or_n > 0) { + pos = last_end_r_or_n; + _add_token(pos); + continue; + } + + // regex: \s+(?!\S) + if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) { + pos += num_whitespaces - 1; + _add_token(pos); + continue; + } + + // regex: \s+ + if (num_whitespaces > 0) { + pos += num_whitespaces; + _add_token(pos); + continue; + } + + // no matches + _add_token(++pos); + } + } + + return bpe_offsets; +} + +static std::vector unicode_regex_split_custom_k2_horizon(const std::string & text, const std::vector & offsets) { + std::vector bpe_offsets; // store the offset of each word + bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size + + const auto cpts = unicode_cpts_from_utf8(text); + + size_t start = 0; + for (auto offset : offsets) { + const size_t offset_ini = start; + const size_t offset_end = start + offset; + assert(offset_end <= cpts.size()); + start = offset_end; + + static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; + auto _get_cpt = [&] (const size_t pos) -> uint32_t { + return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; + }; + + auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; + }; + + // K2-Horizon: letter runs are (?:\p{L}|\p{M}|\u200C|\u200D)+ + auto _is_k2_letter = [&] (const size_t pos) -> bool { + const uint32_t c = _get_cpt(pos); + if (c == 0x200C || c == 0x200D) { + return true; + } + const auto f = _get_flags(pos); + return f.is_letter || f.is_accent_mark; + }; + + size_t _prev_end = offset_ini; + auto _add_token = [&] (const size_t end) -> size_t { + assert(_prev_end <= end && end <= offset_end); + size_t len = end - _prev_end; + if (len > 0) { + bpe_offsets.push_back(len); + } + _prev_end = end; + return len; + }; + + for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { + const uint32_t cpt = _get_cpt(pos); + const auto flags = _get_flags(pos); + + // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive + if (cpt == '\'' && pos+1 < offset_end) { + uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); + if (cpt_next == 0x017F) { + cpt_next = 's'; // Unicode case-folding of long s + } + if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { + pos += _add_token(pos+2); + continue; + } + if (pos+2 < offset_end) { + uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); + if ((cpt_next == 'r' && cpt_next_next == 'e') || + (cpt_next == 'v' && cpt_next_next == 'e') || + (cpt_next == 'l' && cpt_next_next == 'l')) { + pos += _add_token(pos+3); + continue; + } + } + } + + // regex: [^\r\n\p{L}\p{N}]?(?:\p{L}|\p{M}|\u200C|\u200D)+ + if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) { + if (_is_k2_letter(pos) || _is_k2_letter(pos+1)) { // one or more letters/marks/ZWNJ/ZWJ + pos++; + while (_is_k2_letter(pos)) { + pos++; + } + _add_token(pos); + continue; + } + } + + // regex: \p{N}{1,3} + if (flags.is_number) { + size_t ini = pos; + while (_get_flags(pos).is_number) { + if (++pos - ini >= 3 ) { + _add_token(pos); + ini = pos; + } + } + _add_token(pos); + continue; + } + + // regex: ?[^\s\p{L}\p{N}]+[\r\n]* + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); + if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) { + pos += (cpt == ' '); + while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) { + flags2 = _get_flags(++pos); + } + uint32_t cpt2 = _get_cpt(pos); + while (cpt2 == '\r' || cpt2 == '\n') { + cpt2 = _get_cpt(++pos); + } + _add_token(pos); + continue; + } + + size_t num_whitespaces = 0; + size_t last_end_r_or_n = 0; + while (_get_flags(pos+num_whitespaces).is_whitespace) { + uint32_t cpt2 = _get_cpt(pos+num_whitespaces); + if (cpt2 == '\r' || cpt2 == '\n') { + last_end_r_or_n = pos + num_whitespaces + 1; + } + num_whitespaces++; + } + + // regex: \s*[\r\n]+ + if (last_end_r_or_n > 0) { + pos = last_end_r_or_n; + _add_token(pos); + continue; + } + + // regex: \s+(?!\S) + if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) { + pos += num_whitespaces - 1; + _add_token(pos); + continue; + } + + // regex: \s+ + if (num_whitespaces > 0) { + pos += num_whitespaces; + _add_token(pos); + continue; + } + + // no matches + _add_token(++pos); + } + } + + return bpe_offsets; +} + +// Qwen2 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" +static std::vector unicode_regex_split_custom_qwen2(const std::string & text, const std::vector & offsets) { + std::vector bpe_offsets; // store the offset of each word + bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size + + const auto cpts = unicode_cpts_from_utf8(text); + + size_t start = 0; + for (auto offset : offsets) { + const size_t offset_ini = start; + const size_t offset_end = start + offset; + assert(offset_end <= cpts.size()); + start = offset_end; + + static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; + auto _get_cpt = [&] (const size_t pos) -> uint32_t { + return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; + }; + + auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; + }; + + size_t _prev_end = offset_ini; + auto _add_token = [&] (const size_t end) -> size_t { + assert(_prev_end <= end && end <= offset_end); + size_t len = end - _prev_end; + if (len > 0) { + bpe_offsets.push_back(len); + } + _prev_end = end; + //if (len > 0) { + // std::string s = ""; + // for(size_t p = end-len; p < end; p++) + // s += unicode_cpt_to_utf8(cpts[p]); + // printf(">>> '%s'\n", s.c_str()); + //} + return len; + }; + + for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { + const uint32_t cpt = _get_cpt(pos); + const auto flags = _get_flags(pos); + + // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive + if (cpt == '\'' && pos+1 < offset_end) { + uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); + if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { + pos += _add_token(pos+2); + continue; + } + if (pos+2 < offset_end) { + uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); + if ((cpt_next == 'r' && cpt_next_next == 'e') || + (cpt_next == 'v' && cpt_next_next == 'e') || + (cpt_next == 'l' && cpt_next_next == 'l')) { + pos += _add_token(pos+3); + continue; + } + } + } + + // regex: [^\r\n\p{L}\p{N}]?\p{L}+ + if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) { + if (flags.is_letter || _get_flags(pos+1).is_letter) { // one or more letters + pos++; + while (_get_flags(pos).is_letter) { + pos++; + } + _add_token(pos); + continue; + } + } + + // regex: \p{N} + if (flags.is_number) { + pos++; + _add_token(pos); + continue; + } + + // regex: ?[^\s\p{L}\p{N}]+[\r\n]* + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); + if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) { + pos += (cpt == ' '); + while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) { + flags2 = _get_flags(++pos); + } + uint32_t cpt2 = _get_cpt(pos); + while (cpt2 == '\r' || cpt2 == '\n') { + cpt2 = _get_cpt(++pos); + } + _add_token(pos); + continue; + } + + size_t num_whitespaces = 0; + size_t last_end_r_or_n = 0; + while (_get_flags(pos+num_whitespaces).is_whitespace) { + uint32_t cpt2 = _get_cpt(pos+num_whitespaces); + if (cpt2 == '\r' || cpt2 == '\n') { + last_end_r_or_n = pos + num_whitespaces + 1; + } + num_whitespaces++; + } + + // regex: \s*[\r\n]+ + if (last_end_r_or_n > 0) { + pos = last_end_r_or_n; + _add_token(pos); + continue; + } + + // regex: \s+(?!\S) + if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) { + pos += num_whitespaces - 1; + _add_token(pos); + continue; + } + + // regex: \s+ + if (num_whitespaces > 0) { + pos += num_whitespaces; + _add_token(pos); + continue; + } + + // no matches + _add_token(++pos); + } + } + + return bpe_offsets; +} + +// Qwen3.5 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" +// Compared to Qwen2, letter-runs also consume Unicode combining marks (\p{M}): [\p{L}\p{M}]+ instead of \p{L}+ +static std::vector unicode_regex_split_custom_qwen35(const std::string & text, const std::vector & offsets) { + std::vector bpe_offsets; // store the offset of each word + bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size + + const auto cpts = unicode_cpts_from_utf8(text); + + size_t start = 0; + for (auto offset : offsets) { + const size_t offset_ini = start; + const size_t offset_end = start + offset; + assert(offset_end <= cpts.size()); + start = offset_end; + + static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; + auto _get_cpt = [&] (const size_t pos) -> uint32_t { + return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; + }; + + auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; + }; + + size_t _prev_end = offset_ini; + auto _add_token = [&] (const size_t end) -> size_t { + assert(_prev_end <= end && end <= offset_end); + size_t len = end - _prev_end; + if (len > 0) { + bpe_offsets.push_back(len); + } + _prev_end = end; + return len; + }; + + for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { + const uint32_t cpt = _get_cpt(pos); + const auto flags = _get_flags(pos); + + // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive + if (cpt == '\'' && pos+1 < offset_end) { + uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); + if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { + pos += _add_token(pos+2); + continue; + } + if (pos+2 < offset_end) { + uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); + if ((cpt_next == 'r' && cpt_next_next == 'e') || + (cpt_next == 'v' && cpt_next_next == 'e') || + (cpt_next == 'l' && cpt_next_next == 'l')) { + pos += _add_token(pos+3); + continue; + } + } + } + + // regex: [^\r\n\p{L}\p{N}]?[\p{L}\p{M}]+ + if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) { + if (flags.is_letter || flags.is_accent_mark || _get_flags(pos + 1).is_accent_mark || _get_flags(pos+1).is_letter) { + pos++; + while (_get_flags(pos).is_letter || _get_flags(pos).is_accent_mark) { + pos++; + } + _add_token(pos); + continue; + } + } + + // regex: \p{N} + if (flags.is_number) { + pos++; + _add_token(pos); + continue; + } + + // regex: ?[^\s\p{L}\p{M}\p{N}]+[\r\n]* + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); + if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_accent_mark | flags2.is_number) && flags.as_uint()) { + pos += (cpt == ' '); + while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_accent_mark | flags2.is_number) && flags2.as_uint()) { + flags2 = _get_flags(++pos); + } + uint32_t cpt2 = _get_cpt(pos); + while (cpt2 == '\r' || cpt2 == '\n') { + cpt2 = _get_cpt(++pos); + } + _add_token(pos); + continue; + } + + size_t num_whitespaces = 0; + size_t last_end_r_or_n = 0; + while (_get_flags(pos+num_whitespaces).is_whitespace) { + uint32_t cpt2 = _get_cpt(pos+num_whitespaces); + if (cpt2 == '\r' || cpt2 == '\n') { + last_end_r_or_n = pos + num_whitespaces + 1; + } + num_whitespaces++; + } + + // regex: \s*[\r\n]+ + if (last_end_r_or_n > 0) { + pos = last_end_r_or_n; + _add_token(pos); + continue; + } + + // regex: \s+(?!\S) + if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) { + pos += num_whitespaces - 1; + _add_token(pos); + continue; + } + + // regex: \s+ + if (num_whitespaces > 0) { + pos += num_whitespaces; + _add_token(pos); + continue; + } + + // no matches + _add_token(++pos); + } + } + + return bpe_offsets; +} + +template +static std::vector unicode_regex_split_stl(const std::basic_string & text, const std::basic_string & regex, const std::vector & offsets) { + using BidirIt = typename std::basic_string::const_iterator; +#ifdef _MSC_VER + // Bypass bug in MSVC: https://github.com/ggml-org/llama.cpp/issues/17830 + constexpr auto regex_flags = std::regex_constants::ECMAScript; +#else + constexpr auto regex_flags = std::regex_constants::optimize | std::regex_constants::nosubs; +#endif + std::basic_regex expr(regex, regex_flags); + std::vector bpe_offsets; // store the offset of each word + bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size + size_t start = 0; + for (auto offset : offsets) { + std::regex_iterator it(text.begin() + start, text.begin() + start + offset, expr); + std::regex_iterator end; + + int64_t start_idx = 0; + while (it != end) { + std::match_results match = *it; + if (match.position() > start_idx) { + bpe_offsets.emplace_back(match.position() - start_idx); + } + bpe_offsets.emplace_back(match.length()); + start_idx = match.position() + match.length(); + ++it; + } + + if (start_idx < (int64_t) offset) { + bpe_offsets.emplace_back(offset - start_idx); + } + start += offset; + } + + return bpe_offsets; +} + +// K2 system regex patterns (from tokenization_kimi.py): +// [\p{Han}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+ +static std::vector unicode_regex_split_custom_kimi_k2(const std::string & text, const std::vector & offsets) { + std::vector bpe_offsets; + bpe_offsets.reserve(offsets.size()); + + const auto cpts = unicode_cpts_from_utf8(text); + + size_t start = 0; + for (auto offset : offsets) { + const size_t offset_ini = start; + const size_t offset_end = start + offset; + assert(offset_end <= cpts.size()); + start = offset_end; + + static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; + auto _get_cpt = [&] (const size_t pos) -> uint32_t { + return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; + }; + + auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; + }; + + size_t _prev_end = offset_ini; + auto _add_token = [&] (const size_t end) -> size_t { + assert(_prev_end <= end && end <= offset_end); + size_t len = end - _prev_end; + if (len > 0) { + bpe_offsets.push_back(len); + } + _prev_end = end; + return len; + }; + + for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { + const uint32_t cpt = _get_cpt(pos); + const auto flags = _get_flags(pos); + + // Pattern 1: [\p{Han}]+ (Chinese characters) + if (unicode_cpt_is_han(cpt)) { + while (unicode_cpt_is_han(_get_cpt(pos))) { + pos++; + } + _add_token(pos); + continue; + } + + // Pattern 2 & 3: Letter words excluding Han characters with optional contractions + // [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?:'s|'t|'re|'ve|'m|'ll|'d)? + // [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?:'s|'t|'re|'ve|'m|'ll|'d)? + // Check if current char is a letter OR if current char could be a leading char and next char is a letter + bool is_letter_pattern = (flags.is_letter && !unicode_cpt_is_han(cpt)) || + (!(cpt == '\r' || cpt == '\n' || flags.is_letter || flags.is_number) && + _get_flags(pos + 1).is_letter && !unicode_cpt_is_han(_get_cpt(pos + 1))); + + if (is_letter_pattern) { + // Handle optional leading non-letter/non-number character + bool has_leading_char = false; + if (!(cpt == '\r' || cpt == '\n' || flags.is_letter || flags.is_number)) { + has_leading_char = true; + pos++; + } + + // Match letter sequence (excluding Han characters) + bool has_letters = false; + while (_get_flags(pos).is_letter && !unicode_cpt_is_han(_get_cpt(pos))) { + has_letters = true; + pos++; + } + + // Only proceed if we found letters (after potentially skipping leading char) + if (has_letters || (!has_leading_char && _get_flags(pos).is_letter && !unicode_cpt_is_han(_get_cpt(pos)))) { + if (!has_letters) pos++; // consume the first letter if we didn't already + + // Continue consuming letters + while (_get_flags(pos).is_letter && !unicode_cpt_is_han(_get_cpt(pos))) { + pos++; + } + + // Check for optional contractions (?:'s|'t|'re|'ve|'m|'ll|'d) + if (_get_cpt(pos) == '\'' && pos + 1 < offset_end) { + uint32_t cpt_next = unicode_tolower(_get_cpt(pos + 1)); + if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { + pos += 2; + } else if (pos + 2 < offset_end) { + uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos + 2)); + if ((cpt_next == 'r' && cpt_next_next == 'e') || + (cpt_next == 'v' && cpt_next_next == 'e') || + (cpt_next == 'l' && cpt_next_next == 'l')) { + pos += 3; + } + } + } + + _add_token(pos); + continue; + } else if (has_leading_char) { + // We consumed a leading char but found no letters, backtrack + pos--; + } + } + + // Pattern 4: \p{N}{1,3} (numbers 1-3 digits) + if (flags.is_number) { + size_t ini = pos; + while (_get_flags(pos).is_number) { + if (++pos - ini >= 3) { + _add_token(pos); + ini = pos; + } + } + _add_token(pos); + continue; + } + + // Pattern 5: ?[^\s\p{L}\p{N}]+[\r\n]* (optional space + non-word chars + optional newlines) + auto flags2 = (cpt == ' ' ? _get_flags(pos + 1) : flags); + if (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number) && flags2.as_uint()) { + pos += (cpt == ' '); + while (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number) && flags2.as_uint()) { + flags2 = _get_flags(++pos); + } + // Match optional [\r\n]* + uint32_t cpt2 = _get_cpt(pos); + while (cpt2 == '\r' || cpt2 == '\n') { + cpt2 = _get_cpt(++pos); + } + _add_token(pos); + continue; + } + + // Count whitespace characters + size_t num_whitespaces = 0; + size_t last_end_r_or_n = 0; + while (_get_flags(pos + num_whitespaces).is_whitespace) { + uint32_t cpt2 = _get_cpt(pos + num_whitespaces); + if (cpt2 == '\r' || cpt2 == '\n') { + last_end_r_or_n = pos + num_whitespaces + 1; + } + num_whitespaces++; + } + + // Pattern 6: \s*[\r\n]+ (whitespace with newlines) + if (last_end_r_or_n > 0) { + pos = last_end_r_or_n; + _add_token(pos); + continue; + } + + // Pattern 7: \s+(?!\S) (trailing whitespace) + if (num_whitespaces > 1 && _get_cpt(pos + num_whitespaces) != OUT_OF_RANGE) { + pos += num_whitespaces - 1; + _add_token(pos); + continue; + } + + // Pattern 8: \s+ (general whitespace) + if (num_whitespaces > 0) { + pos += num_whitespaces; + _add_token(pos); + continue; + } + + // No matches - consume single character + _add_token(++pos); + } + } + + return bpe_offsets; +} + +// AFMOE digit handling: splits digits with leading 1-2 based on total length modulo 3 +static std::vector unicode_regex_split_custom_afmoe(const std::string & text, const std::vector & offsets) { + std::vector bpe_offsets; + bpe_offsets.reserve(offsets.size()); + + const auto cpts = unicode_cpts_from_utf8(text); + + size_t start = 0; + for (auto offset : offsets) { + const size_t offset_ini = start; + const size_t offset_end = start + offset; + assert(offset_end <= cpts.size()); + start = offset_end; + + auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; + }; + + size_t _prev_end = offset_ini; + auto _add_token = [&] (const size_t end) -> size_t { + assert(_prev_end <= end && end <= offset_end); + size_t len = end - _prev_end; + if (len > 0) { + bpe_offsets.push_back(len); + } + _prev_end = end; + return len; + }; + + for (size_t pos = offset_ini; pos < offset_end; ) { + const auto flags = _get_flags(pos); + + // Handle digit sequences with special splitting logic + if (flags.is_number) { + size_t digit_start = pos; + size_t digit_count = 0; + + // Count consecutive digits + while (_get_flags(pos).is_number && pos < offset_end) { + digit_count++; + pos++; + } + + // Split based on total length modulo 3 + size_t remainder = digit_count % 3; + size_t current = digit_start; + + // Emit leading 1-2 digits if needed + if (remainder > 0) { + _add_token(current + remainder); + current += remainder; + } + + // Emit groups of 3 + while (current < digit_start + digit_count) { + _add_token(current + 3); + current += 3; + } + continue; + } + + // For non-digits, just move forward + pos++; + } + + // Add any remaining content + if (_prev_end < offset_end) { + _add_token(offset_end); + } + } + + return bpe_offsets; +} + +// regex: [^\n]+|[\n]+ +// splits text into runs of non-newline characters and runs of newline characters +static std::vector unicode_regex_split_custom_newlines(const std::string & text, const std::vector & offsets) { + std::vector bpe_offsets; + bpe_offsets.reserve(offsets.size()); + + const auto cpts = unicode_cpts_from_utf8(text); + + size_t start = 0; + for (auto offset : offsets) { + const size_t offset_ini = start; + const size_t offset_end = start + offset; + assert(offset_end <= cpts.size()); + start = offset_end; + + size_t pos = offset_ini; + while (pos < offset_end) { + const bool is_newline = (cpts[pos] == '\n'); + const size_t run_start = pos; + while (pos < offset_end && (cpts[pos] == '\n') == is_newline) { + pos++; + } + bpe_offsets.push_back(pos - run_start); + } + } + + return bpe_offsets; +} + +static std::vector unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector & offsets) { + std::vector bpe_offsets; + + if (regex_expr == "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)") { + bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets); + } else if ( + regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" || + regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { + bpe_offsets = unicode_regex_split_custom_llama3(text, offsets); + } else if ( + regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { + bpe_offsets = unicode_regex_split_custom_qwen2(text, offsets); + } else if ( + regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { + bpe_offsets = unicode_regex_split_custom_qwen35(text, offsets); + } else if ( + regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { + // K2-Horizon: llama3 splitter with marks + ZWNJ/ZWJ inside letter runs + // (the generic std::regex fallback cannot parse \p{..} on MSVC) + bpe_offsets = unicode_regex_split_custom_k2_horizon(text, offsets); + } else if (regex_expr == "\\p{Han}+") { + // K2's first pattern - handle all K2 patterns together + bpe_offsets = unicode_regex_split_custom_kimi_k2(text, offsets); + } else if (regex_expr == "\\p{AFMoE_digits}") { + // AFMOE digit pattern - use custom implementation for proper splitting + bpe_offsets = unicode_regex_split_custom_afmoe(text, offsets); + } else if (regex_expr == "[^\\n]+|[\\n]+") { + bpe_offsets = unicode_regex_split_custom_newlines(text, offsets); + } else if (regex_expr == "\\d{1,3}(?=(?:\\d{3})*\\b)") { + // tiny_aya digit grouping pattern from tokenizer.json: + // {"type": "Split", "pattern": {"Regex": "\\d{1,3}(?=(?:\\d{3})*\\b)"}, "behavior": "Isolated"} + // Splits digits into groups of 3 from the right (e.g., 1234567 -> 1, 234, 567) + // TODO: Revisit this regex, in case there are any subtle tokenization differences with the original regex. + bpe_offsets = unicode_regex_split_custom_afmoe(text, offsets); + } + + return bpe_offsets; +} + +// +// interface +// + +std::string unicode_cpt_to_utf8(uint32_t cpt) { + std::string result; + + if (/* 0x00 <= cpt && */ cpt <= 0x7f) { + result.push_back(cpt); + return result; + } + if (0x80 <= cpt && cpt <= 0x7ff) { + result.push_back(0xc0 | ((cpt >> 6) & 0x1f)); + result.push_back(0x80 | (cpt & 0x3f)); + return result; + } + if (0x800 <= cpt && cpt <= 0xffff) { + result.push_back(0xe0 | ((cpt >> 12) & 0x0f)); + result.push_back(0x80 | ((cpt >> 6) & 0x3f)); + result.push_back(0x80 | (cpt & 0x3f)); + return result; + } + if (0x10000 <= cpt && cpt <= 0x10ffff) { + result.push_back(0xf0 | ((cpt >> 18) & 0x07)); + result.push_back(0x80 | ((cpt >> 12) & 0x3f)); + result.push_back(0x80 | ((cpt >> 6) & 0x3f)); + result.push_back(0x80 | (cpt & 0x3f)); + return result; + } + + throw std::invalid_argument("invalid codepoint"); +} + +std::vector unicode_cpts_normalize_nfd(const std::vector & cpts) { + auto comp = [] (const uint32_t cpt, const range_nfd & range) { + return cpt < range.first; + }; + std::vector result(cpts.size()); + for (size_t i = 0; i < cpts.size(); ++i) { + const uint32_t cpt = cpts[i]; + auto it = std::upper_bound(unicode_ranges_nfd.begin(), unicode_ranges_nfd.end(), cpt, comp) - 1; + result[i] = (it->first <= cpt && cpt <= it->last) ? it->nfd : cpt; + } + return result; +} + +std::vector unicode_cpts_from_utf8(const std::string & utf8) { + std::vector result; + result.reserve(utf8.size()); + size_t offset = 0; + while (offset < utf8.size()) { + try { + result.push_back(unicode_cpt_from_utf8(utf8, offset)); + } + catch (const std::invalid_argument & /*ex*/) { + // Silently ignore invalid UTF-8 input to avoid leaking the exception beyond llama_tokenize + ++offset; + result.emplace_back(0xFFFD); // replacement character + } + } + return result; +} + +unicode_cpt_flags unicode_cpt_flags_from_cpt(const uint32_t cpt) { + static const unicode_cpt_flags undef(unicode_cpt_flags::UNDEFINED); + static const auto cpt_flags = unicode_cpt_flags_array(); + return cpt < cpt_flags.size() ? cpt_flags[cpt] : undef; +} + +unicode_cpt_flags unicode_cpt_flags_from_utf8(const std::string & utf8) { + static const unicode_cpt_flags undef(unicode_cpt_flags::UNDEFINED); + if (utf8.empty()) { + return undef; // undefined + } + size_t offset = 0; + return unicode_cpt_flags_from_cpt(unicode_cpt_from_utf8(utf8, offset)); +} + +std::string unicode_byte_to_utf8(uint8_t byte) { + static std::unordered_map map = unicode_byte_to_utf8_map(); + return map.at(byte); +} + +uint8_t unicode_utf8_to_byte(const std::string & utf8) { + static std::unordered_map map = unicode_utf8_to_byte_map(); + return map.at(utf8); +} + +uint32_t unicode_tolower(uint32_t cpt) { + // binary search + auto it = std::lower_bound(unicode_map_lowercase.begin(), unicode_map_lowercase.end(), cpt, + [](const std::pair & pair, uint32_t value) { + return pair.first < value; + }); + if (it != unicode_map_lowercase.end() && it->first == cpt) { + return it->second; + } + return cpt; // Return the original code point if no lowercase mapping is found +} + +bool unicode_cpt_is_han(uint32_t cpt) { + // Han character ranges (Chinese/CJK characters) + // CJK Unified Ideographs (most common) + if (cpt >= 0x4E00 && cpt <= 0x9FFF) return true; + + // CJK Extension A + if (cpt >= 0x3400 && cpt <= 0x4DBF) return true; + + // CJK Extension B + if (cpt >= 0x20000 && cpt <= 0x2A6DF) return true; + + // CJK Extension C + if (cpt >= 0x2A700 && cpt <= 0x2B73F) return true; + + // CJK Extension D + if (cpt >= 0x2B740 && cpt <= 0x2B81F) return true; + + // CJK Extension E + if (cpt >= 0x2B820 && cpt <= 0x2CEAF) return true; + + // CJK Extension F + if (cpt >= 0x2CEB0 && cpt <= 0x2EBEF) return true; + + // CJK Compatibility Ideographs + if (cpt >= 0xF900 && cpt <= 0xFAFF) return true; + + // CJK Compatibility Ideographs Supplement + if (cpt >= 0x2F800 && cpt <= 0x2FA1F) return true; + + return false; +} + +std::vector unicode_regex_split(const std::string & text, const std::vector & regex_exprs, bool byte_encode) { + if (text.empty()) { + return {}; + } + + // unicode categories + static const std::map k_ucat_enum = { + { "\\p{N}", unicode_cpt_flags::NUMBER }, + { "\\p{L}", unicode_cpt_flags::LETTER }, + { "\\p{P}", unicode_cpt_flags::PUNCTUATION }, + { "\\p{M}", unicode_cpt_flags::ACCENT_MARK }, + { "\\p{S}", unicode_cpt_flags::SYMBOL }, + { "\\p{Lu}", unicode_cpt_flags::LETTER }, // Uppercase letter + { "\\p{Ll}", unicode_cpt_flags::LETTER }, // Lowercase letter + { "\\p{Lt}", unicode_cpt_flags::LETTER }, // Titlecase letter + { "\\p{Lm}", unicode_cpt_flags::LETTER }, // Modifier letter + { "\\p{Lo}", unicode_cpt_flags::LETTER }, // Other letter + }; + + static const std::map k_ucat_cpt = { + { unicode_cpt_flags::NUMBER, 0xD1 }, + { unicode_cpt_flags::LETTER, 0xD2 }, + { unicode_cpt_flags::PUNCTUATION, 0xD3 }, + { unicode_cpt_flags::ACCENT_MARK, 0xD4 }, + { unicode_cpt_flags::SYMBOL, 0xD5 }, + }; + + static const std::map k_ucat_map = { + { unicode_cpt_flags::NUMBER, "\x30-\x39" }, // 0-9 + { unicode_cpt_flags::LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z + { unicode_cpt_flags::PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\} + { unicode_cpt_flags::ACCENT_MARK, "" }, // no sub-128 codepoints + { unicode_cpt_flags::SYMBOL, "\\\x24\\\x2B\x3C-\x3E\x5E\x60\\\x7C\\\x7E" }, // $+<=>^`|~ + }; + + // compute collapsed codepoints only if needed by at least one regex + bool need_collapse = false; + for (const auto & regex_expr : regex_exprs) { + // search for unicode categories + for (const auto & ucat : k_ucat_enum) { + if (std::string::npos != regex_expr.find(ucat.first)) { + need_collapse = true; + break; + } + } + } + + const auto cpts = unicode_cpts_from_utf8(text); + + // generate a "collapsed" representation of the text, where all codepoints are replaced by a single byte + // ref: https://github.com/ggml-org/llama.cpp/pull/6920#issuecomment-2081479935 + std::string text_collapsed; + if (need_collapse) { + // collapse all unicode categories + text_collapsed.resize(cpts.size()); + + for (size_t i = 0; i < cpts.size(); ++i) { + // keep single-byte codepoints as is + if (cpts[i] < 128) { + text_collapsed[i] = cpts[i]; + continue; + } + + const auto flags = unicode_cpt_flags_from_cpt(cpts[i]); + + if (flags.is_whitespace) { + //NOTE: C++ std::regex \s does not mach 0x85, Rust and Python regex does. + //text_collapsed[i] = (char) 0x85; // as whitespace fallback + text_collapsed[i] = (char) 0x0B; // as whitespace fallback + } else if (k_ucat_cpt.find(flags.category_flag()) != k_ucat_cpt.end()) { + text_collapsed[i] = k_ucat_cpt.at(flags.category_flag()); + } else { + text_collapsed[i] = (char) 0xD0; // fallback + } + } + } + + std::vector bpe_offsets = { cpts.size() }; + + for (const auto & regex_expr : regex_exprs) { + // first, see if we have an efficient custom regex implementation + auto tmp = unicode_regex_split_custom(text, regex_expr, bpe_offsets); + + if (!tmp.empty()) { + bpe_offsets = std::move(tmp); + continue; + } + + // fallback to general-purpose std::regex / std::wregex + try { + // if a unicode category is used in the regex, we use the collapsed text and replace the unicode category + // with the corresponding collapsed representation + bool use_collapsed = false; + for (const auto & ucat : k_ucat_enum) { + if (std::string::npos != regex_expr.find(ucat.first)) { + use_collapsed = true; + break; + } + } + const auto cpts_regex = unicode_cpts_from_utf8(regex_expr); + + if (use_collapsed) { + // sanity-check that the original regex does not contain any non-ASCII characters + for (size_t i = 0; i < cpts_regex.size(); ++i) { + if (cpts_regex[i] >= 128) { + throw std::runtime_error("Regex includes both unicode categories and non-ASCII characters - not supported"); + } + } + + // generate a collapsed representation of the regex + std::string regex_expr_collapsed; + + // track if we are inside [], because nested [] are not allowed + bool inside = false; + for (size_t i = 0; i < regex_expr.size(); ++i) { + if (regex_expr[i] == '[' && (i == 0 || regex_expr[i - 1] != '\\')) { + regex_expr_collapsed += '['; + inside = true; + continue; + } + + if (inside && regex_expr[i] == ']' && regex_expr[i - 1] != '\\') { + regex_expr_collapsed += ']'; + inside = false; + continue; + } + + // Match \p{...} Unicode properties of varying lengths + if (regex_expr[i + 0] == '\\' && i + 3 < regex_expr.size() && + regex_expr[i + 1] == 'p' && + regex_expr[i + 2] == '{') { + // Find the closing brace + size_t closing_brace = regex_expr.find('}', i + 3); + if (closing_brace != std::string::npos && closing_brace <= i + 10) { // reasonable limit + const std::string pat = regex_expr.substr(i, closing_brace - i + 1); + if (k_ucat_enum.find(pat) != k_ucat_enum.end()) { + if (!inside) { + regex_expr_collapsed += '['; + } + regex_expr_collapsed += k_ucat_cpt.at(k_ucat_enum.at(pat)); + regex_expr_collapsed += k_ucat_map.at(k_ucat_enum.at(pat)); + if (!inside) { + regex_expr_collapsed += ']'; + } + i = closing_brace; + continue; + } + } + } + + regex_expr_collapsed += regex_expr[i]; + } + + //printf("text_collapsed: %s\n", text_collapsed.c_str()); + //printf("regex_expr_collapsed: %s\n", regex_expr_collapsed.c_str()); + bpe_offsets = unicode_regex_split_stl(text_collapsed, regex_expr_collapsed, bpe_offsets); + } else { + // no unicode category used, we can use std::wregex directly + std::wstring wregex_expr(cpts_regex.begin(), cpts_regex.end()); + + // std::wregex \s does not mach non-ASCII whitespaces, using 0x0B as fallback + std::wstring wtext(cpts.begin(), cpts.end()); + for (size_t i = 0; i < wtext.size(); ++i) { + if (wtext[i] > 0x7F && unicode_cpt_flags_from_cpt(wtext[i]).is_whitespace) { + wtext[i] = 0x0B; + } + } + + //printf("text: %s\n", text.c_str()); + //printf("regex_expr: %s\n", regex_expr.c_str()); + bpe_offsets = unicode_regex_split_stl(wtext, wregex_expr, bpe_offsets); + } + } catch (std::regex_error & e) { + fprintf(stderr, "Failed to process regex: '%s'\n", regex_expr.c_str()); + fprintf(stderr, "Regex error: %s\n", e.what()); + throw std::runtime_error("Failed to process regex"); + } + } + + std::vector bpe_words; + bpe_words.reserve(bpe_offsets.size()); // reserve memory for the approximate size + + size_t start = 0; + for (size_t & offset : bpe_offsets) { + bpe_words.emplace_back(); + for (size_t i = start; i < start + offset; ++i) { + bpe_words.back() += unicode_cpt_to_utf8(cpts[i]); + } + start += offset; + } + + if (byte_encode) { + return unicode_byte_encoding_process(bpe_words); + } + + return bpe_words; +} diff --git a/tools/mixed-eval/README.md b/tools/mixed-eval/README.md new file mode 100644 index 0000000..e95e940 --- /dev/null +++ b/tools/mixed-eval/README.md @@ -0,0 +1,35 @@ +# Mixed retrieval evaluator + +`evaluate.py` verifies the frozen labels and every asset, copies only those +assets into an owned temporary corpus, and indexes it with the actual CLI. +Semantic indexing uses the real shared service and explicit Google worker/model +and CPU FFmpeg paths. Querying uses the same public read-only MCP search tool +that local agents use. Lexical and hybrid modes are reported separately; code +queries compare All files against Code scope. A semantic fallback is an error, +not a lexical result mislabeled as model evidence. Model files are not downloaded. + +```powershell +$env:SEARCHI_FORCE_CPU = '1' +python tools/mixed-eval/evaluate.py --verify-only +python -m unittest discover -s tools/mixed-eval -p 'test_*.py' +python tools/mixed-eval/evaluate.py --cli target/release/searchi.exe --models .searchi/models/embeddinggemma2 --worker .searchi/runtime/auto/Release/searchi-gemma-worker.exe --ffmpeg .searchi/media-runtime/bin/ffmpeg.exe --ffprobe .searchi/media-runtime/bin/ffprobe.exe --output .searchi/qa/mixed-v1.json +``` + +Use actual absolute runtime paths for your platform if these development paths +differ. Inference runs must be coordinated with other CPU measurements. Expect +media indexing to take minutes; its explicit one-hour bound is a safety ceiling, +not a performance target. Indexing and query errors remain visible. Temporary +corpora/indexes are cleaned after the owned sessions close. Partial query results +are retained in the requested output if a later request fails. + +Reported metrics are file-level top1, reciprocal rank and recall@5 after file +identity deduplication. Hard negatives in the first five and no-answer queries +returning any result are reported explicitly. This does **not** calibrate a +rejection threshold. Timestamp labels are retained but scored only for file +identity: use the separate media evaluator for segment localization. Unsupported +temporal filters are recorded as skipped, never counted as passed. + +The raw report retains queries, expected IDs, ordered hits, source locations, +scores, modes/scopes/languages and MCP roundtrip times. Those times exclude +process/model initialization and do not represent native UI latency. Keep raw +reports under ignored `.searchi`; review deliberate evidence before publishing. diff --git a/tools/mixed-eval/evaluate.py b/tools/mixed-eval/evaluate.py new file mode 100644 index 0000000..2337547 --- /dev/null +++ b/tools/mixed-eval/evaluate.py @@ -0,0 +1,156 @@ +"""Frozen mixed diagnostic via the installed CLI's shared engine and MCP API.""" +import argparse +from datetime import datetime, timezone +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +FIXTURE = ROOT / 'fixtures/mixed' + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def load_fixture(directory=FIXTURE): + raw = (directory / 'fixture.json').read_bytes() + lock = json.loads((directory / 'labels.lock.json').read_bytes()) + if hashlib.sha256(raw).hexdigest() != lock['fixture_sha256']: + raise ValueError('Frozen labels changed; create a new version, never refreeze v1') + fixture = json.loads(raw) + allowed = directory.parent.resolve() + ids, names = set(), set() + for asset in fixture['assets']: + path = (directory / asset['path']).resolve() + if not path.is_relative_to(allowed) or not path.is_file(): + raise ValueError('Asset must be an existing file below fixtures') + if asset['id'] in ids or path.name in names: + raise ValueError('Asset IDs and materialized filenames must be unique') + ids.add(asset['id']); names.add(path.name) + if path.stat().st_size != asset['bytes'] or digest(path) != asset['sha256']: + raise ValueError('Frozen asset changed: ' + asset['id']) + for query in fixture['queries']: + # Temporal/media labels can include structured entries; normalize below. + if any(identifier not in ids for identifier in expected_ids(query)): + raise ValueError('Unknown expected asset') + return fixture, lock + + +def expected_ids(query): + return {item if isinstance(item, str) else item['asset'] for item in query['expected']} + + +def score(query, hits): + expected = expected_ids(query) + unique = list(dict.fromkeys(hit['asset'] for hit in hits)) + ranks = [i + 1 for i, asset in enumerate(unique) if asset in expected] + first = min(ranks, default=None) + return {'first_relevant_rank': first, + 'top1': first == 1 if expected else None, + 'recall_at_5': len(expected.intersection(unique[:5])) / len(expected) if expected else None, + 'reciprocal_rank': 1 / first if first else 0 if expected else None, + 'no_answer_returned_any': bool(hits) if not expected else None, + 'hard_negative_in_top5': any(asset in query.get('hard_negatives', []) for asset in unique[:5])} + + +def summarize(rows): + groups = {} + for row in rows: + if 'unsupported' in row: + continue + key = (row['mode'], row['scope'], row['query']['category'], row['query']['language']) + groups.setdefault(key, []).append(row['score']) + result = [] + for key, values in sorted(groups.items()): + group = dict(zip(('mode', 'scope', 'category', 'language'), key)) + group['queries'] = len(values) + for metric in ('top1', 'recall_at_5', 'reciprocal_rank', 'no_answer_returned_any', 'hard_negative_in_top5'): + numbers = [v[metric] for v in values if v[metric] is not None] + group[metric] = sum(numbers) / len(numbers) if numbers else None + result.append(group) + return result + + +def run(args): + if os.environ.get('SEARCHI_FORCE_CPU') != '1': + raise ValueError('Set SEARCHI_FORCE_CPU=1; this evaluator must never run GPU inference') + fixture, lock = load_fixture() + manifest_path = ROOT / 'tools/embeddinggemma2/model-manifest.json' + manifest = json.loads(manifest_path.read_bytes()) + for model in manifest['files']: + path = args.models / model['path'] + if path.stat().st_size != model['bytes'] or digest(path) != model['sha256']: + raise ValueError('Model bytes differ from the pinned Google manifest') + spec = importlib.util.spec_from_file_location('searchi_benchmark', ROOT / 'tools/embeddinggemma2/benchmark.py') + benchmark = importlib.util.module_from_spec(spec); spec.loader.exec_module(benchmark) + args.output.parent.mkdir(parents=True, exist_ok=True) + rows = [] + with tempfile.TemporaryDirectory(prefix='searchi-mixed-') as temporary: + base = Path(temporary); corpus = base / 'corpus'; corpus.mkdir(); database = base / 'index.sqlite' + names = {} + for asset in fixture['assets']: + source = (FIXTURE / asset['path']).resolve() + shutil.copyfile(source, corpus / source.name) + names[source.name] = asset['id'] + subprocess.run([str(args.cli), 'index', '--db', str(database), '--root', str(corpus)], check=True, timeout=300) + subprocess.run([str(args.cli), 'embed', '--db', str(database), '--model-dir', str(args.models), + '--embedding-worker', str(args.worker), '--cpu-threads', '1', + '--ffmpeg', str(args.ffmpeg), '--ffprobe', str(args.ffprobe)], check=True, timeout=3600) + for mode in ('lexical_only', 'hybrid'): + command = [str(args.cli), 'mcp', '--db', str(database)] + if mode == 'hybrid': + command += ['--model-dir', str(args.models), '--embedding-worker', str(args.worker), '--cpu-threads', '1'] + session = benchmark.Session(command) + try: + session.mcp(1, 'initialize', {'protocolVersion': '2025-03-26', 'capabilities': {}, 'clientInfo': {'name': 'mixed-eval', 'version': '1'}}) + session.send({'jsonrpc': '2.0', 'method': 'notifications/initialized'}) + request = 2 + for query in fixture['queries']: + scopes = ('all_files', 'code') if query['category'] == 'code' else ('all_files',) + for scope in scopes: + options = {'query': query['text'], 'limit': 20, 'code_only': scope == 'code'} + filters = query.get('filters', {}) + if 'time_range_s' in filters: + rows.append({'mode': mode, 'scope': scope, 'query': query, 'unsupported': 'Temporal-range search filter is not supported'}) + continue + if 'extension' in filters: + options['extension'] = filters['extension'] + response, elapsed = session.mcp(request, 'tools/call', {'name': 'search', 'arguments': options}); request += 1 + if response.get('isError'): + raise RuntimeError('Shared search failed: ' + json.dumps(response)) + content = response['structuredContent'] + if content['mode'] != mode: + raise RuntimeError('Unexpected semantic fallback; not valid quality evidence') + hits = [{'asset': names[hit['filename']], 'score': hit['score'], 'match_kind': hit['match_kind'], 'source': hit.get('source')} for hit in content['results']] + rows.append({'mode': mode, 'scope': scope, 'query': query, 'hits': hits, 'score': score(query, hits), 'roundtrip_ms': elapsed}) + # Retain partial actual evidence if a later request fails. + args.output.write_text(json.dumps({'complete': False, 'rows': rows}, indent=2, ensure_ascii=False), encoding='utf-8') + finally: + session.close() + report = {'complete': True, 'recorded_at': datetime.now(timezone.utc).isoformat(), 'fixture_sha256': lock['fixture_sha256'], + 'model_manifest_sha256': digest(manifest_path), 'model': manifest, + 'device': 'cpu', 'cpu_threads': 1, 'cli_sha256': digest(args.cli), 'worker_sha256': digest(args.worker), + 'scope_note': 'Small mixed diagnostic, reused previously tested media labels; file-level relevance only. No temporal-localization or production-scale claims. MCP roundtrip excludes initialization and is not UI latency.', + 'rows': rows, 'summary': summarize(rows)} + args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding='utf-8') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--verify-only', action='store_true') + for flag in ('cli', 'models', 'worker', 'ffmpeg', 'ffprobe', 'output'): + parser.add_argument('--' + flag, type=Path) + args = parser.parse_args() + if args.verify_only: + fixture, lock = load_fixture(); print('Verified', len(fixture['assets']), 'assets and', len(fixture['queries']), 'frozen queries', lock['fixture_sha256']) + else: + if any(getattr(args, flag) is None for flag in ('cli', 'models', 'worker', 'ffmpeg', 'ffprobe', 'output')): + parser.error('Supply all explicit runtime paths and output, or use --verify-only') + run(args) diff --git a/tools/mixed-eval/test_evaluate.py b/tools/mixed-eval/test_evaluate.py new file mode 100644 index 0000000..42097ff --- /dev/null +++ b/tools/mixed-eval/test_evaluate.py @@ -0,0 +1,57 @@ +"""Meaningful scorer and frozen-provenance checks; no inference.""" +import importlib.util +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('mixed_eval', Path(__file__).with_name('evaluate.py')) +evaluate = importlib.util.module_from_spec(spec); spec.loader.exec_module(evaluate) + + +class EvaluationTests(unittest.TestCase): + def test_frozen_real_assets_verify(self): + fixture, _ = evaluate.load_fixture() + self.assertEqual(len(fixture['assets']), 32) + self.assertEqual(len(fixture['queries']), 91) + self.assertTrue(any(a.get('source_repository') for a in fixture['assets'])) + + def test_multiple_relevant_files_and_duplicate_hits(self): + query = {'expected': ['a', 'b'], 'hard_negatives': ['x']} + result = evaluate.score(query, [{'asset': x} for x in ['x', 'a', 'a', 'c', 'b']]) + self.assertEqual(result['first_relevant_rank'], 2) + self.assertEqual(result['recall_at_5'], 1) + self.assertEqual(result['reciprocal_rank'], .5) + self.assertTrue(result['hard_negative_in_top5']) + + def test_no_answer_is_not_counted_as_positive_recall(self): + result = evaluate.score({'expected': []}, [{'asset': 'unrelated'}]) + self.assertTrue(result['no_answer_returned_any']) + self.assertIsNone(result['recall_at_5']) + self.assertIsNone(result['top1']) + self.assertIsNone(result['reciprocal_rank']) + + def test_temporal_labels_file_relevance_only(self): + result = evaluate.score({'expected': [{'asset': 'speech', 'start_s': 20, 'end_s': 25}]}, [{'asset': 'speech', 'source': {'number': 0}}]) + self.assertTrue(result['top1']) # Deliberately no invented timestamp success metric. + + def test_label_tampering_rejected_before_assets(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / 'fixture.json').write_text('{"assets": []}') + (path / 'labels.lock.json').write_text('{"fixture_sha256": "wrong"}') + with self.assertRaisesRegex(ValueError, 'Frozen labels changed'): + evaluate.load_fixture(path) + + def test_summary_separates_scope_and_language(self): + rows = [] + for scope, lang, expected, found in [('all_files', 'en', ['a'], ['b']), ('code', 'en', ['a'], ['a']), ('all_files', 'de', [], ['b'])]: + query = {'expected': expected, 'language': lang, 'category': 'code'} + rows.append({'mode': 'hybrid', 'scope': scope, 'query': query, 'score': evaluate.score(query, [{'asset': a} for a in found])}) + groups = evaluate.summarize(rows) + self.assertEqual(len(groups), 3) + self.assertEqual(next(g for g in groups if g['scope'] == 'code')['top1'], 1) + self.assertEqual(next(g for g in groups if g['language'] == 'de')['no_answer_returned_any'], 1) + + +if __name__ == '__main__': + unittest.main()