diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c4c1f90 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +* text=auto eol=lf +*.png binary +*.jpg binary +*.ico binary diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..c969fc8 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,16 @@ +name: Core CI +on: [push, pull_request] +jobs: + core: + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - run: cargo fmt --all -- --check + - run: cargo clippy --workspace --all-targets -- -D warnings + - run: cargo test --workspace --locked diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c969fc8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,16 @@ +name: Core CI +on: [push, pull_request] +jobs: + core: + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - run: cargo fmt --all -- --check + - run: cargo clippy --workspace --all-targets -- -D warnings + - run: cargo test --workspace --locked diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ab46125 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +target/ +node_modules/ +dist/ +*.db +*.db-shm +*.db-wal +.searchi/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6fd8ca7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,9 @@ +# Searchi development + +- All tests in the Windows VM must use CPU only. Never enable GPU providers here. +- Keep the Rust engine independent of the desktop shell; CLI and UI share its API. +- Use bounded work, explicit errors, versioned storage, and offline defaults. +- Never substitute simulated embeddings for working semantic search. +- Add meaningful tests for retrieval, persistence, and filesystem lifecycle changes. +- Keep docs/ROADMAP.md current. Unexecuted platform checks remain unchecked. +- Agents own assigned paths and must coordinate shared interface changes. Do not commit other agents' unfinished work. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..4ee6173 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,528 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "cc" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.8", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "searchi-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "searchi-core", + "serde_json", +] + +[[package]] +name = "searchi-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "rusqlite", + "serde", + "tempfile", + "walkdir", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.8", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01016da373cd8f7ef12624f796309f5c31ba8d646dd08856c02cd741d823c622" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..7b4b96a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] +members = ["crates/searchi-core", "crates/searchi-cli"] +resolver = "2" + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "MIT" + +[workspace.dependencies] +anyhow = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9ddfe05 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 karl + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/searchi-cli/Cargo.toml b/crates/searchi-cli/Cargo.toml new file mode 100644 index 0000000..0a87754 --- /dev/null +++ b/crates/searchi-cli/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "searchi-cli" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Local, offline Searchi indexing and search command line interface" + +[[bin]] +name = "searchi" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap = { version = "4", features = ["derive"] } +searchi-core = { path = "../searchi-core" } +serde_json.workspace = true diff --git a/crates/searchi-cli/README.md b/crates/searchi-cli/README.md new file mode 100644 index 0000000..73e0f29 --- /dev/null +++ b/crates/searchi-cli/README.md @@ -0,0 +1,28 @@ +# Searchi CLI + +The CLI is a thin adapter over `searchi-core`; it performs no network requests or +GPU setup. Index only folders you explicitly choose. + +```powershell +cargo run -p searchi-cli -- index --db ./searchi.db --root ./documents +cargo run -p searchi-cli -- search --db ./searchi.db "termination clause" --limit 10 +cargo run -p searchi-cli -- search --db ./searchi.db "invoice" --json +cargo run -p searchi-cli -- stats --db ./searchi.db --json +``` + +The same arguments work on Linux and macOS. After building, invoke the `searchi` +binary directly instead of `cargo run -p searchi-cli --`. + +`index` creates the database if needed; its parent directory must already exist. +`search` and `stats` require an existing database so a mistyped path does not +silently create an empty index. Failures exit with a nonzero status and include +the affected folder or database path. Result limits must be between 1 and 1000. + +`search --json` emits an array of engine results, including the path, filename, +snippet, ranking score, and match kind. An empty search emits `[]`. `stats --json` +emits an object with `files` and `content_files`. JSON is the scripting interface; +the human-readable output may change as preview support develops. + +The foundation CLI searches filenames and extracted text. Semantic inference, +document formats beyond the core extractor, and desktop integration remain +tracked in the project roadmap; this interface does not fabricate embeddings. diff --git a/crates/searchi-cli/src/main.rs b/crates/searchi-cli/src/main.rs new file mode 100644 index 0000000..833c3f7 --- /dev/null +++ b/crates/searchi-cli/src/main.rs @@ -0,0 +1,153 @@ +//! Thin command-line adapter around the shared, CPU-only search engine. + +use std::path::PathBuf; + +use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; +use searchi_core::SearchIndex; + +#[derive(Debug, Parser)] +#[command(name = "searchi", version, about = "Search your local files offline")] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Discover files and refresh an index for one folder. + Index { + /// SQLite index file; parent directory must exist. + #[arg(long)] + db: PathBuf, + /// Folder to index recursively. + #[arg(long)] + root: PathBuf, + }, + /// Search filenames and indexed document content. + Search { + #[arg(long)] + db: PathBuf, + /// Search text; quote phrases containing spaces in your shell. + query: String, + /// Maximum number of returned results (1–1000). + #[arg(long, default_value_t = 20, value_parser = clap::value_parser!(u16).range(1..=1000))] + limit: u16, + /// Emit a JSON array for scripting. + #[arg(long)] + json: bool, + }, + /// Show the indexed file and content counts. + Stats { + #[arg(long)] + db: PathBuf, + #[arg(long)] + json: bool, + }, +} + +fn main() -> Result<()> { + match Cli::parse().command { + Command::Index { db, root } => { + let mut index = SearchIndex::open(&db) + .with_context(|| format!("Cannot open index {}", db.display()))?; + let report = index + .index_folder(&root) + .with_context(|| format!("Cannot index folder {}", root.display()))?; + println!( + "Discovered: {} Indexed: {} Removed: {} Skipped: {}", + report.discovered, report.indexed, report.removed, report.skipped + ); + } + Command::Search { + db, + query, + limit, + json, + } => { + if query.trim().is_empty() { + bail!("Search text cannot be empty"); + } + require_existing_index(&db)?; + let index = SearchIndex::open(&db) + .with_context(|| format!("Cannot open index {}", db.display()))?; + let results = index + .search(&query, usize::from(limit)) + .context("Search failed")?; + if json { + println!("{}", serde_json::to_string_pretty(&results)?); + } else if results.is_empty() { + println!("No matching files."); + } else { + for result in results { + println!("{} [{}]", result.path, result.match_kind); + if !result.snippet.is_empty() { + println!(" {}", result.snippet.replace(['\r', '\n'], " ")); + } + } + } + } + Command::Stats { db, json } => { + require_existing_index(&db)?; + let index = SearchIndex::open(&db) + .with_context(|| format!("Cannot open index {}", db.display()))?; + let stats = index.stats().context("Cannot read index statistics")?; + if json { + println!("{}", serde_json::to_string_pretty(&stats)?); + } else { + println!( + "Files: {} Files with searchable content: {}", + stats.files, stats.content_files + ); + } + } + } + Ok(()) +} + +/// Read commands must not silently create a new empty database after a typo. +fn require_existing_index(db: &std::path::Path) -> Result<()> { + if !db.is_file() { + bail!( + "Index {} does not exist; run `searchi index --db --root ` first", + db.display() + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_zero_or_excessive_result_limits() { + for limit in ["0", "1001"] { + assert!(Cli::try_parse_from([ + "searchi", "search", "--db", "index.db", "hello", "--limit", limit + ]) + .is_err()); + } + } + + #[test] + fn supports_json_search_and_quoted_query() { + let cli = Cli::try_parse_from([ + "searchi", + "search", + "--db", + "index.db", + "termination clause", + "--json", + ]) + .unwrap(); + assert!(matches!( + cli.command, + Command::Search { + json: true, + limit: 20, + .. + } + )); + } +} diff --git a/crates/searchi-cli/tests/commands.rs b/crates/searchi-cli/tests/commands.rs new file mode 100644 index 0000000..e56fbcd --- /dev/null +++ b/crates/searchi-cli/tests/commands.rs @@ -0,0 +1,90 @@ +//! Exercise the executable boundary: exit status, durable indexing, and JSON. +use std::{ + fs, + path::PathBuf, + process::{Command, Output}, + sync::atomic::{AtomicU64, Ordering}, +}; + +static NEXT_ID: AtomicU64 = AtomicU64::new(0); + +struct Fixture(PathBuf); + +impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "searchi-cli-test-{}-{}", + std::process::id(), + NEXT_ID.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + Self(path) + } + + fn run(&self, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_searchi")) + .current_dir(&self.0) + .args(args) + .output() + .unwrap() + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +#[test] +fn indexes_persists_and_returns_search_and_stats_json() { + let fixture = Fixture::new(); + fs::create_dir(fixture.0.join("documents")).unwrap(); + fs::write( + fixture.0.join("documents/contract.txt"), + "Supplier termination clause requires sixty days notice.", + ) + .unwrap(); + let indexed = fixture.run(&["index", "--db", "index.db", "--root", "documents"]); + assert!( + indexed.status.success(), + "{}", + String::from_utf8_lossy(&indexed.stderr) + ); + + // Each command is a new process, so retrieval proves the index survived exit. + let searched = fixture.run(&["search", "--db", "index.db", "termination", "--json"]); + assert!( + searched.status.success(), + "{}", + String::from_utf8_lossy(&searched.stderr) + ); + let results: serde_json::Value = serde_json::from_slice(&searched.stdout).unwrap(); + let results = results.as_array().unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0]["filename"], "contract.txt"); + assert!(results[0]["snippet"] + .as_str() + .unwrap() + .contains("termination")); + + let stats = fixture.run(&["stats", "--db", "index.db", "--json"]); + assert!(stats.status.success()); + let stats: serde_json::Value = serde_json::from_slice(&stats.stdout).unwrap(); + assert_eq!(stats["files"], 1); + assert_eq!(stats["content_files"], 1); +} + +#[test] +fn errors_are_nonzero_and_missing_database_is_not_created() { + let fixture = Fixture::new(); + let missing = fixture.run(&["search", "--db", "missing.db", "invoice", "--json"]); + assert!(!missing.status.success()); + assert!(String::from_utf8_lossy(&missing.stderr).contains("does not exist")); + assert!(!fixture.0.join("missing.db").exists()); + assert!(missing.stdout.is_empty()); + + let bad_root = fixture.run(&["index", "--db", "index.db", "--root", "missing-folder"]); + assert!(!bad_root.status.success()); + assert!(String::from_utf8_lossy(&bad_root.stderr).contains("missing-folder")); +} diff --git a/crates/searchi-core/Cargo.toml b/crates/searchi-core/Cargo.toml new file mode 100644 index 0000000..72fd156 --- /dev/null +++ b/crates/searchi-core/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "searchi-core" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +serde.workspace = true +rusqlite = { version = "0.32", features = ["bundled"] } +walkdir = "2" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/searchi-core/README.md b/crates/searchi-core/README.md new file mode 100644 index 0000000..ae29985 --- /dev/null +++ b/crates/searchi-core/README.md @@ -0,0 +1,49 @@ +# Searchi core + +The core has no desktop or inference dependencies. Its current retrieval engine +combines literal filename matching with SQLite FTS5 BM25 keyword search. It does +not implement semantic search. + +`SearchIndex::open` creates or migrates schema version 2. Current-schema opens do +not acquire a write lock. WAL allows independent readers to use the committed +index while reconciliation runs. All indexing changes are committed atomically; +a traversal or supported-file read error rolls back the entire reconciliation. + +`index_folder` applies `IndexOptions::default()`: directories named `.git`, +`node_modules`, and `target` are pruned. `index_folder_with_options` lets callers +replace those names. Matching directory names are ASCII case insensitive. An +explicit root is always traversed, even if its name is excluded. Symlinks are not +followed. The index's own database, WAL, and shared-memory files are excluded. + +Metadata includes byte size and modification time at nanosecond resolution. +Unchanged files bypass extraction. `IndexReport.indexed` counts newly extracted +or updated records; `unchanged` counts reused records; `skipped` counts new or +changed records whose content could not be extracted because of format, encoding, +or size. Unsupported files remain searchable by filename. Content reads are +limited to 2 MiB. Reconciliation tracks seen files in SQLite rather than retaining +every path in process memory. + +`search_with_options` supports canonical folder boundaries, extension, inclusive +size bounds, and inclusive Unix millisecond modification-time bounds. Filters +apply before limiting results. `search` retains the unfiltered API. Query text is +escaped as literal FTS terms, not interpreted as operators. Results are capped at +1,000 and queries at 4,096 bytes. + +## Known limits + +- Filename substring lookup uses `instr` and scans metadata. A dedicated substring + index and benchmarks are needed before claiming million-file latency targets. +- Files changed without changing size or modification time require a future + forced refresh/content-fingerprint option. Timestamp precision depends on the + filesystem. Indexing does not yet run a filesystem watcher. +- Metadata modification times before the Unix epoch or beyond the nanosecond + integer range are explicit errors. Unicode filename case folding is currently + limited by SQLite's built-in `lower` function. +- Lossy path conversion does not preserve non-Unicode native filenames. This must + be addressed before Linux production support for arbitrary byte filenames. +- The `files.root` ownership model does not yet support overlapping selected + roots independently. Use disjoint roots until root memberships are implemented. +- Extraction runs in the caller and a large reconciliation holds one transaction. + Bounded worker isolation, persistent jobs, and crash-resumable scans remain + production milestones. +- PDF, DOCX, OCR, media extraction, and real model inference remain unimplemented. diff --git a/crates/searchi-core/src/lib.rs b/crates/searchi-core/src/lib.rs new file mode 100644 index 0000000..965ea98 --- /dev/null +++ b/crates/searchi-core/src/lib.rs @@ -0,0 +1,482 @@ +//! Offline lexical search engine shared by the CLI and desktop shell. +//! +//! Reconciliation is atomic. Incomplete directory walks never delete records. +//! Text extraction is deliberately bounded; binary document extractors belong in +//! isolated workers in the next milestone, not in this indexing transaction. +use anyhow::{bail, Context, Result}; +use rusqlite::{params, Connection, OptionalExtension}; +use serde::Serialize; +use std::{ + fs, + io::Read, + path::{Path, PathBuf}, + time::UNIX_EPOCH, +}; +use walkdir::WalkDir; + +const SCHEMA_VERSION: i64 = 2; +const MAX_TEXT_BYTES: u64 = 2 * 1024 * 1024; +const MAX_RESULTS: usize = 1000; + +#[derive(Debug, Default, Serialize)] +pub struct IndexReport { + pub discovered: usize, + pub indexed: usize, + pub removed: usize, + pub skipped: usize, + pub unchanged: usize, +} + +/// Names of directories to prune anywhere below an explicitly chosen root. +#[derive(Debug, Clone)] +pub struct IndexOptions { + pub excluded_directory_names: Vec, +} + +impl Default for IndexOptions { + fn default() -> Self { + Self { + excluded_directory_names: vec![".git".into(), "node_modules".into(), "target".into()], + } + } +} + +/// Filters are independent of query syntax and apply before the result limit. +/// The folder is canonicalized and includes descendants, with a path-boundary check. +#[derive(Debug, Default, Clone)] +pub struct SearchOptions { + pub folder: Option, + pub extension: Option, + pub min_size: Option, + pub max_size: Option, + pub modified_after_unix_ms: Option, + pub modified_before_unix_ms: Option, +} + +#[derive(Debug, Serialize)] +pub struct SearchResult { + pub path: String, + pub filename: String, + pub snippet: String, + pub score: f64, + pub match_kind: String, +} + +#[derive(Debug, Serialize)] +pub struct IndexStats { + pub files: usize, + pub content_files: usize, +} + +pub struct SearchIndex { + connection: Connection, + database_path: Option, +} + +impl SearchIndex { + pub fn open(path: impl AsRef) -> Result { + let path = path.as_ref(); + let mut connection = Connection::open(path).context("open search database")?; + let database_path = if path == Path::new(":memory:") { + None + } else { + Some(fs::canonicalize(path).context("resolve database path")?) + }; + connection.busy_timeout(std::time::Duration::from_secs(5))?; + let version: i64 = connection.query_row("PRAGMA user_version", [], |row| row.get(0))?; + if !(0..=SCHEMA_VERSION).contains(&version) { + bail!("unsupported database version {version}; expected {SCHEMA_VERSION}"); + } + if version != SCHEMA_VERSION { + connection.execute_batch("PRAGMA journal_mode=WAL;")?; + let initialization = + connection.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + // Another opener may have completed initialization while we waited. + let version: i64 = + initialization.query_row("PRAGMA user_version", [], |row| row.get(0))?; + if !(0..=SCHEMA_VERSION).contains(&version) { + bail!("unsupported database version {version}"); + } + initialization.execute_batch( + " + CREATE TABLE IF NOT EXISTS files ( + id INTEGER PRIMARY KEY, path TEXT NOT NULL UNIQUE, + root TEXT NOT NULL, filename TEXT NOT NULL, content TEXT NOT NULL, + size INTEGER NOT NULL DEFAULT -1, mtime INTEGER NOT NULL DEFAULT -1, + extension TEXT NOT NULL DEFAULT ''); + CREATE INDEX IF NOT EXISTS files_root ON files(root); + CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5(filename, path, content, + content='files', content_rowid='id', tokenize='unicode61'); + CREATE TRIGGER IF NOT EXISTS files_insert AFTER INSERT ON files BEGIN + INSERT INTO search(rowid, filename, path, content) + VALUES (new.id, new.filename, new.path, new.content); END; + CREATE TRIGGER IF NOT EXISTS files_delete AFTER DELETE ON files BEGIN + INSERT INTO search(search, rowid, filename, path, content) + VALUES ('delete', old.id, old.filename, old.path, old.content); END; + CREATE TRIGGER IF NOT EXISTS files_update AFTER UPDATE ON files BEGIN + INSERT INTO search(search, rowid, filename, path, content) + VALUES ('delete', old.id, old.filename, old.path, old.content); + INSERT INTO search(rowid, filename, path, content) + VALUES (new.id, new.filename, new.path, new.content); END; + ", + )?; + if version == 1 { + initialization.execute_batch( + " + ALTER TABLE files ADD COLUMN size INTEGER NOT NULL DEFAULT -1; + ALTER TABLE files ADD COLUMN mtime INTEGER NOT NULL DEFAULT -1; + ALTER TABLE files ADD COLUMN extension TEXT NOT NULL DEFAULT ''; + PRAGMA user_version=2;", + )?; + } else { + initialization.execute_batch("PRAGMA user_version=2;")?; + } + initialization.commit()?; + } + Ok(Self { + connection, + database_path, + }) + } + + /// Reconcile one explicitly selected root. Symlinks are never followed. + /// Every file receives a filename record, including unsupported formats. + pub fn index_folder(&mut self, root: impl AsRef) -> Result { + self.index_folder_with_options(root, &IndexOptions::default()) + } + + pub fn index_folder_with_options( + &mut self, + root: impl AsRef, + options: &IndexOptions, + ) -> Result { + let root = fs::canonicalize(root).context("resolve index root")?; + if !root.is_dir() { + bail!("index root must be a directory"); + } + let root_key = root.to_string_lossy().into_owned(); + let transaction = self.connection.transaction()?; + transaction.execute_batch( + "CREATE TEMP TABLE IF NOT EXISTS seen_paths(path TEXT PRIMARY KEY); + DELETE FROM seen_paths;", + )?; + let mut report = IndexReport::default(); + for entry in WalkDir::new(&root) + .follow_links(false) + .into_iter() + .filter_entry(|entry| { + entry.depth() == 0 + || !entry.file_type().is_dir() + || !options.excluded_directory_names.iter().any(|name| { + entry + .file_name() + .to_string_lossy() + .eq_ignore_ascii_case(name) + }) + }) + { + let entry = entry.context("directory traversal failed; reconciliation rolled back")?; + if !entry.file_type().is_file() { + continue; + } + if let Some(database) = &self.database_path { + let path_key = entry.path().to_string_lossy(); + let db_key = database.to_string_lossy(); + if path_key == db_key + || path_key == format!("{db_key}-wal") + || path_key == format!("{db_key}-shm") + { + continue; + } + } + let path = entry.path().to_string_lossy().into_owned(); + let filename = entry.file_name().to_string_lossy().into_owned(); + transaction.execute("INSERT INTO seen_paths(path) VALUES (?1)", [&path])?; + report.discovered += 1; + let metadata = entry.metadata()?; + let size = i64::try_from(metadata.len()).context("file exceeds supported size")?; + let modified = metadata + .modified()? + .duration_since(UNIX_EPOCH) + .context("file timestamp predates Unix epoch")?; + let mtime = i64::try_from(modified.as_nanos()) + .context("file timestamp exceeds supported range")?; + let extension = entry + .path() + .extension() + .and_then(|value| value.to_str()) + .unwrap_or("") + .to_ascii_lowercase(); + let previous: Option<(i64, i64, String)> = transaction + .query_row( + "SELECT size,mtime,root FROM files WHERE path=?1", + [&path], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + if previous == Some((size, mtime, root_key.clone())) { + report.unchanged += 1; + continue; + } + let content = extract_text(entry.path()).with_context(|| format!("read {path}"))?; + if content.is_none() { + report.skipped += 1; + } + transaction.execute( + "INSERT INTO files(path,root,filename,content,size,mtime,extension) VALUES (?1,?2,?3,?4,?5,?6,?7) + ON CONFLICT(path) DO UPDATE SET root=excluded.root, filename=excluded.filename, + content=excluded.content, size=excluded.size, mtime=excluded.mtime, + extension=excluded.extension", + params![path, root_key, filename, content.unwrap_or_default(), size, mtime, extension], + )?; + report.indexed += 1; + } + report.removed = transaction.execute( + "DELETE FROM files WHERE root=?1 + AND NOT EXISTS (SELECT 1 FROM seen_paths WHERE seen_paths.path=files.path)", + [&root_key], + )?; + transaction.execute("DELETE FROM seen_paths", [])?; + transaction.commit()?; + Ok(report) + } + + /// Literal input is escaped before FTS parsing; user text cannot inject operators. + /// Exact filenames precede partial filenames, which precede content matches. + /// Filename substring matching currently scans metadata; a dedicated substring + /// index is required before claiming million-file latency targets. + pub fn search(&self, query: &str, limit: usize) -> Result> { + self.search_with_options(query, limit, &SearchOptions::default()) + } + + pub fn search_with_options( + &self, + query: &str, + limit: usize, + options: &SearchOptions, + ) -> Result> { + if options + .min_size + .zip(options.max_size) + .is_some_and(|(min, max)| min > max) + { + bail!("minimum size exceeds maximum size"); + } + if options + .modified_after_unix_ms + .zip(options.modified_before_unix_ms) + .is_some_and(|(min, max)| min > max) + { + bail!("modified-after exceeds modified-before"); + } + let folder = options + .folder + .as_ref() + .map(fs::canonicalize) + .transpose()? + .map(|path| path.to_string_lossy().into_owned()); + let folder_prefix = folder.as_ref().map(|value| { + format!( + "{}{}", + value.trim_end_matches(['/', '\\']), + std::path::MAIN_SEPARATOR + ) + }); + let extension = options + .extension + .as_ref() + .map(|value| value.trim_start_matches('.').to_ascii_lowercase()); + let min_size = options + .min_size + .map(i64::try_from) + .transpose() + .context("minimum size exceeds supported range")?; + let max_size = options + .max_size + .map(i64::try_from) + .transpose() + .context("maximum size exceeds supported range")?; + let query = query.trim(); + if query.is_empty() || limit == 0 { + return Ok(Vec::new()); + } + if query.len() > 4096 { + bail!("query exceeds 4096 bytes"); + } + let fts = query + .split_whitespace() + .map(|term| format!("\"{}\"", term.replace('"', "\"\""))) + .collect::>() + .join(" AND "); + let mut statement = self.connection.prepare( + "WITH hits AS ( + SELECT rowid AS id, bm25(search, 8.0, 2.0, 1.0) AS rank, + snippet(search, 2, '', '', ' … ', 24) AS excerpt + FROM search WHERE search MATCH ?1) + SELECT f.path, f.filename, COALESCE(h.excerpt, ''), COALESCE(h.rank, 0), + CASE WHEN lower(f.filename)=lower(?2) THEN 0 + WHEN instr(lower(f.filename), lower(?2)) > 0 THEN 1 ELSE 2 END AS tier + FROM files f LEFT JOIN hits h ON h.id=f.id + WHERE (h.id IS NOT NULL OR instr(lower(f.filename), lower(?2)) > 0) + AND (?4 IS NULL OR substr(f.path,1,length(?4))=?4) + AND (?5 IS NULL OR f.extension=?5) + AND (?6 IS NULL OR f.size>=?6) AND (?7 IS NULL OR f.size<=?7) + AND (?8 IS NULL OR f.mtime/1000000>=?8) AND (?9 IS NULL OR f.mtime/1000000<=?9) + ORDER BY tier, h.rank, f.path LIMIT ?3", + )?; + let results = statement + .query_map( + params![ + fts, + query, + limit.min(MAX_RESULTS) as i64, + folder_prefix, + extension, + min_size, + max_size, + options.modified_after_unix_ms, + options.modified_before_unix_ms + ], + |row| { + let tier: i64 = row.get(4)?; + Ok(SearchResult { + path: row.get(0)?, + filename: row.get(1)?, + snippet: row.get(2)?, + score: -row.get::<_, f64>(3)?, + match_kind: match tier { + 0 => "exact_filename", + 1 => "filename", + _ => "content", + } + .to_owned(), + }) + }, + )? + .collect::>>()?; + Ok(results) + } + + pub fn stats(&self) -> Result { + Ok(self.connection.query_row( + "SELECT count(*), count(NULLIF(content, '')) FROM files", + [], + |row| { + Ok(IndexStats { + files: row.get(0)?, + content_files: row.get(1)?, + }) + }, + )?) + } +} + +fn extract_text(path: &Path) -> Result> { + let extension = path + .extension() + .and_then(|value| value.to_str()) + .unwrap_or("") + .to_ascii_lowercase(); + if !matches!( + extension.as_str(), + "txt" + | "md" + | "rs" + | "py" + | "js" + | "ts" + | "tsx" + | "jsx" + | "json" + | "toml" + | "yaml" + | "yml" + | "csv" + | "html" + | "css" + | "xml" + | "log" + | "sh" + ) { + return Ok(None); + } + let file = fs::File::open(path)?; + if file.metadata()?.len() > MAX_TEXT_BYTES { + return Ok(None); + } + let mut bytes = Vec::new(); + file.take(MAX_TEXT_BYTES + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_TEXT_BYTES || bytes.contains(&0) { + return Ok(None); + } + Ok(String::from_utf8(bytes).ok()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn persistence_updates_deletes_and_exact_priority() -> Result<()> { + let folder = tempfile::tempdir()?; + let root = folder.path().join("docs"); + fs::create_dir(&root)?; + let db = folder.path().join("index.db"); + fs::write(root.join("contract.txt"), "supplier cancellation agreement")?; + fs::write(root.join("other.txt"), "contract contract contract")?; + let mut index = SearchIndex::open(&db)?; + assert_eq!(index.index_folder(&root)?.indexed, 2); + assert_eq!( + index.search("contract.txt", 10)?[0].match_kind, + "exact_filename" + ); + assert_eq!(index.search("cancellation", 10)?.len(), 1); + drop(index); + let mut index = SearchIndex::open(&db)?; + assert_eq!(index.stats()?.files, 2); + fs::write(root.join("contract.txt"), "replacement document")?; + fs::rename(root.join("other.txt"), root.join("renamed.txt"))?; + let report = index.index_folder(&root)?; + assert_eq!(report.removed, 1); + assert!(index.search("cancellation", 10)?.is_empty()); + assert!(index.search("other.txt", 10)?.is_empty()); + assert_eq!(index.search("renamed.txt", 10)?.len(), 1); + fs::remove_file(root.join("contract.txt"))?; + index.index_folder(&root)?; + assert_eq!(index.stats()?.files, 1); + Ok(()) + } + + #[test] + fn safe_queries_limits_and_unsupported_files() -> Result<()> { + let root = tempfile::tempdir()?; + fs::write( + root.path().join("notes.md"), + "German Kündigung and English cancellation", + )?; + fs::write(root.path().join("photo.jpg"), [0, 1, 2])?; + let mut index = SearchIndex::open(":memory:")?; + assert_eq!(index.index_folder(root.path())?.skipped, 1); + assert_eq!(index.search("Kündigung", 1)?.len(), 1); + assert!(index.search("\" OR *", 10)?.is_empty()); + assert!(index.search("", 10)?.is_empty()); + assert!(index.search("notes", 0)?.is_empty()); + assert_eq!(index.search("photo.jpg", 10)?.len(), 1); + Ok(()) + } + + #[test] + fn invalid_root_preserves_index_and_unknown_schema_is_rejected() -> Result<()> { + let folder = tempfile::tempdir()?; + fs::write(folder.path().join("notes.txt"), "persistent evidence")?; + let mut index = SearchIndex::open(":memory:")?; + index.index_folder(folder.path())?; + assert!(index.index_folder(folder.path().join("missing")).is_err()); + assert_eq!(index.search("evidence", 10)?.len(), 1); + let db = folder.path().join("future.db"); + let connection = Connection::open(&db)?; + connection.execute_batch("PRAGMA user_version=999")?; + drop(connection); + assert!(SearchIndex::open(db).is_err()); + Ok(()) + } +} diff --git a/crates/searchi-core/tests/index_options.rs b/crates/searchi-core/tests/index_options.rs new file mode 100644 index 0000000..44e6d9a --- /dev/null +++ b/crates/searchi-core/tests/index_options.rs @@ -0,0 +1,136 @@ +use anyhow::Result; +use searchi_core::{IndexOptions, SearchIndex, SearchOptions}; +use std::fs; + +#[test] +fn exclusions_database_and_unchanged_metadata() -> Result<()> { + let root = tempfile::tempdir()?; + for directory in [".git", "node_modules", "target", "private"] { + fs::create_dir(root.path().join(directory))?; + fs::write(root.path().join(directory).join("hidden.txt"), "secret")?; + } + fs::write(root.path().join("visible.txt"), "original content")?; + let mut index = SearchIndex::open(root.path().join("index.db"))?; + let options = IndexOptions { + excluded_directory_names: vec![ + ".git".into(), + "node_modules".into(), + "target".into(), + "private".into(), + ], + }; + assert_eq!( + index + .index_folder_with_options(root.path(), &options)? + .indexed, + 1 + ); + assert_eq!(index.stats()?.files, 1); + let unchanged = index.index_folder_with_options(root.path(), &options)?; + assert_eq!(unchanged.unchanged, 1); + assert_eq!(unchanged.indexed, 0); + assert!(index.search("secret", 10)?.is_empty()); + assert!(index.search("index.db", 10)?.is_empty()); + fs::write( + root.path().join("visible.txt"), + "changed and longer content", + )?; + assert_eq!( + index + .index_folder_with_options(root.path(), &options)? + .indexed, + 1 + ); + assert!(index.search("original", 10)?.is_empty()); + assert_eq!(index.search("changed", 10)?.len(), 1); + // A new exclusion removes previously indexed entries from this root. + let mut options = IndexOptions::default(); + index.index_folder_with_options(root.path(), &options)?; + assert_eq!(index.search("secret", 10)?.len(), 1); + options.excluded_directory_names.push("private".into()); + assert_eq!( + index + .index_folder_with_options(root.path(), &options)? + .removed, + 1 + ); + Ok(()) +} + +#[test] +fn filters_apply_before_limit_and_respect_folder_boundaries() -> Result<()> { + let root = tempfile::tempdir()?; + let docs = root.path().join("docs"); + let sibling = root.path().join("docs-more"); + fs::create_dir(&docs)?; + fs::create_dir(&sibling)?; + fs::write(docs.join("shared.md"), "shared short")?; + fs::write(docs.join("shared.txt"), "shared considerably longer")?; + fs::write(sibling.join("shared.md"), "shared sibling")?; + let mut index = SearchIndex::open(":memory:")?; + index.index_folder(root.path())?; + let options = SearchOptions { + folder: Some(docs), + extension: Some(".MD".into()), + max_size: Some(15), + ..Default::default() + }; + let results = index.search_with_options("shared", 1, &options)?; + assert_eq!(results.len(), 1); + assert_eq!(results[0].filename, "shared.md"); + assert!(!results[0].path.contains("docs-more")); + let options = SearchOptions { + min_size: Some(20), + ..Default::default() + }; + assert_eq!(index.search_with_options("shared", 10, &options)?.len(), 1); + let options = SearchOptions { + modified_before_unix_ms: Some(0), + ..Default::default() + }; + assert!(index + .search_with_options("shared", 10, &options)? + .is_empty()); + let options = SearchOptions { + min_size: Some(20), + max_size: Some(10), + ..Default::default() + }; + assert!(index.search_with_options("shared", 10, &options).is_err()); + Ok(()) +} + +#[test] +fn migrates_v1_database_without_losing_search_records() -> Result<()> { + let root = tempfile::tempdir()?; + let db = root.path().join("legacy.db"); + let connection = rusqlite::Connection::open(&db)?; + connection.execute_batch( + "CREATE TABLE files(id INTEGER PRIMARY KEY, path TEXT NOT NULL UNIQUE, + root TEXT NOT NULL, filename TEXT NOT NULL, content TEXT NOT NULL); + PRAGMA user_version=1;", + )?; + drop(connection); + let mut index = SearchIndex::open(&db)?; + fs::write(root.path().join("sample.txt"), "migration evidence")?; + index.index_folder(root.path())?; + assert_eq!(index.search("evidence", 10)?.len(), 1); + Ok(()) +} + +#[test] +fn readers_open_and_search_during_uncommitted_writer() -> Result<()> { + let root = tempfile::tempdir()?; + let db = root.path().join("index.db"); + fs::write(root.path().join("notes.txt"), "committed evidence")?; + let mut index = SearchIndex::open(&db)?; + index.index_folder(root.path())?; + let writer = rusqlite::Connection::open(&db)?; + writer.execute_batch("BEGIN IMMEDIATE; UPDATE files SET content='uncommitted replacement';")?; + // Opening a current schema must not issue DDL or acquire a write lock. + let reader = SearchIndex::open(&db)?; + assert_eq!(reader.search("evidence", 10)?.len(), 1); + assert!(reader.search("replacement", 10)?.is_empty()); + writer.execute_batch("ROLLBACK;")?; + Ok(()) +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..634720d --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,38 @@ +# Architecture and boundaries + +The Rust engine owns storage and search. Its callers do not construct SQL. +The CLI and desktop shell share the engine; the Tauri bridge runs filesystem +work outside the UI thread using independent database connections. + +The current lexical index is a foundation. It uses SQLite WAL and FTS5 with +exact-name priority and source snippets. Reconciliation is transactional: +incomplete traversal must not delete previously indexed files. A root that +cannot be read produces an error, not an empty successful reconciliation. + +The next storage design separates metadata discovery, persistent extraction +jobs and passage publication. Bounded workers publish small batches so search +can see newly indexed names before a complete root finishes. A generation +marker only removes stale entries after a successful walk. Watchers enqueue +changes; reconciliation catches lost events. Overlapping roots need explicit +ownership before root removal is implemented. + +Semantic inference is a replaceable CPU adapter with explicit model artifact +download, pinned checksums, normalization and versioned chunk vectors. Search +must never download a model implicitly. No synthetic embeddings qualify as +semantic search. An exact lexical match keeps priority during hybrid fusion. + +Document extractors eventually run in isolated workers with input size, +timeout and output limits. PDF pages, code lines and media timestamps are +stored as source locations, not inferred by the frontend. + +Indexes contain private extracted text. No telemetry or remote search is +enabled. Model downloads and release checks must be explicit operations. + +## Known foundation limits + +- Substring filename search scans the metadata table; the million-file target + requires an additional substring index and measured query planning. +- Initial indexing publishes after the root transaction finishes. +- File extraction currently covers bounded UTF-8 text, not PDF/DOCX/media. +- No automatic watcher, persistent extraction queue or ANN index yet. +- Native CI and signed package validation require actual platform runners. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000..c836870 --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,50 @@ +# Production readiness roadmap + +Status: implementation started; no production release yet. Windows VM tests use CPU only. + +## 1. Search foundation +- [x] Rust workspace, reusable engine, CLI, versioned SQLite storage. +- [ ] Filename/path and ranked full-text retrieval with strict filters. +- [ ] Safe folder discovery, exclusions, bounded extraction, passage locations. +- [ ] Incremental updates, rename/deletion reconciliation, crash recovery. +- [x] Persistence and filesystem lifecycle integration tests (Windows CPU). + +## 2. Document and semantic retrieval +- [ ] Text, Markdown, code, PDF and DOCX extraction with fixtures. +- [ ] Validate actual embedding model artifacts, license, CPU runtime and quantization. +- [ ] Versioned embeddings, ANN index, exact-match priority and hybrid fusion. +- [ ] English/German relevance corpus and measured top-10 recall. +- [ ] Atomic model/index replacement and resumable persistent work queues. + +## 3. Desktop +- [ ] Tauri shell, accessible keyboard search, progressive results and previews. +- [ ] Folder selection, exclusion controls, resource budgets and indexing status. +- [ ] Global shortcut, opening/revealing source files, PDF page navigation. +- [ ] Watchers and reconciliation across Windows, macOS and Linux. + +## 4. Media +- [ ] CPU OCR, image retrieval, audio transcription, video segment locations. +- [ ] Isolated extraction workers with timeouts and resource ceilings. + +## 5. Release gates +- [ ] Formatting, lint, tests and security/dependency review. +- [ ] CPU benchmarks: filename p95 <50 ms/1M records; text <150 ms/100K documents; semantic <500 ms on declared hardware. +- [ ] Measure cold start, indexing throughput, RAM, disk and idle CPU. +- [ ] Test migrations, interrupted work, invalid documents, inaccessible/offline roots and cloud placeholders. +- [ ] Automated native build matrix and documented supported cross-compilation paths. +- [ ] Execute Linux/macOS platform tests; create Windows/macOS/Linux installers. +- [ ] Signing/notarization configuration, upgrade/rollback validation, release instructions. + +Completion requires recorded evidence; targets are not measured results. + +## Completed foundation details + +- SQLite FTS5 with exact filename priority, text snippets and strict filters. +- Default directory exclusions and protection against indexing the index itself. +- Metadata reuse, atomic root reconciliation and schema-v1 migration coverage. +- Actual CLI process integration tests and independent-reader concurrency test. +- TypeScript desktop frontend build and dependency audit passed; native validation underway. +- Initial README commit pushed to Gitea main via protected Infisical injection. + +The partial features above do not close broader production milestones such as +crash-resumable jobs, fuzzy retrieval, watcher reliability or performance targets. diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md new file mode 100644 index 0000000..6c8ee7b --- /dev/null +++ b/docs/VALIDATION.md @@ -0,0 +1,18 @@ +# Validation evidence + +Development host: Windows VM, x86_64 MSVC, Rust stable 1.99.0. +All local tests run on CPU; no GPU provider is configured. + +Initial foundation checks (2026-10-12): +- `cargo test --workspace`: CLI parser/executable checks and engine retrieval, + lifecycle, exclusions, filters, migration and concurrent-reader checks passed. +- `cargo clippy --workspace --all-targets -- -D warnings`: passed. +- Rust formatting applied; final formatting verification follows integration. +- Desktop TypeScript/Vite build passed; npm audit reported zero vulnerabilities. + +Not yet established: production latency targets, semantic inference, desktop +runtime behavior, Linux/macOS execution, installer signing or upgrades. +CI configuration is source configuration, not evidence that remote jobs ran. +Gitea workflows require matching Windows, Linux and macOS runners; runner +availability has not been established. GitHub workflows are also supplied for +future mirroring, but do not run merely because code was pushed to Gitea.