feat: add Google code retrieval across desktop CLI and MCP

crates/searchi-semantic/native/worker.cpp

@@ -197,11 +197,50 @@ public:
         require(!text.empty() && text.size() <= max_passage_bytes, "Passage exceeds 1 MiB or is empty");
         auto tokens = tokenize(text,false);
         require(!tokens.empty(), "Passage has no tokens");
-        // Preserve actual source offsets, rather than inventing locations from
-        // a lossy decoded representation. Unexpected normalization is an error.
-        require(decode(tokens,tokens.size()) == text, "Tokenizer roundtrip changed source text; cannot verify source offsets");
         size_t prompt = tokenize(document_prefix(title),true).size();
         require(prompt + 132 < size_t(context_tokens), "Document prefix exceeds token budget");
+        // SentencePiece may normalize source characters. In that case decoded
+        // token lengths are not source offsets: split the original UTF-8 bytes
+        // and verify each complete prefixed fragment against the real tokenizer.
+        if (decode(tokens,tokens.size()) != text) {
+            const auto prefix = document_prefix(title);
+            json result = json::array();
+            if (prefix.size()+text.size() <= max_text_bytes && tokenize(prefix+text,true).size() <= size_t(context_tokens)) {
+                return json::array({{{"text",text},{"start_byte",0},{"end_byte",text.size()}}});
+            }
+            std::vector<size_t> boundaries;
+            for (size_t i=0;i<text.size();++i) if ((uint8_t(text[i]) & 0xc0) != 0x80) boundaries.push_back(i);
+            boundaries.push_back(text.size());
+            size_t start_index = 0;
+            while (start_index+1 < boundaries.size()) {
+                const size_t start = boundaries[start_index];
+                require(result.size() < max_windows, "Passage exceeds 256 windows");
+                size_t low = start_index;
+                size_t high = size_t(std::upper_bound(boundaries.begin(),boundaries.end(),std::min(text.size(),start+max_text_bytes-prefix.size()))-boundaries.begin())-1;
+                size_t best_index = start_index;
+                while (low < high) {
+                    size_t candidate_index = low + (high-low+1)/2;
+                    size_t candidate = boundaries[candidate_index];
+                    if (tokenize(prefix+text.substr(start,candidate-start),true).size() <= size_t(context_tokens)-4) {
+                        best_index = candidate_index;
+                        low = candidate_index;
+                    } else {
+                        high = candidate_index-1;
+                    }
+                }
+                size_t best = boundaries[best_index];
+                require(best > start, "Cannot fit source fragment in model context");
+                auto fragment = text.substr(start,best-start);
+                require(tokenize(prefix+fragment,true).size() <= size_t(context_tokens), "Source fragment exceeds context budget");
+                result.push_back({{"text",fragment},{"start_byte",start},{"end_byte",best}});
+                if (best == text.size()) break;
+                // Bounded original-byte overlap; always advance, even for
+                // unusual Unicode/token expansion. No decoded text is indexed.
+                size_t next = best - std::min(size_t(128),(best-start)/2);
+                start_index = size_t(std::lower_bound(boundaries.begin(),boundaries.end(),next)-boundaries.begin());
+            }
+            return result;
+        }
         size_t budget = size_t(context_tokens) - prompt - 4;
         size_t stride = budget - 128;
         json result = json::array();
@@ -253,10 +292,11 @@ public:
             return {{"vectors",json::array({embed_bitmap(bitmap.get())})}};
         }
         if (op == "windows") return {{"windows",windows(request.at("text").get<std::string>(),request.value("title",std::string()))}};
-        if (op == "query") {
+        if (op == "query" || op == "code_query") {
             bool debug = request.value("debug_performance",false);
             json timings;
-            auto vector = embed("task: search result | query: "+request.at("text").get<std::string>(),512,debug ? &timings : nullptr);
+            auto prefix = op == "code_query" ? "task: code retrieval | query: " : "task: search result | query: ";
+            auto vector = embed(prefix+request.at("text").get<std::string>(),512,debug ? &timings : nullptr);
             json response = {{"vectors",json::array({vector})}};
             if (debug) response["performance"] = timings;
             return response;
@@ -289,7 +329,7 @@ int main(int argc, char ** argv) {
         bool force_cpu = std::getenv("SEARCHI_FORCE_CPU") && std::string(std::getenv("SEARCHI_FORCE_CPU")) == "1";
         Encoder encoder(path,threads,device == "auto" && !force_cpu);
         std::cout << json({{"ready",true},{"model","google/embeddinggemma-2"},{"dimensions",dimensions},
-            {"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true}}).dump() << '\n' << std::flush;
+            {"device",encoder.device},{"threads",threads},{"runtime",SEARCHI_LLAMA_TAG},{"media_supported",true},{"code_query_supported",true}}).dump() << '\n' << std::flush;
         for (;;) {
             std::string line; char c;
             while (std::cin.get(c) && c != '\n') {