Distinguish main matches from related suggestions and ship 0.1.9

crates/searchi-service/src/lib.rs

@@ -12,6 +12,7 @@ mod prepared;
 pub use prepared::PreparedGraphs;
 mod pipeline;
 mod relevance;
+pub use relevance::is_related_result;
 mod routing;
 pub use routing::SemanticOrigin;
 mod store;
@@ -595,8 +596,9 @@ impl HybridSearch {
             self.routed_semantic_traced(query, &vector, candidates, options, trace, parent)?;
         trace.timed("hybrid.semantic_shortlist", parent, |trace, span| {
             trace.counter(span, "candidates_before", semantic.len() as u64);
-            let removed = relevance::prune_semantic_tail(&mut semantic);
-            trace.counter(span, "tail_candidates_pruned", removed as u64);
+            let related = relevance::classify_semantic_evidence(&mut semantic);
+            trace.counter(span, "related_candidates", related as u64);
+            trace.counter(span, "main_candidates", (semantic.len() - related) as u64);
             trace.counter(span, "candidates_after", semantic.len() as u64);
         });
         let results = trace.timed("hybrid.rank_fusion", parent, |trace, span| {
@@ -805,7 +807,7 @@ fn fuse(
         let contribution = 1.0 / (61.0 + rank as f64);
         if let Some(existing) = results.get_mut(&result.path) {
             existing.score += contribution;
-            if existing.match_kind == "content" {
+            if existing.match_kind == "content" && !is_related_result(&result) {
                 existing.match_kind = "hybrid".into();
             }
         } else {
@@ -816,6 +818,7 @@ fn fuse(
     let tier = |result: &SearchResult| match result.match_kind.as_str() {
         "exact_filename" => 0,
         "exact_identifier" => 1,
+        _ if is_related_result(result) => 3,
         _ => 2,
     };
     let mut results: Vec<_> = results.into_values().collect();

crates/searchi-service/src/relevance.rs

@@ -1,4 +1,4 @@
-//! Hybrid result shortlisting, not a confidence or no-answer classifier.
+//! Hybrid evidence presentation, not confidence or proof that content exists.
 use searchi_core::SearchResult;
 
 // Rounded upward from the largest adjacent gap (0.0650497) observed in the
@@ -7,15 +7,37 @@ use searchi_core::SearchResult;
 // accessible. Raw semantic diagnostics are deliberately not changed.
 const MIN_CLEAR_GAP: f64 = 0.07;
 const NUMERIC_TIE: f64 = 2.0 * f32::EPSILON as f64;
+const MAIN_COSINE_MINIMUM: f64 = 0.70;
+
+/// Related suggestions remain accessible but must follow primary search results.
+pub fn is_related_result(result: &SearchResult) -> bool {
+    result.match_kind.starts_with("semantic") && result.match_kind.ends_with("_related")
+}
+
+/// Label weak and trailing neighbors without deleting eligible sources.
+/// The frozen display policy is deliberately distinct from raw semantic retrieval.
+pub(super) fn classify_semantic_evidence(results: &mut [SearchResult]) -> usize {
+    let boundary = leading_cluster_len(results);
+    let mut related = 0;
+    for (index, result) in results.iter_mut().enumerate() {
+        if index >= boundary || !result.score.is_finite() || result.score < MAIN_COSINE_MINIMUM {
+            if !is_related_result(result) {
+                result.match_kind.push_str("_related");
+            }
+            related += 1;
+        }
+    }
+    related
+}
 
 /// Keep the strongest semantic cluster only when one clear score gap separates
 /// it from the tail. Inputs are already source-checked, filtered and file-deduped.
-pub(super) fn prune_semantic_tail(results: &mut Vec<SearchResult>) -> usize {
+fn leading_cluster_len(results: &[SearchResult]) -> usize {
     if results.len() < 2
         || results.iter().any(|result| !result.score.is_finite())
         || results.windows(2).any(|pair| pair[0].score < pair[1].score)
     {
-        return 0;
+        return results.len();
     }
     let mut largest = 0.0;
     let mut second = 0.0;
@@ -33,11 +55,9 @@ pub(super) fn prune_semantic_tail(results: &mut Vec<SearchResult>) -> usize {
     // Equal large gaps indicate no unique cluster boundary, e.g. a uniform
     // staircase. Do not pick an arbitrary split due to f32 rounding.
     if largest <= MIN_CLEAR_GAP || largest - second <= NUMERIC_TIE {
-        return 0;
+        return results.len();
     }
-    let removed = results.len() - boundary;
-    results.truncate(boundary);
-    removed
+    boundary
 }
 
 #[cfg(test)]
@@ -61,11 +81,14 @@ mod tests {
     #[test]
     fn weak_correct_best_and_multiple_strong_candidates_survive() {
         let mut weak = candidates(&[0.6766, 0.50, 0.49]);
-        assert_eq!(prune_semantic_tail(&mut weak), 2);
+        assert_eq!(classify_semantic_evidence(&mut weak), 3);
         assert_eq!(weak[0].score, 0.6766);
+        assert!(weak.iter().all(is_related_result));
         let mut multiple = candidates(&[0.82, 0.79, 0.57, 0.55]);
-        assert_eq!(prune_semantic_tail(&mut multiple), 2);
-        assert_eq!(multiple.len(), 2);
+        assert_eq!(classify_semantic_evidence(&mut multiple), 2);
+        assert_eq!(multiple.len(), 4);
+        assert!(!is_related_result(&multiple[0]));
+        assert!(is_related_result(&multiple[2]));
     }
     #[test]
     fn small_flat_and_ambiguous_lists_are_preserved() {
@@ -78,13 +101,13 @@ mod tests {
             &[0.61, 0.61001, 0.60],
             &[0.60, 0.599999, 0.599998],
         ] {
-            let mut results = candidates(scores);
-            assert_eq!(prune_semantic_tail(&mut results), 0, "{scores:?}");
+            let results = candidates(scores);
+            assert_eq!(leading_cluster_len(&results), scores.len(), "{scores:?}");
             assert_eq!(results.len(), scores.len());
         }
         let mut two = candidates(&[0.82, 0.52]);
-        assert_eq!(prune_semantic_tail(&mut two), 1);
-        assert_eq!(two.len(), 1);
+        assert_eq!(classify_semantic_evidence(&mut two), 1);
+        assert_eq!(two.len(), 2);
     }
     #[test]
     fn independent_exact_lexical_match_survives_semantic_tail_pruning() {
@@ -92,10 +115,51 @@ mod tests {
         let mut exact = candidates(&[0.52]).remove(0);
         exact.path = "candidate-1".into();
         exact.match_kind = "exact_filename".into();
-        assert_eq!(prune_semantic_tail(&mut semantic), 1);
+        assert_eq!(classify_semantic_evidence(&mut semantic), 1);
         let results = crate::fuse(vec![exact], semantic, 10);
         assert_eq!(results.len(), 2);
         assert_eq!(results[0].path, "candidate-1");
         assert_eq!(results[0].match_kind, "exact_filename");
     }
+    #[test]
+    fn floor_is_inclusive_and_related_classification_is_idempotent() {
+        let mut results = candidates(&[0.70, 0.699999]);
+        results[1].match_kind = "semantic_image".into();
+        assert_eq!(classify_semantic_evidence(&mut results), 1);
+        assert!(!is_related_result(&results[0]));
+        assert_eq!(results[1].match_kind, "semantic_image_related");
+        assert_eq!(classify_semantic_evidence(&mut results), 1);
+        assert_eq!(results[1].match_kind, "semantic_image_related");
+        assert_eq!(classify_semantic_evidence(&mut []), 0);
+    }
+    #[test]
+    fn strong_tail_stays_related_and_never_displaces_main_under_limit() {
+        let mut semantic = candidates(&[0.95, 0.75, 0.74]);
+        assert_eq!(classify_semantic_evidence(&mut semantic), 2);
+        let mut limited = candidates(&[0.95, 0.75, 0.74]);
+        classify_semantic_evidence(&mut limited);
+        let results = crate::fuse(vec![], limited, 1);
+        assert_eq!(results.len(), 1);
+        assert!(!is_related_result(&results[0]));
+        let all = crate::fuse(vec![], semantic, 10);
+        assert_eq!(all.len(), 3);
+        assert!(all[1..].iter().all(is_related_result));
+        assert!(crate::fuse(vec![], candidates(&[0.8]), 0).is_empty());
+    }
+    #[test]
+    fn independent_content_evidence_overrides_related_without_duplicate() {
+        let mut semantic = candidates(&[0.65, 0.64]);
+        classify_semantic_evidence(&mut semantic);
+        let mut lexical = candidates(&[0.65, 0.64]).remove(1);
+        lexical.match_kind = "content".into();
+        let results = crate::fuse(vec![lexical], semantic, 10);
+        assert_eq!(results.len(), 2);
+        assert_eq!(results[0].path, "candidate-1");
+        assert_eq!(results[0].match_kind, "content");
+        assert!(is_related_result(&results[1]));
+        let mut tied = candidates(&[0.8, 0.8]);
+        assert_eq!(classify_semantic_evidence(&mut tied), 0);
+        let results = crate::fuse(vec![], tied, 2);
+        assert_eq!(results[0].path, "candidate-0");
+    }
 }