searchi/apps/desktop/src/match-display.ts

/** Literal text evidence only. Semantic relevance never implies word importance. */
import { timestampLabel } from "./media-display.ts";

const fold = (value: string) =>
  value.normalize("NFD").replace(/\p{M}/gu, "").toLowerCase();
const stopwords = new Set(
  "a an and are as at be been but by can could did do does for from had has have how i if in into is it its me my of on or our should so than that the their them there these they this those to was we were what when where which who will with would you your am auch auf aus bei bin bis das dass dem den der des die dieser du ein eine einer einem einen eines er es für hat ich im ist mit nach nicht noch nur oder sein sind so sie sich und uns vom von vor war waren was wie wir wird zu zum zur"
    .split(" ")
    .map(fold),
);
export interface LiteralQuery {
  readonly query: string;
  readonly patterns: readonly string[];
}
export interface HighlightPart {
  text: string;
  matched: boolean;
}

export function literalQuery(value: string): LiteralQuery {
  const query = value.slice(0, 4096);
  const patterns = new Set<string>();
  const add = (text: string) => {
    const normalized = fold(text.trim());
    if (
      normalized &&
      !stopwords.has(normalized) &&
      /[\p{L}\p{N}]/u.test(normalized) &&
      patterns.size < 64
    )
      patterns.add(normalized);
  };
  const meaningful = (text: string) =>
    [...fold(text).matchAll(/[\p{L}\p{N}]+/gu)].some(
      (word) => !stopwords.has(word[0]),
    );
  // Fixed parsers only: user input is never compiled as a regular expression.
  for (const phrase of query.matchAll(/["“]([^"”]+)["”]/gu)) {
    if (meaningful(phrase[1])) add(phrase[1]);
  }
  const unquoted = query.replace(/["“”]/gu, "");
  const words = unquoted.split(/\s+/u);
  for (const raw of words) {
    const word = raw.replace(/^[,;:!?()[\]{}]+|[,;:!?()[\]{}.]+$/gu, "");
    add(word);
    if (patterns.size >= 64) break;
  }
  return Object.freeze({
    query,
    patterns: Object.freeze([...patterns].sort((a, b) => b.length - a.length)),
  });
}

export function highlightParts(
  text: string,
  query: LiteralQuery,
): HighlightPart[] {
  if (!text || !query.patterns.length) return [{ text, matched: false }];
  const offsets: { start: number; end: number }[] = [];
  let normalized = "";
  // Index bounded text only; untouched suffixes still preserve the complete source.
  for (const segment of text
    .slice(0, 65_536)
    .matchAll(/[^\p{M}]\p{M}*|\p{M}+/gu)) {
    const value = fold(segment[0]);
    normalized += value;
    for (let index = 0; index < value.length; index++)
      offsets.push({
        start: segment.index,
        end: segment.index + segment[0].length,
      });
  }
  const intervals: { start: number; end: number }[] = [];
  for (const pattern of query.patterns) {
    let from = 0;
    let matches = 0;
    while (matches < 256) {
      const start = normalized.indexOf(pattern, from);
      if (start < 0) break;
      const end = start + pattern.length;
      const identifier = /[_-]/u.test(pattern);
      const left = normalized.slice(Math.max(0, start - 2), start);
      const right = normalized.slice(end, end + 2);
      const joinedLeft = identifier
        ? /[\p{L}\p{N}_-]$/u.test(left)
        : /[\p{L}\p{N}]$/u.test(left);
      const joinedRight = identifier
        ? /^[\p{L}\p{N}_-]/u.test(right)
        : /^[\p{L}\p{N}]/u.test(right);
      if (!joinedLeft && !joinedRight && offsets[start] && offsets[end - 1]) {
        intervals.push({
          start: offsets[start].start,
          end: offsets[end - 1].end,
        });
        matches++;
      }
      from = end;
    }
  }
  intervals.sort((a, b) => a.start - b.start || b.end - a.end);
  const merged: { start: number; end: number }[] = [];
  for (const interval of intervals) {
    const previous = merged.at(-1);
    if (previous && interval.start < previous.end)
      previous.end = Math.max(previous.end, interval.end);
    else merged.push({ ...interval });
  }
  const parts: HighlightPart[] = [];
  let position = 0;
  for (const interval of merged.slice(0, 256)) {
    if (position < interval.start)
      parts.push({
        text: text.slice(position, interval.start),
        matched: false,
      });
    parts.push({
      text: text.slice(interval.start, interval.end),
      matched: true,
    });
    position = interval.end;
  }
  if (position < text.length)
    parts.push({ text: text.slice(position), matched: false });
  return parts.length ? parts : [{ text, matched: false }];
}

const rendered = new WeakMap<
  HTMLElement,
  { text: string; query: LiteralQuery }
>();
const excerpts = new WeakMap<LiteralQuery, Map<string, string>>();

export function displayExcerpt(
  text: string,
  query: LiteralQuery,
  length = 420,
) {
  if (text.length <= length) return text;
  let cache = excerpts.get(query);
  if (!cache) {
    cache = new Map();
    excerpts.set(query, cache);
  }
  const key = `${length}:${text}`;
  const stored = cache.get(key);
  if (stored !== undefined) return stored;
  let position = 0;
  let firstMatch = 0;
  for (const part of highlightParts(text, query)) {
    if (part.matched) {
      firstMatch = position;
      break;
    }
    position += part.text.length;
  }
  let start = Math.max(0, firstMatch - 100);
  if (start > 0 && /[\uDC00-\uDFFF]/u.test(text[start])) start--;
  let end = Math.min(text.length, start + length);
  if (end < text.length && /[\uD800-\uDBFF]/u.test(text[end - 1])) end--;
  const excerpt = `${start ? "… " : ""}${text.slice(start, end)}${end < text.length ? " …" : ""}`;
  if (cache.size >= 128) cache.delete(cache.keys().next().value!);
  cache.set(key, excerpt);
  return excerpt;
}

export function renderHighlighted(
  target: HTMLElement,
  text: string,
  query: LiteralQuery,
) {
  const previous = rendered.get(target);
  if (previous?.text === text && previous.query === query) return;
  const doc = target.ownerDocument;
  const fragment = doc.createDocumentFragment();
  for (const part of highlightParts(text, query)) {
    if (part.matched) {
      const mark = doc.createElement("mark");
      mark.dataset.match = "literal";
      mark.textContent = part.text;
      fragment.append(mark);
    } else fragment.append(doc.createTextNode(part.text));
  }
  target.replaceChildren(fragment);
  rendered.set(target, { text, query });
}

interface MatchResult {
  match_kind: string;
  source?: { kind: string; number: number };
}
export function matchExplanation(result: MatchResult) {
  const source = result.source;
  const location =
    source?.kind === "page"
      ? ` on page ${source.number}`
      : source?.kind === "paragraph"
        ? ` in paragraph ${source.number}`
        : source?.kind === "line"
          ? ` at line ${source.number}`
          : "";
  const time =
    source?.kind === "timestamp_ms"
      ? ` near ${timestampLabel(source.number)}`
      : "";
  const literalLegend =
    "Marked words are literal query matches, ignoring case and accents.";
  switch (result.match_kind) {
    case "semantic_image":
      return {
        reason:
          "Related image content. The match applies to the image as a whole.",
        legend: literalLegend,
        semanticPassage: false,
      };
    case "semantic_video":
      return {
        reason: `Related video frame${time}. The timestamp is the indexed frame position.`,
        legend: literalLegend,
        semanticPassage: false,
      };
    case "semantic_audio":
      return {
        reason: `Related audio${time}. The preview starts at an indexed segment, not an exact spoken phrase.`,
        legend: literalLegend,
        semanticPassage: false,
      };
    case "hybrid":
      return {
        reason: `Query words and related meaning matched this file; showing the passage${location}.`,
        legend: `${literalLegend} Meaning can match without the same words.`,
        semanticPassage: false,
      };
    case "semantic":
      return {
        reason: `Related passage${location}. Similar meaning was found in this passage or its file title.`,
        legend: `${literalLegend} Meaning can match without the same words.`,
        semanticPassage: true,
      };
    case "exact_filename":
      return {
        reason: "The filename matches your query.",
        legend: literalLegend,
        semanticPassage: false,
      };
    case "filename":
      return {
        reason: "Your query appears in the filename.",
        legend: literalLegend,
        semanticPassage: false,
      };
    case "path":
      return {
        reason: "Your query appears in the file path.",
        legend: literalLegend,
        semanticPassage: false,
      };
    case "name_terms":
      return {
        reason: "Query words matched indexed filename, path or content terms.",
        legend: literalLegend,
        semanticPassage: false,
      };
    case "media_metadata":
      return {
        reason: "Query words matched artist, title or other media metadata.",
        legend: literalLegend,
        semanticPassage: false,
      };
    case "exact_identifier":
      return {
        reason: "An exact identifier matched extracted content.",
        legend: literalLegend,
        semanticPassage: false,
      };
    case "content":
    case "keyword":
      return {
        reason: `Query words matched extracted content${location}.`,
        legend: literalLegend,
        semanticPassage: false,
      };
    default:
      return {
        reason: "This result was returned by local search.",
        legend: literalLegend,
        semanticPassage: false,
      };
  }
}