feat: highlight search matches and document the working UI

tools/desktop-capture/capture.mjs

@@ -0,0 +1,398 @@
+/** Capture the production frontend against a live, real local Rust/Gemma fixture. */
+import { mkdir, writeFile } from "node:fs/promises";
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { execFileSync } from "node:child_process";
+process.env.PLAYWRIGHT_BROWSERS_PATH = "0";
+const { chromium } = await import("playwright");
+const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
+const target = new URL(
+  process.env.SEARCHI_CAPTURE_URL ?? "http://127.0.0.1:1431/?capture=1",
+);
+assert(
+  target.protocol === "http:" &&
+    ["127.0.0.1", "localhost"].includes(target.hostname),
+  "Capture must use an owned loopback server",
+);
+assert(
+  target.searchParams.get("capture") === "1",
+  "The explicit capture adapter must be enabled",
+);
+const out = resolve(
+  repository,
+  process.env.SEARCHI_CAPTURE_OUTPUT ?? "docs/screenshots",
+);
+const infoResponse = await fetch(new URL("/capture-info", target), {
+  signal: AbortSignal.timeout(15000),
+});
+assert(infoResponse.ok, "Real fixture backend is not ready");
+const info = await infoResponse.json();
+assert(
+  info.mode === "readme-capture" && info.ready === true,
+  "Requires an initialized real-engine capture backend",
+);
+await mkdir(out, { recursive: true });
+const browser = await chromium.launch({
+  headless: true,
+  args: [
+    "--disable-gpu",
+    "--disable-accelerated-video-decode",
+    "--disable-webgl",
+  ],
+});
+const failures = [];
+const evidence = [];
+const measured = [];
+let latestMeasured;
+const source = {
+  base_commit: execFileSync("git", ["rev-parse", "HEAD"], {
+    cwd: repository,
+    encoding: "utf8",
+  }).trim(),
+  working_tree_dirty:
+    execFileSync("git", ["status", "--porcelain"], {
+      cwd: repository,
+      encoding: "utf8",
+    }).trim().length > 0,
+};
+try {
+  const context = await browser.newContext({
+    viewport: { width: 1440, height: 1000 },
+    deviceScaleFactor: 1,
+    colorScheme: "light",
+    reducedMotion: "reduce",
+  });
+  await context.route("**/*", (route) => {
+    const url = new URL(route.request().url());
+    return url.origin === target.origin ||
+      ["data:", "blob:"].includes(url.protocol)
+      ? route.continue()
+      : route.abort();
+  });
+  const page = await context.newPage();
+  page.on("pageerror", (error) => failures.push(String(error)));
+  await page.goto(target.href, { waitUntil: "networkidle", timeout: 30000 });
+  await page.waitForFunction(
+    () =>
+      document
+        .querySelector("#semantic-status")
+        ?.textContent?.includes("Semantic search ready"),
+    {},
+    { timeout: 120000 },
+  );
+  await page.evaluate(async () => {
+    await document.fonts.ready;
+  });
+  assert(
+    await page
+      .locator(".brand-symbol")
+      .evaluate((image) => image.complete && image.naturalWidth > 0),
+    "Brand asset failed to load",
+  );
+  async function query(text, debug = false) {
+    const awaited = page.waitForResponse(
+      (response) => {
+        if (
+          new URL(response.url()).pathname !==
+          "/capture-api/hybrid_search_measured"
+        )
+          return false;
+        const args = response.request().postDataJSON();
+        return args?.query === text && Boolean(args.debugPerformance) === debug;
+      },
+      { timeout: 60000 },
+    );
+    await page.locator("#query").fill(text);
+    const response = await awaited;
+    assert(response.ok(), "Real measured engine query failed");
+    const body = await response.json();
+    assert(
+      Array.isArray(body.results) && body.metrics,
+      "Measured backend response missing actual results/metrics",
+    );
+    latestMeasured = {
+      query: text,
+      mode: debug ? "debug" : "basic",
+      result_count: body.results.length,
+      metrics: body.metrics,
+      first_filename: body.results[0]?.filename,
+      first_source: body.results[0]?.source,
+    };
+    measured.push(latestMeasured);
+    await page.waitForFunction(
+      (expected) => {
+        const summary =
+          document.querySelector("#result-summary")?.textContent ?? "";
+        return (
+          document.querySelector("#query")?.value === expected.query &&
+          summary.includes("KEYWORD + MEANING") &&
+          document.querySelector("#results .result strong")?.textContent ===
+            expected.filename &&
+          document.querySelectorAll("#results .result").length ===
+            expected.count
+        );
+      },
+      {
+        query: text,
+        filename: latestMeasured.first_filename,
+        count: latestMeasured.result_count,
+      },
+      { timeout: 60000 },
+    );
+    assert(
+      (await page.locator("#results .result").count()) > 0,
+      `Real query returned no matches: ${text}`,
+    );
+    await page.locator("#results .result").first().click();
+    await page.waitForFunction(
+      () => !document.querySelector("#preview")?.hidden,
+    );
+    if (text === "cancel the supplier partnership") {
+      assert.equal(
+        latestMeasured.first_filename,
+        "Supplier Agreement.pdf",
+        "Contract query must retrieve the known PDF fixture",
+      );
+      assert.equal(
+        latestMeasured.first_source?.kind,
+        "page",
+        "PDF match must retain actual page source",
+      );
+      assert.equal(
+        latestMeasured.first_source?.number,
+        2,
+        "Termination clause must be on actual page2",
+      );
+      const passage = await page.locator("#preview-text").innerText();
+      assert.match(
+        passage,
+        /sixty|\b60\b/i,
+        "Actual PDF preview is missing notice period",
+      );
+      assert.match(
+        passage,
+        /terminat/i,
+        "Actual PDF preview is missing termination section",
+      );
+      const literalWords = ["supplier", "partnership"];
+      const rowMarks = await page
+        .locator("#results .result")
+        .first()
+        .locator('.snippet mark[data-match="literal"]')
+        .allTextContents();
+      const passageMarks = await page
+        .locator('#preview-text mark[data-match="literal"]')
+        .allTextContents();
+      for (const word of literalWords) {
+        assert(
+          rowMarks.some((mark) => mark.toLowerCase() === word),
+          `Result snippet must highlight actual query word ${word}`,
+        );
+        assert(
+          passageMarks.some((mark) => mark.toLowerCase() === word),
+          `PDF passage must highlight actual query word ${word}`,
+        );
+      }
+      const why = page.locator("#match-reason");
+      const disclosure = why.locator("xpath=ancestor::details[1]");
+      assert.equal(
+        await disclosure.count(),
+        1,
+        "Match reason must live in a compact disclosure",
+      );
+      assert.equal(
+        await disclosure.evaluate((element) => element.open),
+        false,
+        "Why details should stay collapsed by default",
+      );
+      await disclosure.locator(":scope > summary").click();
+      await why.waitFor({ state: "visible" });
+      assert.match(
+        await page.locator("#match-reason").innerText(),
+        /page\s*2/i,
+        "Match explanation must use actual PDF page2",
+      );
+      await disclosure.locator(":scope > summary").click();
+      assert.equal(
+        await disclosure.evaluate((element) => element.open),
+        false,
+        "Normal screenshot must preserve collapsed why details",
+      );
+      latestMeasured.ui_checks = {
+        literal_query_marks: true,
+        actual_page_explanation: true,
+        why_disclosure_collapsed_by_default: true,
+      };
+    }
+    if (text === "mountains reflected in calm water") {
+      assert.equal(
+        latestMeasured.first_filename,
+        "Mountain Lake.jpg",
+        "Visual query must retrieve the known licensed lake",
+      );
+      assert.equal(
+        await page.locator("#preview-text mark").count(),
+        0,
+        "Meaning-only photo result must not invent word-level alignment",
+      );
+      latestMeasured.ui_checks = { meaning_only_image_has_no_text_marks: true };
+    }
+    await page.evaluate(
+      () =>
+        new Promise((resolve) =>
+          requestAnimationFrame(() => requestAnimationFrame(resolve)),
+        ),
+    );
+  }
+  async function screenshot(name, queryText) {
+    assert.deepEqual(failures, [], "Frontend page errors occurred");
+    const path = resolve(out, `${name}.png`);
+    await page.screenshot({ path, animations: "disabled", fullPage: false });
+    const selected = await page.locator("#preview-title").innerText();
+    assert.equal(
+      selected,
+      latestMeasured.first_filename,
+      "Selected DOM result is not the actual first hybrid result",
+    );
+    evidence.push({
+      file: `${name}.png`,
+      sha256: createHash("sha256")
+        .update(await readFile(path))
+        .digest("hex"),
+      query: queryText,
+      mode: latestMeasured.mode,
+      result_count: latestMeasured.result_count,
+      selected_filename: selected,
+      first_source_kind: latestMeasured.first_source?.kind,
+      first_source_number: latestMeasured.first_source?.number,
+      theme: await page.evaluate(() =>
+        matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light",
+      ),
+      summary: await page.locator("#result-summary").innerText(),
+      state_assertions: {
+        semantic_ready: true,
+        hybrid_final: true,
+        image_visible: name === "media-search",
+        actual_worker_inference:
+          latestMeasured.mode === "debug" &&
+          latestMeasured.metrics.trace.some(
+            (span) => span.counters.worker_inference_us > 0,
+          ),
+        ...latestMeasured.ui_checks,
+      },
+    });
+  }
+  const documentQuery = "cancel the supplier partnership";
+  await query(documentQuery);
+  await screenshot("search-light", documentQuery);
+  await page.emulateMedia({ colorScheme: "dark" });
+  await screenshot("search-dark", documentQuery);
+  await page.emulateMedia({ colorScheme: "light" });
+  const imageQuery = "mountains reflected in calm water";
+  await query(imageQuery);
+  await page.waitForFunction(
+    () => {
+      const image = document.querySelector("#media-preview img");
+      return (
+        image instanceof HTMLImageElement &&
+        image.complete &&
+        image.naturalWidth > 0
+      );
+    },
+    {},
+    { timeout: 60000 },
+  );
+  await screenshot("media-search", imageQuery);
+  await page.locator("#settings summary").click();
+  await page.locator("#performance-toggle").check();
+  await page.locator("#settings summary").click();
+  await query(documentQuery, true);
+  await page.locator("#performance-panel").waitFor({ state: "visible" });
+  await page.locator("#performance-panel summary").click();
+  await page.waitForFunction(
+    () =>
+      document
+        .querySelector("#performance-detail")
+        ?.textContent?.includes("hybrid") ||
+      document
+        .querySelector("#performance-detail")
+        ?.textContent?.includes("Hybrid"),
+  );
+  assert(
+    latestMeasured.metrics.trace?.some(
+      (span) => span.counters.worker_inference_us > 0,
+    ),
+    "Detailed trace did not record real worker inference",
+  );
+  await page
+    .locator("#performance-detail .performance-span")
+    .filter({ hasText: "worker inference" })
+    .first()
+    .scrollIntoViewIfNeeded();
+  await screenshot("performance-details", documentQuery);
+  const semanticResponse = await context.request.post(
+    new URL("/capture-api/semantic_status", target).href,
+    {
+      headers: { "X-Searchi-Capture-Token": info.token, Origin: target.origin },
+      data: {},
+    },
+  );
+  assert(semanticResponse.ok(), "Actual semantic status unavailable");
+  const semantic = await semanticResponse.json();
+  assert.equal(
+    semantic.worker_info?.model,
+    "google/embeddinggemma-2",
+    "Capture requires the actual Google model",
+  );
+  assert.equal(
+    semantic.worker_info?.threads,
+    1,
+    "Capture inference requires one CPU thread",
+  );
+  assert.equal(
+    semantic.worker_info?.device,
+    "cpu",
+    "Capture worker is not CPU",
+  );
+  await writeFile(
+    resolve(out, "capture-manifest.json"),
+    JSON.stringify(
+      {
+        kind: "browser-rendered-production-ui-real-cpu-engine",
+        source,
+        viewport: { width: 1440, height: 1000 },
+        runner: { playwright: "1.64.0", chromium: browser.version() },
+        backend: {
+          mode: info.mode,
+          model: semantic.worker_info.model,
+          device: semantic.worker_info.device,
+          runtime: semantic.worker_info.runtime,
+          threads: semantic.worker_info.threads,
+          dimensions: semantic.worker_info.dimensions,
+          ready: info.ready,
+          rust_build_profile: info.rust_build_profile,
+        },
+        cpu_flags: [
+          "--disable-gpu",
+          "--disable-accelerated-video-decode",
+          "--disable-webgl",
+        ],
+        native_window_chrome: false,
+        native_visual_qa_complete: false,
+        measured_queries: measured,
+        screenshots: evidence,
+      },
+      null,
+      2,
+    ) + "\n",
+  );
+  console.log(
+    `Captured ${evidence.length} real-engine UI screenshots in ${out}`,
+  );
+  await context.close();
+} finally {
+  await browser.close();
+}