497 lines
19 KiB
JavaScript
497 lines
19 KiB
JavaScript
/** Capture the production frontend against a live, real local Rust/Gemma fixture. */
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import { resolve, dirname } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import assert from "node:assert/strict";
|
|
import { createHash } from "node:crypto";
|
|
import { readFile } from "node:fs/promises";
|
|
import { execFileSync } from "node:child_process";
|
|
process.env.PLAYWRIGHT_BROWSERS_PATH = "0";
|
|
const { chromium } = await import("playwright");
|
|
const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const target = new URL(
|
|
process.env.SEARCHI_CAPTURE_URL ?? "http://127.0.0.1:1431/?capture=1",
|
|
);
|
|
assert(
|
|
target.protocol === "http:" &&
|
|
["127.0.0.1", "localhost"].includes(target.hostname),
|
|
"Capture must use an owned loopback server",
|
|
);
|
|
assert(
|
|
target.searchParams.get("capture") === "1",
|
|
"The explicit capture adapter must be enabled",
|
|
);
|
|
const out = resolve(
|
|
repository,
|
|
process.env.SEARCHI_CAPTURE_OUTPUT ?? "docs/screenshots",
|
|
);
|
|
const frontendVersion = JSON.parse(await readFile(
|
|
resolve(repository, "apps/desktop/package.json"), "utf8")).version;
|
|
if (process.env.SEARCHI_CAPTURE_EXPECTED_VERSION) {
|
|
assert.equal(frontendVersion, process.env.SEARCHI_CAPTURE_EXPECTED_VERSION,
|
|
"Frontend metadata must match the version intended for this capture");
|
|
}
|
|
const infoResponse = await fetch(new URL("/capture-info", target), {
|
|
signal: AbortSignal.timeout(15000),
|
|
});
|
|
assert(infoResponse.ok, "Real fixture backend is not ready");
|
|
const info = await infoResponse.json();
|
|
assert(
|
|
info.mode === "readme-capture" && info.ready === true,
|
|
"Requires an initialized real-engine capture backend",
|
|
);
|
|
await mkdir(out, { recursive: true });
|
|
assert(process.env.SEARCHI_CAPTURE_BACKEND_EXE,
|
|
"Set SEARCHI_CAPTURE_BACKEND_EXE to the actual owned backend executable for provenance");
|
|
const captureBackendSha256 = createHash("sha256")
|
|
.update(await readFile(process.env.SEARCHI_CAPTURE_BACKEND_EXE)).digest("hex");
|
|
const browser = await chromium.launch({
|
|
headless: true,
|
|
args: [
|
|
"--disable-gpu",
|
|
"--disable-accelerated-video-decode",
|
|
"--disable-webgl",
|
|
],
|
|
});
|
|
const failures = [];
|
|
const evidence = [];
|
|
const measured = [];
|
|
let latestMeasured;
|
|
const isRelated = (result) => result.match_kind?.endsWith("_related") === true;
|
|
const source = {
|
|
base_commit: execFileSync("git", ["rev-parse", "HEAD"], {
|
|
cwd: repository,
|
|
encoding: "utf8",
|
|
}).trim(),
|
|
working_tree_dirty:
|
|
execFileSync("git", ["status", "--porcelain"], {
|
|
cwd: repository,
|
|
encoding: "utf8",
|
|
}).trim().length > 0,
|
|
};
|
|
source.source_sha256 = {};
|
|
for (const name of [
|
|
"crates/searchi-service/src/lib.rs", "crates/searchi-service/src/relevance.rs",
|
|
"apps/desktop/src/main.ts", "apps/desktop/src/style.css",
|
|
"tools/desktop-capture/backend/src/main.rs", "tools/desktop-capture/capture.mjs",
|
|
"tools/relevance-eval/policies/evidence-v1.json",
|
|
]) {
|
|
source.source_sha256[name] = createHash("sha256")
|
|
.update(await readFile(resolve(repository, name))).digest("hex");
|
|
}
|
|
source.capture_backend_sha256 = captureBackendSha256;
|
|
try {
|
|
const context = await browser.newContext({
|
|
viewport: { width: 1440, height: 1000 },
|
|
deviceScaleFactor: 1,
|
|
colorScheme: "light",
|
|
reducedMotion: "reduce",
|
|
});
|
|
await context.route("**/*", (route) => {
|
|
const url = new URL(route.request().url());
|
|
return url.origin === target.origin ||
|
|
["data:", "blob:"].includes(url.protocol)
|
|
? route.continue()
|
|
: route.abort();
|
|
});
|
|
const page = await context.newPage();
|
|
page.on("pageerror", (error) => failures.push(String(error)));
|
|
await page.goto(target.href, { waitUntil: "networkidle", timeout: 30000 });
|
|
await page.waitForFunction(
|
|
() =>
|
|
document
|
|
.querySelector("#semantic-status")
|
|
?.textContent?.includes("Semantic search ready"),
|
|
{},
|
|
{ timeout: 120000 },
|
|
);
|
|
await page.evaluate(async () => {
|
|
await document.fonts.ready;
|
|
});
|
|
assert(
|
|
await page
|
|
.locator(".brand-symbol")
|
|
.evaluate((image) => image.complete && image.naturalWidth > 0),
|
|
"Brand asset failed to load",
|
|
);
|
|
// Capture the actual ready/empty production layout before typing. This is
|
|
// browser-rendered UI, not native window chrome or synthesized app results.
|
|
await page.waitForFunction(() =>
|
|
document.documentElement.dataset.launcherLayout === "compact" &&
|
|
document.querySelector("#query")?.value === "",
|
|
);
|
|
const compactPath = resolve(out, "launcher-empty.png");
|
|
const compactBounds = await page.locator("#app").boundingBox();
|
|
assert(compactBounds && compactBounds.width <= 720 && compactBounds.height <= 112,
|
|
"Ready empty launcher must remain a compact bar");
|
|
await page.locator("#app").screenshot({ path: compactPath, animations: "disabled" });
|
|
evidence.push({
|
|
file: "launcher-empty.png",
|
|
sha256: createHash("sha256").update(await readFile(compactPath)).digest("hex"),
|
|
query: "",
|
|
mode: "empty-ready",
|
|
result_count: 0,
|
|
selected_filename: null,
|
|
theme: "light",
|
|
capture_bounds: compactBounds,
|
|
state_assertions: { semantic_ready: true, compact_empty_layout: true, native_pixels: false },
|
|
});
|
|
async function query(text, debug = false) {
|
|
const awaited = page.waitForResponse(
|
|
(response) => {
|
|
if (
|
|
new URL(response.url()).pathname !==
|
|
"/capture-api/hybrid_search_measured"
|
|
)
|
|
return false;
|
|
const args = response.request().postDataJSON();
|
|
return args?.query === text && Boolean(args.debugPerformance) === debug;
|
|
},
|
|
{ timeout: 60000 },
|
|
);
|
|
await page.locator("#query").fill(text);
|
|
const response = await awaited;
|
|
assert(response.ok(), "Real measured engine query failed");
|
|
const body = await response.json();
|
|
assert(
|
|
Array.isArray(body.results) && body.metrics,
|
|
"Measured backend response missing actual results/metrics",
|
|
);
|
|
latestMeasured = {
|
|
query: text,
|
|
mode: debug ? "debug" : "basic",
|
|
result_count: body.results.length,
|
|
main_result_count: body.results.filter((result) => !isRelated(result)).length,
|
|
related_result_count: body.results.filter(isRelated).length,
|
|
metrics: body.metrics,
|
|
first_filename: body.results[0]?.filename,
|
|
first_source: body.results[0]?.source,
|
|
};
|
|
measured.push(latestMeasured);
|
|
await page.waitForFunction(
|
|
(expected) => {
|
|
const summary =
|
|
document.querySelector("#result-summary")?.textContent ?? "";
|
|
return (
|
|
document.querySelector("#query")?.value === expected.query &&
|
|
summary.includes("KEYWORD + MEANING") &&
|
|
document.querySelector("#results .result strong")?.textContent ===
|
|
expected.filename &&
|
|
document.querySelectorAll("#results .result").length ===
|
|
expected.count
|
|
);
|
|
},
|
|
{
|
|
query: text,
|
|
filename: latestMeasured.first_filename,
|
|
count: latestMeasured.main_result_count,
|
|
},
|
|
{ timeout: 60000 },
|
|
);
|
|
assert(
|
|
(await page.locator("#results .result").count()) > 0,
|
|
`Real query returned no matches: ${text}`,
|
|
);
|
|
assert.equal(await page.locator("#related-results .result").count(), 0,
|
|
"Collapsed related neighbors must stay out of keyboard/focusable result rows");
|
|
assert.equal(await page.locator("#related-suggestions").evaluate((element) => element.open),
|
|
false, "Related suggestions must be collapsed for a new query");
|
|
await page.locator("#results .result").first().click();
|
|
await page.waitForFunction(
|
|
() => !document.querySelector("#preview")?.hidden,
|
|
);
|
|
if (text === "cancel the supplier partnership") {
|
|
assert(
|
|
body.results.filter((result) => ["Mountain Lake.jpg", "Night Pier.jpg"].includes(result.filename))
|
|
.every(isRelated),
|
|
"Contract photos must remain related suggestions rather than primary results",
|
|
);
|
|
assert(body.results.some((result) => result.filename === "Mountain Lake.jpg"),
|
|
"Eligible mountain photo must remain accessible as a related suggestion");
|
|
assert.equal(
|
|
latestMeasured.first_filename,
|
|
"Supplier Agreement.pdf",
|
|
"Contract query must retrieve the known PDF fixture",
|
|
);
|
|
assert.equal(
|
|
latestMeasured.first_source?.kind,
|
|
"page",
|
|
"PDF match must retain actual page source",
|
|
);
|
|
assert.equal(
|
|
latestMeasured.first_source?.number,
|
|
2,
|
|
"Termination clause must be on actual page2",
|
|
);
|
|
const passage = await page.locator("#preview-text").innerText();
|
|
assert.match(
|
|
passage,
|
|
/sixty|\b60\b/i,
|
|
"Actual PDF preview is missing notice period",
|
|
);
|
|
assert.match(
|
|
passage,
|
|
/terminat/i,
|
|
"Actual PDF preview is missing termination section",
|
|
);
|
|
const literalWords = ["supplier", "partnership"];
|
|
const rowMarks = await page
|
|
.locator("#results .result")
|
|
.first()
|
|
.locator('.snippet mark[data-match="literal"]')
|
|
.allTextContents();
|
|
const passageMarks = await page
|
|
.locator('#preview-text mark[data-match="literal"]')
|
|
.allTextContents();
|
|
for (const word of literalWords) {
|
|
assert(
|
|
rowMarks.some((mark) => mark.toLowerCase() === word),
|
|
`Result snippet must highlight actual query word ${word}`,
|
|
);
|
|
assert(
|
|
passageMarks.some((mark) => mark.toLowerCase() === word),
|
|
`PDF passage must highlight actual query word ${word}`,
|
|
);
|
|
}
|
|
const why = page.locator("#match-reason");
|
|
const disclosure = why.locator("xpath=ancestor::details[1]");
|
|
assert.equal(
|
|
await disclosure.count(),
|
|
1,
|
|
"Match reason must live in a compact disclosure",
|
|
);
|
|
assert.equal(
|
|
await disclosure.evaluate((element) => element.open),
|
|
false,
|
|
"Why details should stay collapsed by default",
|
|
);
|
|
await disclosure.locator(":scope > summary").click();
|
|
await why.waitFor({ state: "visible" });
|
|
assert.match(
|
|
await page.locator("#match-reason").innerText(),
|
|
/page\s*2/i,
|
|
"Match explanation must use actual PDF page2",
|
|
);
|
|
await disclosure.locator(":scope > summary").click();
|
|
assert.equal(
|
|
await disclosure.evaluate((element) => element.open),
|
|
false,
|
|
"Normal screenshot must preserve collapsed why details",
|
|
);
|
|
latestMeasured.ui_checks = {
|
|
literal_query_marks: true,
|
|
actual_page_explanation: true,
|
|
why_disclosure_collapsed_by_default: true,
|
|
unrelated_photos_in_collapsed_related_group: true,
|
|
};
|
|
}
|
|
if (text === "mountains reflected in calm water") {
|
|
assert(body.results.filter((result) => !isRelated(result))
|
|
.every((result) => result.match_kind === "semantic_image"),
|
|
"Visual search primary results must not contain unrelated fixture documents");
|
|
assert.equal(
|
|
latestMeasured.first_filename,
|
|
"Mountain Lake.jpg",
|
|
"Visual query must retrieve the known licensed lake",
|
|
);
|
|
assert.equal(
|
|
await page.locator("#preview-text mark").count(),
|
|
0,
|
|
"Meaning-only photo result must not invent word-level alignment",
|
|
);
|
|
latestMeasured.ui_checks = { meaning_only_image_has_no_text_marks: true, related_sources_retained: true };
|
|
}
|
|
await page.evaluate(
|
|
() =>
|
|
new Promise((resolve) =>
|
|
requestAnimationFrame(() => requestAnimationFrame(resolve)),
|
|
),
|
|
);
|
|
}
|
|
async function screenshot(name, queryText) {
|
|
assert.deepEqual(failures, [], "Frontend page errors occurred");
|
|
const pane = await page.locator(".results-pane").boundingBox();
|
|
const preview = await page.locator("#preview").boundingBox();
|
|
assert(pane && preview && preview.x >= pane.x + pane.width - 1,
|
|
"Wide result and preview panes must remain side by side");
|
|
const path = resolve(out, `${name}.png`);
|
|
await page.locator("#app").screenshot({ path, animations: "disabled" });
|
|
const selected = await page.locator("#preview-title").innerText();
|
|
assert.equal(
|
|
selected,
|
|
latestMeasured.first_filename,
|
|
"Selected DOM result is not the actual first hybrid result",
|
|
);
|
|
evidence.push({
|
|
file: `${name}.png`,
|
|
sha256: createHash("sha256")
|
|
.update(await readFile(path))
|
|
.digest("hex"),
|
|
query: queryText,
|
|
mode: latestMeasured.mode,
|
|
result_count: latestMeasured.result_count,
|
|
main_result_count: latestMeasured.main_result_count,
|
|
related_result_count: latestMeasured.related_result_count,
|
|
related_expanded: await page.locator("#related-suggestions").evaluate((element) => element.open),
|
|
selected_filename: selected,
|
|
first_source_kind: latestMeasured.first_source?.kind,
|
|
first_source_number: latestMeasured.first_source?.number,
|
|
theme: await page.evaluate(() =>
|
|
matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light",
|
|
),
|
|
summary: await page.locator("#result-summary").innerText(),
|
|
capture_bounds: await page.locator("#app").boundingBox(),
|
|
state_assertions: {
|
|
wide_split_preview: true,
|
|
semantic_ready: true,
|
|
hybrid_final: true,
|
|
image_visible: name === "media-search",
|
|
actual_worker_inference:
|
|
latestMeasured.mode === "debug" &&
|
|
latestMeasured.metrics.trace.some(
|
|
(span) => span.counters.worker_inference_us > 0,
|
|
),
|
|
...latestMeasured.ui_checks,
|
|
},
|
|
});
|
|
}
|
|
const documentQuery = "cancel the supplier partnership";
|
|
await query(documentQuery);
|
|
await page.locator("#query").press("ArrowDown");
|
|
assert.equal(await page.locator("#preview-title").innerText(), "Supplier Agreement.pdf",
|
|
"Collapsed related suggestions must not enter arrow-key selection");
|
|
latestMeasured.ui_checks.collapsed_related_skipped_by_keyboard = true;
|
|
await screenshot("search-light", documentQuery);
|
|
await page.locator("#related-suggestions > summary").focus();
|
|
await page.locator("#related-suggestions > summary").press("ArrowDown");
|
|
await page.locator("#related-results .result").first().waitFor({ state: "visible" });
|
|
assert.equal(await page.locator("#related-results .result").count(),
|
|
latestMeasured.related_result_count, "Expanding must restore every actual related neighbor");
|
|
assert.equal(await page.locator("#preview-title").innerText(),
|
|
await page.locator("#related-results .result strong").first().innerText(),
|
|
"ArrowDown on related disclosure must select its first actual candidate");
|
|
await page.locator("#results .result").first().click();
|
|
assert(await page.locator("#related-results .result").filter({ hasText: "Mountain Lake.jpg" }).count(),
|
|
"Expanded related suggestions must expose the actual mountain candidate");
|
|
await screenshot("related-suggestions", documentQuery);
|
|
await page.locator("#related-results .result").first().click();
|
|
await page.locator("#related-suggestions > summary").click();
|
|
await page.waitForFunction(() => document.querySelector("#preview-title")?.textContent === "Supplier Agreement.pdf");
|
|
assert.equal(await page.locator("#preview-title").innerText(), "Supplier Agreement.pdf",
|
|
"Collapsing related suggestions must restore visible primary selection");
|
|
await page.emulateMedia({ colorScheme: "dark" });
|
|
await screenshot("search-dark", documentQuery);
|
|
await page.emulateMedia({ colorScheme: "light" });
|
|
const imageQuery = "mountains reflected in calm water";
|
|
await query(imageQuery);
|
|
await page.waitForFunction(
|
|
() => {
|
|
const image = document.querySelector("#media-preview img");
|
|
return (
|
|
image instanceof HTMLImageElement &&
|
|
image.complete &&
|
|
image.naturalWidth > 0
|
|
);
|
|
},
|
|
{},
|
|
{ timeout: 60000 },
|
|
);
|
|
await screenshot("media-search", imageQuery);
|
|
await page.locator("#settings > summary").click();
|
|
await page.locator("#performance-toggle").check();
|
|
await page.locator("#settings > summary").click();
|
|
// Use a genuinely unseen query so this screenshot records real inference,
|
|
// rather than attributing a bounded query-vector cache hit to the worker.
|
|
const performanceQuery = "how can we cancel the supplier partnership";
|
|
await query(performanceQuery, true);
|
|
await page.locator("#performance-panel").waitFor({ state: "visible" });
|
|
await page.locator("#performance-panel summary").click();
|
|
await page.waitForFunction(
|
|
() =>
|
|
document
|
|
.querySelector("#performance-detail")
|
|
?.textContent?.includes("hybrid") ||
|
|
document
|
|
.querySelector("#performance-detail")
|
|
?.textContent?.includes("Hybrid"),
|
|
);
|
|
assert(
|
|
latestMeasured.metrics.trace?.some(
|
|
(span) => span.counters.worker_inference_us > 0,
|
|
),
|
|
"Detailed trace did not record real worker inference",
|
|
);
|
|
await page
|
|
.locator("#performance-detail .performance-span")
|
|
.filter({ hasText: "worker inference" })
|
|
.first()
|
|
.scrollIntoViewIfNeeded();
|
|
await screenshot("performance-details", performanceQuery);
|
|
const semanticResponse = await context.request.post(
|
|
new URL("/capture-api/semantic_status", target).href,
|
|
{
|
|
headers: { "X-Searchi-Capture-Token": info.token, Origin: target.origin },
|
|
data: {},
|
|
},
|
|
);
|
|
assert(semanticResponse.ok(), "Actual semantic status unavailable");
|
|
const semantic = await semanticResponse.json();
|
|
assert.equal(
|
|
semantic.worker_info?.model,
|
|
"google/embeddinggemma-2",
|
|
"Capture requires the actual Google model",
|
|
);
|
|
assert.equal(
|
|
semantic.worker_info?.threads,
|
|
1,
|
|
"Capture inference requires one CPU thread",
|
|
);
|
|
assert.equal(
|
|
semantic.worker_info?.device,
|
|
"cpu",
|
|
"Capture worker is not CPU",
|
|
);
|
|
await writeFile(
|
|
resolve(out, "capture-manifest.json"),
|
|
JSON.stringify(
|
|
{
|
|
kind: "browser-rendered-production-ui-real-cpu-engine",
|
|
source,
|
|
frontend_build: {
|
|
version: frontendVersion,
|
|
index_sha256: createHash("sha256").update(await readFile(resolve(repository, "apps/desktop/dist/index.html"))).digest("hex"),
|
|
},
|
|
viewport: { width: 1440, height: 1000 },
|
|
runner: { playwright: "1.64.0", chromium: browser.version() },
|
|
backend: {
|
|
mode: info.mode,
|
|
model: semantic.worker_info.model,
|
|
device: semantic.worker_info.device,
|
|
runtime: semantic.worker_info.runtime,
|
|
threads: semantic.worker_info.threads,
|
|
dimensions: semantic.worker_info.dimensions,
|
|
ready: info.ready,
|
|
rust_build_profile: info.rust_build_profile,
|
|
},
|
|
cpu_flags: [
|
|
"--disable-gpu",
|
|
"--disable-accelerated-video-decode",
|
|
"--disable-webgl",
|
|
],
|
|
native_window_chrome: false,
|
|
native_visual_qa_complete: false,
|
|
measured_queries: measured,
|
|
screenshots: evidence,
|
|
},
|
|
null,
|
|
2,
|
|
) + "\n",
|
|
);
|
|
console.log(
|
|
`Captured ${evidence.length} real-engine UI screenshots in ${out}`,
|
|
);
|
|
await context.close();
|
|
} finally {
|
|
await browser.close();
|
|
}
|