evOo-local/apps/web/e2e/chart-links.spec.ts
cynarAI 42eb420545
Agent replies without thinking aloud or tool names; chart-links E2E names its failure (#199)
A run of the real prompt against the local model showed reasoning written as chat text, internal tool names in a capabilities answer, and only a data fetch when asked to build a tool. The working principles now say that everything written is shown, to describe abilities in plain words, and that a request for a tool means building one.

The chart-links spec now waits for the chart or its error and reports browser errors, so the recurring 90-second timeout names its cause. The ZIP upload spec matches the file chip exactly, since the agent reply can quote the name.
2026-10-10 17:24:34 +02:00

124 lines
5.3 KiB
TypeScript

import { expect, type Locator, type Page, test } from "@playwright/test";
import { captureScreenshot } from "./helpers";
const fixture = "/e2e/fixtures/chart-links.html";
/** Browser errors per page, so a chart that never renders fails with its cause. */
const browserErrors = new WeakMap<Page, string[]>();
test.beforeEach(({ page }) => {
const errors: string[] = [];
browserErrors.set(page, errors);
page.on("pageerror", (error) => errors.push(error.message));
page.on("console", (message) => {
if (message.type() === "error") errors.push(message.text());
});
page.on("response", (response) => {
if (response.status() >= 400) errors.push(`${response.status()} ${response.url()}`);
});
});
/** Waits for the plotted dot, failing fast with the chart error and browser errors instead. */
async function expectChart(page: Page, scope: Locator | Page = page) {
const circle = scope.locator("svg circle").first();
const failure = scope.getByText("Chart failed to render", { exact: false }).first();
try {
// Every navigation renders Plot again, which takes seconds on a loaded CI runner.
await expect(circle.or(failure).first()).toBeVisible({ timeout: 90_000 });
if (await failure.isVisible()) throw new Error(await failure.innerText());
} catch (error) {
const errors = browserErrors.get(page) ?? [];
throw new Error(
`${error instanceof Error ? error.message : String(error)}\nBrowser errors:\n${errors.join("\n") || "none"}`,
);
}
return circle;
}
test.describe
.serial("chart links", () => {
test("cannot execute script when clicked or expanded", async ({ page }, testInfo) => {
test.setTimeout(120_000);
// ChartCanvas lazy-loads Observable Plot; pay the import cost once before the matrix below.
await page.goto(`${fixture}?${new URLSearchParams({ href: "#details", inspect: "0" })}`);
await expectChart(page);
for (const inspect of ["0", "1"]) {
for (const href of [
'javascript:void(document.body.dataset.plotExecuted="yes")',
' \u0000\u001fJaVa\tScRi\nPt:void(document.body.dataset.plotExecuted="yes")',
'javascript:%76oid(document.body.dataset.plotExecuted="yes")',
"data:text/html,<script>document.body.dataset.plotExecuted='yes'</script>",
"file:///example.txt",
]) {
await page.goto(`${fixture}?${new URLSearchParams({ href, inspect })}`);
const circle = await expectChart(page);
const assertInert = async () => {
const links = page.locator("svg a");
expect(await links.count()).toBeGreaterThan(0);
expect(
await links.evaluateAll((anchors) =>
anchors.every(
(anchor) =>
!anchor.hasAttribute("href") &&
!anchor.hasAttributeNS("http://www.w3.org/1999/xlink", "href"),
),
),
).toBe(true);
await expect(page.locator("body")).not.toHaveAttribute("data-plot-executed");
};
await assertInert();
await circle.click();
// Hover inspection still works, without reintroducing unsafe links.
if (inspect === "1") await expect(page.locator('[aria-label="tip"]')).toBeVisible();
await assertInert();
await page.getByRole("button", { name: "Expand", exact: true }).click();
const dialog = page.getByRole("dialog");
await expectChart(page, dialog);
await dialog.locator("svg circle").click();
await assertInert();
await expect(page).toHaveURL(`${fixture}?${new URLSearchParams({ href, inspect })}`);
await page.getByRole("button", { name: "Close chart" }).click();
}
}
await captureScreenshot(page, testInfo, "chart-with-inert-link");
});
test("retain supported URLs and browser encoding semantics", async ({ page }) => {
test.setTimeout(120_000);
await page.route("https://example.test/**", (route) =>
route.fulfill({ body: "Chart destination" }),
);
await page.route("**/e2e/fixtures/java*", (route) =>
route.fulfill({ body: "Chart destination" }),
);
await page.goto(`${fixture}?${new URLSearchParams({ href: "#details" })}`);
await expectChart(page);
for (const href of [
"https://example.test/chart?a=1&b=2",
"mailto:chart@example.test",
"tel:+15555550123",
"#details",
"java&#x73;cript:void(0)",
"javascript%3Avoid(0)",
"java%73cript:void(0)",
]) {
await page.goto(`${fixture}?${new URLSearchParams({ href })}`);
await expectChart(page);
const link = page.locator("svg a").first();
await expect(link).toBeAttached();
expect(
await link.evaluate((anchor) =>
anchor.getAttributeNS("http://www.w3.org/1999/xlink", "href"),
),
).toBe(href);
const destination = new URL(href, page.url());
expect(destination.protocol).not.toBe("javascript:");
if (destination.protocol === "http:" || destination.protocol === "https:") {
await link.locator("circle").click();
await expect(page).toHaveURL(destination.href);
await expect(page.locator("body")).not.toHaveAttribute("data-plot-executed");
}
}
});
});