A run of the real prompt against the local model showed reasoning written as chat text, internal tool names in a capabilities answer, and only a data fetch when asked to build a tool. The working principles now say that everything written is shown, to describe abilities in plain words, and that a request for a tool means building one. The chart-links spec now waits for the chart or its error and reports browser errors, so the recurring 90-second timeout names its cause. The ZIP upload spec matches the file chip exactly, since the agent reply can quote the name.
124 lines
5.3 KiB
TypeScript
124 lines
5.3 KiB
TypeScript
import { expect, type Locator, type Page, test } from "@playwright/test";
|
|
import { captureScreenshot } from "./helpers";
|
|
|
|
const fixture = "/e2e/fixtures/chart-links.html";
|
|
|
|
/** Browser errors per page, so a chart that never renders fails with its cause. */
|
|
const browserErrors = new WeakMap<Page, string[]>();
|
|
|
|
test.beforeEach(({ page }) => {
|
|
const errors: string[] = [];
|
|
browserErrors.set(page, errors);
|
|
page.on("pageerror", (error) => errors.push(error.message));
|
|
page.on("console", (message) => {
|
|
if (message.type() === "error") errors.push(message.text());
|
|
});
|
|
page.on("response", (response) => {
|
|
if (response.status() >= 400) errors.push(`${response.status()} ${response.url()}`);
|
|
});
|
|
});
|
|
|
|
/** Waits for the plotted dot, failing fast with the chart error and browser errors instead. */
|
|
async function expectChart(page: Page, scope: Locator | Page = page) {
|
|
const circle = scope.locator("svg circle").first();
|
|
const failure = scope.getByText("Chart failed to render", { exact: false }).first();
|
|
try {
|
|
// Every navigation renders Plot again, which takes seconds on a loaded CI runner.
|
|
await expect(circle.or(failure).first()).toBeVisible({ timeout: 90_000 });
|
|
if (await failure.isVisible()) throw new Error(await failure.innerText());
|
|
} catch (error) {
|
|
const errors = browserErrors.get(page) ?? [];
|
|
throw new Error(
|
|
`${error instanceof Error ? error.message : String(error)}\nBrowser errors:\n${errors.join("\n") || "none"}`,
|
|
);
|
|
}
|
|
return circle;
|
|
}
|
|
|
|
test.describe
|
|
.serial("chart links", () => {
|
|
test("cannot execute script when clicked or expanded", async ({ page }, testInfo) => {
|
|
test.setTimeout(120_000);
|
|
// ChartCanvas lazy-loads Observable Plot; pay the import cost once before the matrix below.
|
|
await page.goto(`${fixture}?${new URLSearchParams({ href: "#details", inspect: "0" })}`);
|
|
await expectChart(page);
|
|
for (const inspect of ["0", "1"]) {
|
|
for (const href of [
|
|
'javascript:void(document.body.dataset.plotExecuted="yes")',
|
|
' \u0000\u001fJaVa\tScRi\nPt:void(document.body.dataset.plotExecuted="yes")',
|
|
'javascript:%76oid(document.body.dataset.plotExecuted="yes")',
|
|
"data:text/html,<script>document.body.dataset.plotExecuted='yes'</script>",
|
|
"file:///example.txt",
|
|
]) {
|
|
await page.goto(`${fixture}?${new URLSearchParams({ href, inspect })}`);
|
|
const circle = await expectChart(page);
|
|
const assertInert = async () => {
|
|
const links = page.locator("svg a");
|
|
expect(await links.count()).toBeGreaterThan(0);
|
|
expect(
|
|
await links.evaluateAll((anchors) =>
|
|
anchors.every(
|
|
(anchor) =>
|
|
!anchor.hasAttribute("href") &&
|
|
!anchor.hasAttributeNS("http://www.w3.org/1999/xlink", "href"),
|
|
),
|
|
),
|
|
).toBe(true);
|
|
await expect(page.locator("body")).not.toHaveAttribute("data-plot-executed");
|
|
};
|
|
await assertInert();
|
|
await circle.click();
|
|
// Hover inspection still works, without reintroducing unsafe links.
|
|
if (inspect === "1") await expect(page.locator('[aria-label="tip"]')).toBeVisible();
|
|
await assertInert();
|
|
await page.getByRole("button", { name: "Expand", exact: true }).click();
|
|
const dialog = page.getByRole("dialog");
|
|
await expectChart(page, dialog);
|
|
await dialog.locator("svg circle").click();
|
|
await assertInert();
|
|
await expect(page).toHaveURL(`${fixture}?${new URLSearchParams({ href, inspect })}`);
|
|
await page.getByRole("button", { name: "Close chart" }).click();
|
|
}
|
|
}
|
|
await captureScreenshot(page, testInfo, "chart-with-inert-link");
|
|
});
|
|
|
|
test("retain supported URLs and browser encoding semantics", async ({ page }) => {
|
|
test.setTimeout(120_000);
|
|
await page.route("https://example.test/**", (route) =>
|
|
route.fulfill({ body: "Chart destination" }),
|
|
);
|
|
await page.route("**/e2e/fixtures/java*", (route) =>
|
|
route.fulfill({ body: "Chart destination" }),
|
|
);
|
|
await page.goto(`${fixture}?${new URLSearchParams({ href: "#details" })}`);
|
|
await expectChart(page);
|
|
for (const href of [
|
|
"https://example.test/chart?a=1&b=2",
|
|
"mailto:chart@example.test",
|
|
"tel:+15555550123",
|
|
"#details",
|
|
"javascript:void(0)",
|
|
"javascript%3Avoid(0)",
|
|
"java%73cript:void(0)",
|
|
]) {
|
|
await page.goto(`${fixture}?${new URLSearchParams({ href })}`);
|
|
await expectChart(page);
|
|
const link = page.locator("svg a").first();
|
|
await expect(link).toBeAttached();
|
|
expect(
|
|
await link.evaluate((anchor) =>
|
|
anchor.getAttributeNS("http://www.w3.org/1999/xlink", "href"),
|
|
),
|
|
).toBe(href);
|
|
const destination = new URL(href, page.url());
|
|
expect(destination.protocol).not.toBe("javascript:");
|
|
if (destination.protocol === "http:" || destination.protocol === "https:") {
|
|
await link.locator("circle").click();
|
|
await expect(page).toHaveURL(destination.href);
|
|
await expect(page.locator("body")).not.toHaveAttribute("data-plot-executed");
|
|
}
|
|
}
|
|
});
|
|
});
|