import { expect, test } from "@playwright/test";
import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers";
const SCREEN_URL = "https://screen.example/embed.html";
test("the bot's cursor glides to where it clicks on its live screen", async ({
page,
}, testInfo) => {
await signup(page, `agent-cursor-${Date.now()}@rakazo.test`, "password12", "Cursor Watcher");
await completeOnboarding(page);
const botId = activeBotId(page);
await rpc(page, "computer/boot", { botId });
// Stand in for the noVNC viewer: a 16:10 desktop with a field in the middle.
await page.route("https://screen.example/**", (route) =>
route.fulfill({
contentType: "text/html",
body: `
Desktop
`,
}),
);
await page.route("**/rpc/computer/screenUrl", (route) =>
route.fulfill({
contentType: "application/json",
body: JSON.stringify({ json: { url: SCREEN_URL } }),
}),
);
await page.getByTitle("Agent computer").click();
const preview = page.getByTestId("oo-computer-browser-viewport");
await expect(preview.locator("iframe")).toHaveAttribute("src", SCREEN_URL);
const cursor = preview.getByTestId("agent-cursor");
await expect(cursor).toHaveCSS("opacity", "0");
// The click and typing last about a second, so record every action rather than racing a poll.
await cursor.evaluate((node) => {
const seen: string[] = [];
(window as unknown as { cursorActions: string[] }).cursorActions = seen;
new MutationObserver(() => seen.push(node.getAttribute("data-action") ?? "")).observe(node, {
attributeFilter: ["data-action"],
});
});
// The scripted runtime clicks the screen center (640, 400) and types.
await rpc(page, "threads/send", { botId, text: "use your screen and type cursor-demo" });
await expect(cursor).toHaveCSS("opacity", "1");
await expect
.poll(() =>
page.evaluate(() => (window as unknown as { cursorActions: string[] }).cursorActions),
)
.toEqual(expect.arrayContaining(["click", "type"]));
const layer = (await preview.getByTestId("agent-cursor-layer").boundingBox())!;
await expect
.poll(async () => {
const tip = (await cursor.boundingBox())!;
return [
Math.round(tip.x - layer.x - layer.width / 2),
Math.round(tip.y - layer.y - layer.height / 2),
];
})
.toEqual([0, 0]);
await captureScreenshot(page, testInfo, "agent-cursor-on-screen");
// It fades once the bot stops pointing, so it never lingers over the screen.
await expect(cursor).toHaveCSS("opacity", "0", { timeout: 10_000 });
});