import { expect, test } from "@playwright/test"; import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers"; const SCREEN_URL = "https://screen.example/embed.html"; test("the bot's cursor glides to where it clicks on its live screen", async ({ page, }, testInfo) => { await signup(page, `agent-cursor-${Date.now()}@rakazo.test`, "password12", "Cursor Watcher"); await completeOnboarding(page); const botId = activeBotId(page); await rpc(page, "computer/boot", { botId }); // Stand in for the noVNC viewer: a 16:10 desktop with a field in the middle. await page.route("https://screen.example/**", (route) => route.fulfill({ contentType: "text/html", body: `Desktop `, }), ); await page.route("**/rpc/computer/screenUrl", (route) => route.fulfill({ contentType: "application/json", body: JSON.stringify({ json: { url: SCREEN_URL } }), }), ); await page.getByTitle("Agent computer").click(); const preview = page.getByTestId("oo-computer-browser-viewport"); await expect(preview.locator("iframe")).toHaveAttribute("src", SCREEN_URL); const cursor = preview.getByTestId("agent-cursor"); await expect(cursor).toHaveCSS("opacity", "0"); // The click and typing last about a second, so record every action rather than racing a poll. await cursor.evaluate((node) => { const seen: string[] = []; (window as unknown as { cursorActions: string[] }).cursorActions = seen; new MutationObserver(() => seen.push(node.getAttribute("data-action") ?? "")).observe(node, { attributeFilter: ["data-action"], }); }); // The scripted runtime clicks the screen center (640, 400) and types. await rpc(page, "threads/send", { botId, text: "use your screen and type cursor-demo" }); await expect(cursor).toHaveCSS("opacity", "1"); await expect .poll(() => page.evaluate(() => (window as unknown as { cursorActions: string[] }).cursorActions), ) .toEqual(expect.arrayContaining(["click", "type"])); const layer = (await preview.getByTestId("agent-cursor-layer").boundingBox())!; await expect .poll(async () => { const tip = (await cursor.boundingBox())!; return [ Math.round(tip.x - layer.x - layer.width / 2), Math.round(tip.y - layer.y - layer.height / 2), ]; }) .toEqual([0, 0]); await captureScreenshot(page, testInfo, "agent-cursor-on-screen"); // It fades once the bot stops pointing, so it never lingers over the screen. await expect(cursor).toHaveCSS("opacity", "0", { timeout: 10_000 }); });