49431def83
Bots now announce every click and typing step before it runs, and the web app draws a cursor in the bot's colour that glides to that point over the live computer screen, ripples on clicks and shows typing. - contracts: ComputerPointerPayload on the existing thread.computer event - executor: computer_act and browser_act announce pointer steps (best effort); the page-browser helper reports element screen boxes, which are stripped before results reach the model - web: agentPointerFeed + AgentCursor over the side-panel preview and the full-window screen, mapped through the noVNC letterbox - in-app browser: native WebContentsView on desktop (own session, http(s) only, clamped bounds, back/forward, reload, DevTools, load errors) and a sandboxed iframe on the web; replaces the dead panel, the fake 50%/40% cursors and the unimplemented devServer bridge Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi Co-authored-by: Claude <noreply@anthropic.com>
54 lines
2.2 KiB
TypeScript
54 lines
2.2 KiB
TypeScript
import { expect, test } from "@playwright/test";
|
|
import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers";
|
|
|
|
const SCREEN_URL = "https://screen.example/embed.html";
|
|
|
|
test("the bot's cursor glides to where it clicks on its live screen", async ({
|
|
page,
|
|
}, testInfo) => {
|
|
await signup(page, `agent-cursor-${Date.now()}@rakazo.test`, "password12", "Cursor Watcher");
|
|
await completeOnboarding(page);
|
|
const botId = activeBotId(page);
|
|
await rpc(page, "computer/boot", { botId });
|
|
// Stand in for the noVNC viewer: a 16:10 desktop with a field in the middle.
|
|
await page.route("https://screen.example/**", (route) =>
|
|
route.fulfill({
|
|
contentType: "text/html",
|
|
body: `<!doctype html><title>Desktop</title><body style="margin:0;background:#1d1d20;display:grid;place-items:center;height:100vh">
|
|
<input style="width:40%;padding:8px;font:16px system-ui" placeholder="Search">`,
|
|
}),
|
|
);
|
|
await page.route("**/rpc/computer/screenUrl", (route) =>
|
|
route.fulfill({
|
|
contentType: "application/json",
|
|
body: JSON.stringify({ json: { url: SCREEN_URL } }),
|
|
}),
|
|
);
|
|
|
|
await page.getByTitle("Agent computer").click();
|
|
const preview = page.getByTestId("computer-preview");
|
|
await expect(preview.locator("iframe")).toHaveAttribute("src", SCREEN_URL);
|
|
const cursor = preview.getByTestId("agent-cursor");
|
|
await expect(cursor).toHaveCSS("opacity", "0");
|
|
|
|
// The scripted runtime clicks the screen center (640, 400) and types.
|
|
await rpc(page, "threads/send", { botId, text: "use your screen and type cursor-demo" });
|
|
|
|
await expect(cursor).toHaveCSS("opacity", "1");
|
|
await expect(cursor).toHaveAttribute("data-action", /click|type/);
|
|
const layer = (await preview.getByTestId("agent-cursor-layer").boundingBox())!;
|
|
await expect
|
|
.poll(async () => {
|
|
const tip = (await cursor.boundingBox())!;
|
|
return [
|
|
Math.round(tip.x - layer.x - layer.width / 2),
|
|
Math.round(tip.y - layer.y - layer.height / 2),
|
|
];
|
|
})
|
|
.toEqual([0, 0]);
|
|
await captureScreenshot(page, testInfo, "agent-cursor-on-screen");
|
|
|
|
// It fades once the bot stops pointing, so it never lingers over the screen.
|
|
await expect(cursor).toHaveCSS("opacity", "0", { timeout: 10_000 });
|
|
});
|