evOo-local/apps/desktop/e2e/browser-preview.spec.ts
cynarAI 49431def83
feat: show the bot's live cursor and make the in-app browser work (#113)
Bots now announce every click and typing step before it runs, and the
web app draws a cursor in the bot's colour that glides to that point
over the live computer screen, ripples on clicks and shows typing.

- contracts: ComputerPointerPayload on the existing thread.computer event
- executor: computer_act and browser_act announce pointer steps
  (best effort); the page-browser helper reports element screen boxes,
  which are stripped before results reach the model
- web: agentPointerFeed + AgentCursor over the side-panel preview and
  the full-window screen, mapped through the noVNC letterbox
- in-app browser: native WebContentsView on desktop (own session, http(s)
  only, clamped bounds, back/forward, reload, DevTools, load errors) and
  a sandboxed iframe on the web; replaces the dead panel, the fake
  50%/40% cursors and the unimplemented devServer bridge


Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi

Co-authored-by: Claude <noreply@anthropic.com>
2026-10-03 19:08:48 +02:00

156 lines
6.3 KiB
TypeScript

import http from "node:http";
import path from "node:path";
import { _electron as electron, expect, test } from "@playwright/test";
import type { RakazoDesktop, RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
const fixture = `<!doctype html>
<html lang="en">
<head><meta charset="utf-8"><title>Rakazo desktop browser</title></head>
<body><main>App with a browser panel</main></body>
</html>`;
type Bridge = NonNullable<RakazoDesktop["browserPreview"]>;
test("the in-app browser is a native view the app window lays over its panel", async () => {
const pages = http.createServer((req, res) => {
res.setHeader("content-type", "text/html");
if (req.url === "/second") {
res.end("<!doctype html><title>Second page</title><p>second</p>");
return;
}
res.end(`<!doctype html><title>First page</title>
<a id="next" href="/second">next</a>
<a id="popup" href="/second" target="_blank">popup</a>
<a id="file" href="file:///etc/hosts">file</a>`);
});
await new Promise<void>((resolve) => pages.listen(0, "127.0.0.1", resolve));
const address = pages.address();
if (!address || typeof address === "string") throw new Error("No fixture server");
const origin = `http://127.0.0.1:${address.port}`;
const app = await electron.launch({
args: ["."],
cwd: path.resolve(import.meta.dirname, ".."),
env: {
...process.env,
RAKAZO_WEB_URL: `data:text/html;charset=utf-8,${encodeURIComponent(fixture)}`,
},
});
try {
const page = await app.firstWindow();
await expect(page.getByText("App with a browser panel")).toBeVisible();
const call = <K extends keyof Bridge>(method: K, ...args: unknown[]) =>
page.evaluate(
([name, values]) => {
const bridge = (window as typeof window & { rakazoDesktop?: RakazoDesktop }).rakazoDesktop
?.browserPreview as unknown as Record<
string,
(...input: unknown[]) => Promise<unknown>
>;
return bridge[name as string]!(...(values as unknown[]));
},
[method, args] as const,
);
const state = () => call("state") as Promise<RakazoDesktopBrowserPreviewState>;
const view = () =>
app.evaluate(({ BrowserWindow }) => {
const win = BrowserWindow.getAllWindows()[0]!;
const child = win.contentView.children[0] as
| (Electron.View & { webContents?: Electron.WebContents })
| undefined;
return child
? {
count: win.contentView.children.length,
visible: child.getVisible(),
bounds: child.getBounds(),
url: child.webContents?.getURL(),
partition: child.webContents?.session === win.webContents.session,
sandbox: child.webContents?.getLastWebPreferences()?.sandbox,
nodeIntegration: child.webContents?.getLastWebPreferences()?.nodeIntegration,
}
: null;
});
// Pages load in a native view in their own session, never the app's.
await call("navigate", `${origin}/`);
await expect.poll(async () => (await state()).title).toBe("First page");
expect(await view()).toMatchObject({
count: 1,
visible: false,
url: `${origin}/`,
partition: false,
sandbox: true,
nodeIntegration: false,
});
await call("show", { x: 400, y: 50, width: 300, height: 200 });
expect(await view()).toMatchObject({
visible: true,
bounds: { x: 400, y: 50, width: 300, height: 200 },
});
// Renderer bounds outside the window are clamped to it.
await call("show", { x: -10, y: -10, width: 100_000, height: 100_000 });
const [width, height] = await app.evaluate(({ BrowserWindow }) =>
BrowserWindow.getAllWindows()[0]!.getContentSize(),
);
expect((await view())?.bounds).toEqual({ x: 0, y: 0, width, height });
const contents = () =>
app.evaluate(({ BrowserWindow }, script) => {
const win = BrowserWindow.getAllWindows()[0]!;
const child = win.contentView.children[0] as Electron.View & {
webContents: Electron.WebContents;
};
return child.webContents.executeJavaScript(script, true);
}, "document.getElementById('next').click()");
await contents();
await expect.poll(async () => (await state()).title).toBe("Second page");
expect(await state()).toMatchObject({ canGoBack: true, canGoForward: false, error: null });
await call("back");
await expect.poll(async () => (await state()).title).toBe("First page");
expect(await state()).toMatchObject({ canGoForward: true });
await call("forward");
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
// New-window links stay in the one view; file links never load.
await call("back");
await expect.poll(async () => (await state()).title).toBe("First page");
await app.evaluate(({ BrowserWindow }) => {
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
webContents: Electron.WebContents;
};
return child.webContents.executeJavaScript("document.getElementById('popup').click()", true);
});
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
expect(await app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows().length)).toBe(1);
await call("back");
await expect.poll(async () => (await state()).title).toBe("First page");
await app.evaluate(({ BrowserWindow }) => {
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
webContents: Electron.WebContents;
};
return child.webContents.executeJavaScript("document.getElementById('file').click()", true);
});
await page.waitForTimeout(300);
expect((await state()).url).toBe(`${origin}/`);
await expect(call("navigate", "file:///etc/hosts")).rejects.toThrow(/http/);
// A dev server that is not running yet shows as an error, then recovers on reload.
pages.close();
await new Promise((resolve) => setTimeout(resolve, 100));
await call("navigate", `${origin}/gone`);
await expect.poll(async () => (await state()).error).toMatch(/ERR_CONNECTION_REFUSED/);
await call("hide");
expect((await view())?.visible).toBe(false);
} finally {
await app.close();
pages.closeAllConnections();
pages.close();
}
});