Bots now announce every click and typing step before it runs, and the web app draws a cursor in the bot's colour that glides to that point over the live computer screen, ripples on clicks and shows typing. - contracts: ComputerPointerPayload on the existing thread.computer event - executor: computer_act and browser_act announce pointer steps (best effort); the page-browser helper reports element screen boxes, which are stripped before results reach the model - web: agentPointerFeed + AgentCursor over the side-panel preview and the full-window screen, mapped through the noVNC letterbox - in-app browser: native WebContentsView on desktop (own session, http(s) only, clamped bounds, back/forward, reload, DevTools, load errors) and a sandboxed iframe on the web; replaces the dead panel, the fake 50%/40% cursors and the unimplemented devServer bridge Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi Co-authored-by: Claude <noreply@anthropic.com>
156 lines
6.3 KiB
TypeScript
156 lines
6.3 KiB
TypeScript
import http from "node:http";
|
|
import path from "node:path";
|
|
import { _electron as electron, expect, test } from "@playwright/test";
|
|
import type { RakazoDesktop, RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
|
|
|
|
const fixture = `<!doctype html>
|
|
<html lang="en">
|
|
<head><meta charset="utf-8"><title>Rakazo desktop browser</title></head>
|
|
<body><main>App with a browser panel</main></body>
|
|
</html>`;
|
|
|
|
type Bridge = NonNullable<RakazoDesktop["browserPreview"]>;
|
|
|
|
test("the in-app browser is a native view the app window lays over its panel", async () => {
|
|
const pages = http.createServer((req, res) => {
|
|
res.setHeader("content-type", "text/html");
|
|
if (req.url === "/second") {
|
|
res.end("<!doctype html><title>Second page</title><p>second</p>");
|
|
return;
|
|
}
|
|
res.end(`<!doctype html><title>First page</title>
|
|
<a id="next" href="/second">next</a>
|
|
<a id="popup" href="/second" target="_blank">popup</a>
|
|
<a id="file" href="file:///etc/hosts">file</a>`);
|
|
});
|
|
await new Promise<void>((resolve) => pages.listen(0, "127.0.0.1", resolve));
|
|
const address = pages.address();
|
|
if (!address || typeof address === "string") throw new Error("No fixture server");
|
|
const origin = `http://127.0.0.1:${address.port}`;
|
|
|
|
const app = await electron.launch({
|
|
args: ["."],
|
|
cwd: path.resolve(import.meta.dirname, ".."),
|
|
env: {
|
|
...process.env,
|
|
RAKAZO_WEB_URL: `data:text/html;charset=utf-8,${encodeURIComponent(fixture)}`,
|
|
},
|
|
});
|
|
|
|
try {
|
|
const page = await app.firstWindow();
|
|
await expect(page.getByText("App with a browser panel")).toBeVisible();
|
|
|
|
const call = <K extends keyof Bridge>(method: K, ...args: unknown[]) =>
|
|
page.evaluate(
|
|
([name, values]) => {
|
|
const bridge = (window as typeof window & { rakazoDesktop?: RakazoDesktop }).rakazoDesktop
|
|
?.browserPreview as unknown as Record<
|
|
string,
|
|
(...input: unknown[]) => Promise<unknown>
|
|
>;
|
|
return bridge[name as string]!(...(values as unknown[]));
|
|
},
|
|
[method, args] as const,
|
|
);
|
|
const state = () => call("state") as Promise<RakazoDesktopBrowserPreviewState>;
|
|
const view = () =>
|
|
app.evaluate(({ BrowserWindow }) => {
|
|
const win = BrowserWindow.getAllWindows()[0]!;
|
|
const child = win.contentView.children[0] as
|
|
| (Electron.View & { webContents?: Electron.WebContents })
|
|
| undefined;
|
|
return child
|
|
? {
|
|
count: win.contentView.children.length,
|
|
visible: child.getVisible(),
|
|
bounds: child.getBounds(),
|
|
url: child.webContents?.getURL(),
|
|
partition: child.webContents?.session === win.webContents.session,
|
|
sandbox: child.webContents?.getLastWebPreferences()?.sandbox,
|
|
nodeIntegration: child.webContents?.getLastWebPreferences()?.nodeIntegration,
|
|
}
|
|
: null;
|
|
});
|
|
|
|
// Pages load in a native view in their own session, never the app's.
|
|
await call("navigate", `${origin}/`);
|
|
await expect.poll(async () => (await state()).title).toBe("First page");
|
|
expect(await view()).toMatchObject({
|
|
count: 1,
|
|
visible: false,
|
|
url: `${origin}/`,
|
|
partition: false,
|
|
sandbox: true,
|
|
nodeIntegration: false,
|
|
});
|
|
|
|
await call("show", { x: 400, y: 50, width: 300, height: 200 });
|
|
expect(await view()).toMatchObject({
|
|
visible: true,
|
|
bounds: { x: 400, y: 50, width: 300, height: 200 },
|
|
});
|
|
// Renderer bounds outside the window are clamped to it.
|
|
await call("show", { x: -10, y: -10, width: 100_000, height: 100_000 });
|
|
const [width, height] = await app.evaluate(({ BrowserWindow }) =>
|
|
BrowserWindow.getAllWindows()[0]!.getContentSize(),
|
|
);
|
|
expect((await view())?.bounds).toEqual({ x: 0, y: 0, width, height });
|
|
|
|
const contents = () =>
|
|
app.evaluate(({ BrowserWindow }, script) => {
|
|
const win = BrowserWindow.getAllWindows()[0]!;
|
|
const child = win.contentView.children[0] as Electron.View & {
|
|
webContents: Electron.WebContents;
|
|
};
|
|
return child.webContents.executeJavaScript(script, true);
|
|
}, "document.getElementById('next').click()");
|
|
await contents();
|
|
await expect.poll(async () => (await state()).title).toBe("Second page");
|
|
expect(await state()).toMatchObject({ canGoBack: true, canGoForward: false, error: null });
|
|
|
|
await call("back");
|
|
await expect.poll(async () => (await state()).title).toBe("First page");
|
|
expect(await state()).toMatchObject({ canGoForward: true });
|
|
await call("forward");
|
|
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
|
|
|
|
// New-window links stay in the one view; file links never load.
|
|
await call("back");
|
|
await expect.poll(async () => (await state()).title).toBe("First page");
|
|
await app.evaluate(({ BrowserWindow }) => {
|
|
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
|
|
webContents: Electron.WebContents;
|
|
};
|
|
return child.webContents.executeJavaScript("document.getElementById('popup').click()", true);
|
|
});
|
|
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
|
|
expect(await app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows().length)).toBe(1);
|
|
await call("back");
|
|
await expect.poll(async () => (await state()).title).toBe("First page");
|
|
await app.evaluate(({ BrowserWindow }) => {
|
|
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
|
|
webContents: Electron.WebContents;
|
|
};
|
|
return child.webContents.executeJavaScript("document.getElementById('file').click()", true);
|
|
});
|
|
await page.waitForTimeout(300);
|
|
expect((await state()).url).toBe(`${origin}/`);
|
|
|
|
await expect(call("navigate", "file:///etc/hosts")).rejects.toThrow(/http/);
|
|
|
|
// A dev server that is not running yet shows as an error, then recovers on reload.
|
|
pages.close();
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
await call("navigate", `${origin}/gone`);
|
|
await expect.poll(async () => (await state()).error).toMatch(/ERR_CONNECTION_REFUSED/);
|
|
|
|
await call("hide");
|
|
expect((await view())?.visible).toBe(false);
|
|
} finally {
|
|
await app.close();
|
|
pages.closeAllConnections();
|
|
pages.close();
|
|
}
|
|
});
|