evOo-local/apps/web/e2e/in-app-browser.spec.ts
cynarAI 49431def83
feat: show the bot's live cursor and make the in-app browser work (#113)
Bots now announce every click and typing step before it runs, and the
web app draws a cursor in the bot's colour that glides to that point
over the live computer screen, ripples on clicks and shows typing.

- contracts: ComputerPointerPayload on the existing thread.computer event
- executor: computer_act and browser_act announce pointer steps
  (best effort); the page-browser helper reports element screen boxes,
  which are stripped before results reach the model
- web: agentPointerFeed + AgentCursor over the side-panel preview and
  the full-window screen, mapped through the noVNC letterbox
- in-app browser: native WebContentsView on desktop (own session, http(s)
  only, clamped bounds, back/forward, reload, DevTools, load errors) and
  a sandboxed iframe on the web; replaces the dead panel, the fake
  50%/40% cursors and the unimplemented devServer bridge


Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi

Co-authored-by: Claude <noreply@anthropic.com>
2026-10-03 19:08:48 +02:00

160 lines
6.4 KiB
TypeScript

import { expect, test } from "@playwright/test";
import { captureScreenshot, completeOnboarding, signup } from "./helpers";
test("the in-app browser previews a page and follows it back to the browser", async ({
page,
}, testInfo) => {
await signup(page, `in-app-browser-${Date.now()}@rakazo.test`, "password12", "Previewer");
await completeOnboarding(page);
// Context-wide so the "Open in browser" tab is served too.
await page.context().route("https://preview.example/**", (route) =>
route.fulfill({
contentType: "text/html",
body: `<!doctype html><title>Preview</title><body style="font:16px system-ui;padding:24px">
<h1>Live preview</h1><p>Rendered inside the app.</p>`,
}),
);
await page.getByRole("button", { name: "Browser" }).click();
const browser = page.getByTestId("in-app-browser");
const address = browser.getByRole("textbox", { name: "Address" });
await expect(address).toBeFocused();
await address.fill("not an address");
await address.press("Enter");
await expect(address).toHaveAttribute("aria-invalid", "true");
await address.fill("preview.example/app");
await address.press("Enter");
await expect(address).toHaveValue("https://preview.example/app");
const frame = browser.getByTestId("in-app-browser-frame");
await expect(frame).toHaveAttribute("src", "https://preview.example/app");
await expect(
page.frameLocator('[data-testid="in-app-browser-frame"]').getByRole("heading"),
).toHaveText("Live preview");
await captureScreenshot(page, testInfo, "in-app-browser");
const popup = page.waitForEvent("popup");
await browser.getByRole("button", { name: "Open in browser" }).click();
expect((await popup).url()).toBe("https://preview.example/app");
await browser.getByRole("button", { name: "Close panel" }).click();
await expect(browser).toHaveCount(0);
// The last address comes back when the browser reopens.
await page.getByRole("button", { name: "Browser" }).click();
await expect(page.getByTestId("in-app-browser-frame")).toHaveAttribute(
"src",
"https://preview.example/app",
);
});
test("on desktop the page is a native view glued to the panel", async ({ page }) => {
await page.addInitScript(() => {
const calls: Array<[string, unknown?]> = [];
let listener: ((state: unknown) => void) | undefined;
let state = {
url: "",
title: "",
loading: false,
canGoBack: false,
canGoForward: false,
error: null as string | null,
};
const record = (name: string) => async (value?: unknown) => {
calls.push([name, value]);
};
Object.assign(window, { __browserCalls: calls });
Object.defineProperty(window, "rakazoDesktop", {
value: {
platform: "linux",
window: {
close: async () => {},
minimize: async () => {},
toggleMaximize: async () => {},
state: async () => ({ minimized: false, maximized: false, fullScreen: false }),
},
oauth: { onCallback: () => () => {} },
update: {
state: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
check: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
download: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
install: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
},
browserPreview: {
state: async () => state,
onState: (next: (value: unknown) => void) => {
listener = next;
return () => {
listener = undefined;
};
},
navigate: async (url: string) => {
calls.push(["navigate", url]);
state = { ...state, url, title: "Dev server", loading: false, canGoBack: true };
listener?.(state);
},
show: record("show"),
hide: record("hide"),
back: record("back"),
forward: record("forward"),
reload: record("reload"),
stop: record("stop"),
openDevTools: record("openDevTools"),
},
},
});
});
await signup(page, `native-browser-${Date.now()}@rakazo.test`, "password12", "Desktop");
await completeOnboarding(page);
const calls = () =>
page.evaluate(
() => (window as unknown as { __browserCalls: Array<[string, unknown?]> }).__browserCalls,
);
await page.getByRole("button", { name: "Browser" }).click();
const browser = page.getByTestId("in-app-browser");
await browser.getByRole("textbox", { name: "Address" }).fill("localhost:5173");
await browser.getByRole("textbox", { name: "Address" }).press("Enter");
await expect.poll(calls).toContainEqual(["navigate", "http://localhost:5173/"]);
await expect(browser.getByRole("button", { name: "Back" })).toBeEnabled();
await expect(browser.getByRole("button", { name: "Forward" })).toBeDisabled();
await expect(browser.getByTestId("in-app-browser-frame")).toHaveCount(0);
// The reserved page area and the native view's bounds stay identical.
const viewport = browser.getByTestId("in-app-browser-viewport");
await expect
.poll(async () => {
const box = (await viewport.boundingBox())!;
const shown = (await calls()).filter(([name]) => name === "show").at(-1)?.[1];
return (
JSON.stringify(shown) ===
JSON.stringify({
x: Math.round(box.x),
y: Math.round(box.y),
width: Math.round(box.width),
height: Math.round(box.height),
})
);
})
.toBe(true);
await browser.getByRole("button", { name: "Back" }).click();
await browser.getByRole("button", { name: "Reload" }).click();
await browser.getByRole("button", { name: "Developer tools" }).click();
await expect
.poll(async () => (await calls()).map(([name]) => name))
.toEqual(expect.arrayContaining(["back", "reload", "openDevTools"]));
// App menus float above the page, so the native view steps aside while one is open.
const before = (await calls()).length;
await page.getByTestId("create-menu-trigger").click();
await expect
.poll(async () => (await calls()).slice(before).map(([name]) => name))
.toContain("hide");
await page.keyboard.press("Escape");
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("show");
await browser.getByRole("button", { name: "Close panel" }).click();
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("hide");
});