Bots now announce every click and typing step before it runs, and the web app draws a cursor in the bot's colour that glides to that point over the live computer screen, ripples on clicks and shows typing. - contracts: ComputerPointerPayload on the existing thread.computer event - executor: computer_act and browser_act announce pointer steps (best effort); the page-browser helper reports element screen boxes, which are stripped before results reach the model - web: agentPointerFeed + AgentCursor over the side-panel preview and the full-window screen, mapped through the noVNC letterbox - in-app browser: native WebContentsView on desktop (own session, http(s) only, clamped bounds, back/forward, reload, DevTools, load errors) and a sandboxed iframe on the web; replaces the dead panel, the fake 50%/40% cursors and the unimplemented devServer bridge Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi Co-authored-by: Claude <noreply@anthropic.com>
160 lines
6.4 KiB
TypeScript
160 lines
6.4 KiB
TypeScript
import { expect, test } from "@playwright/test";
|
|
import { captureScreenshot, completeOnboarding, signup } from "./helpers";
|
|
|
|
test("the in-app browser previews a page and follows it back to the browser", async ({
|
|
page,
|
|
}, testInfo) => {
|
|
await signup(page, `in-app-browser-${Date.now()}@rakazo.test`, "password12", "Previewer");
|
|
await completeOnboarding(page);
|
|
// Context-wide so the "Open in browser" tab is served too.
|
|
await page.context().route("https://preview.example/**", (route) =>
|
|
route.fulfill({
|
|
contentType: "text/html",
|
|
body: `<!doctype html><title>Preview</title><body style="font:16px system-ui;padding:24px">
|
|
<h1>Live preview</h1><p>Rendered inside the app.</p>`,
|
|
}),
|
|
);
|
|
|
|
await page.getByRole("button", { name: "Browser" }).click();
|
|
const browser = page.getByTestId("in-app-browser");
|
|
const address = browser.getByRole("textbox", { name: "Address" });
|
|
await expect(address).toBeFocused();
|
|
|
|
await address.fill("not an address");
|
|
await address.press("Enter");
|
|
await expect(address).toHaveAttribute("aria-invalid", "true");
|
|
|
|
await address.fill("preview.example/app");
|
|
await address.press("Enter");
|
|
await expect(address).toHaveValue("https://preview.example/app");
|
|
const frame = browser.getByTestId("in-app-browser-frame");
|
|
await expect(frame).toHaveAttribute("src", "https://preview.example/app");
|
|
await expect(
|
|
page.frameLocator('[data-testid="in-app-browser-frame"]').getByRole("heading"),
|
|
).toHaveText("Live preview");
|
|
await captureScreenshot(page, testInfo, "in-app-browser");
|
|
|
|
const popup = page.waitForEvent("popup");
|
|
await browser.getByRole("button", { name: "Open in browser" }).click();
|
|
expect((await popup).url()).toBe("https://preview.example/app");
|
|
|
|
await browser.getByRole("button", { name: "Close panel" }).click();
|
|
await expect(browser).toHaveCount(0);
|
|
// The last address comes back when the browser reopens.
|
|
await page.getByRole("button", { name: "Browser" }).click();
|
|
await expect(page.getByTestId("in-app-browser-frame")).toHaveAttribute(
|
|
"src",
|
|
"https://preview.example/app",
|
|
);
|
|
});
|
|
|
|
test("on desktop the page is a native view glued to the panel", async ({ page }) => {
|
|
await page.addInitScript(() => {
|
|
const calls: Array<[string, unknown?]> = [];
|
|
let listener: ((state: unknown) => void) | undefined;
|
|
let state = {
|
|
url: "",
|
|
title: "",
|
|
loading: false,
|
|
canGoBack: false,
|
|
canGoForward: false,
|
|
error: null as string | null,
|
|
};
|
|
const record = (name: string) => async (value?: unknown) => {
|
|
calls.push([name, value]);
|
|
};
|
|
Object.assign(window, { __browserCalls: calls });
|
|
Object.defineProperty(window, "rakazoDesktop", {
|
|
value: {
|
|
platform: "linux",
|
|
window: {
|
|
close: async () => {},
|
|
minimize: async () => {},
|
|
toggleMaximize: async () => {},
|
|
state: async () => ({ minimized: false, maximized: false, fullScreen: false }),
|
|
},
|
|
oauth: { onCallback: () => () => {} },
|
|
update: {
|
|
state: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
|
check: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
|
download: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
|
install: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
|
},
|
|
browserPreview: {
|
|
state: async () => state,
|
|
onState: (next: (value: unknown) => void) => {
|
|
listener = next;
|
|
return () => {
|
|
listener = undefined;
|
|
};
|
|
},
|
|
navigate: async (url: string) => {
|
|
calls.push(["navigate", url]);
|
|
state = { ...state, url, title: "Dev server", loading: false, canGoBack: true };
|
|
listener?.(state);
|
|
},
|
|
show: record("show"),
|
|
hide: record("hide"),
|
|
back: record("back"),
|
|
forward: record("forward"),
|
|
reload: record("reload"),
|
|
stop: record("stop"),
|
|
openDevTools: record("openDevTools"),
|
|
},
|
|
},
|
|
});
|
|
});
|
|
await signup(page, `native-browser-${Date.now()}@rakazo.test`, "password12", "Desktop");
|
|
await completeOnboarding(page);
|
|
const calls = () =>
|
|
page.evaluate(
|
|
() => (window as unknown as { __browserCalls: Array<[string, unknown?]> }).__browserCalls,
|
|
);
|
|
|
|
await page.getByRole("button", { name: "Browser" }).click();
|
|
const browser = page.getByTestId("in-app-browser");
|
|
await browser.getByRole("textbox", { name: "Address" }).fill("localhost:5173");
|
|
await browser.getByRole("textbox", { name: "Address" }).press("Enter");
|
|
await expect.poll(calls).toContainEqual(["navigate", "http://localhost:5173/"]);
|
|
await expect(browser.getByRole("button", { name: "Back" })).toBeEnabled();
|
|
await expect(browser.getByRole("button", { name: "Forward" })).toBeDisabled();
|
|
await expect(browser.getByTestId("in-app-browser-frame")).toHaveCount(0);
|
|
|
|
// The reserved page area and the native view's bounds stay identical.
|
|
const viewport = browser.getByTestId("in-app-browser-viewport");
|
|
await expect
|
|
.poll(async () => {
|
|
const box = (await viewport.boundingBox())!;
|
|
const shown = (await calls()).filter(([name]) => name === "show").at(-1)?.[1];
|
|
return (
|
|
JSON.stringify(shown) ===
|
|
JSON.stringify({
|
|
x: Math.round(box.x),
|
|
y: Math.round(box.y),
|
|
width: Math.round(box.width),
|
|
height: Math.round(box.height),
|
|
})
|
|
);
|
|
})
|
|
.toBe(true);
|
|
|
|
await browser.getByRole("button", { name: "Back" }).click();
|
|
await browser.getByRole("button", { name: "Reload" }).click();
|
|
await browser.getByRole("button", { name: "Developer tools" }).click();
|
|
await expect
|
|
.poll(async () => (await calls()).map(([name]) => name))
|
|
.toEqual(expect.arrayContaining(["back", "reload", "openDevTools"]));
|
|
|
|
// App menus float above the page, so the native view steps aside while one is open.
|
|
const before = (await calls()).length;
|
|
await page.getByTestId("create-menu-trigger").click();
|
|
await expect
|
|
.poll(async () => (await calls()).slice(before).map(([name]) => name))
|
|
.toContain("hide");
|
|
await page.keyboard.press("Escape");
|
|
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("show");
|
|
|
|
await browser.getByRole("button", { name: "Close panel" }).click();
|
|
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("hide");
|
|
});
|