diff --git a/apps/desktop/e2e/browser-preview.spec.ts b/apps/desktop/e2e/browser-preview.spec.ts new file mode 100644 index 0000000..b94f901 --- /dev/null +++ b/apps/desktop/e2e/browser-preview.spec.ts @@ -0,0 +1,155 @@ +import http from "node:http"; +import path from "node:path"; +import { _electron as electron, expect, test } from "@playwright/test"; +import type { RakazoDesktop, RakazoDesktopBrowserPreviewState } from "@rakazo/contracts"; + +const fixture = ` + + Rakazo desktop browser +
App with a browser panel
+`; + +type Bridge = NonNullable; + +test("the in-app browser is a native view the app window lays over its panel", async () => { + const pages = http.createServer((req, res) => { + res.setHeader("content-type", "text/html"); + if (req.url === "/second") { + res.end("Second page

second

"); + return; + } + res.end(`First page + + popup + file`); + }); + await new Promise((resolve) => pages.listen(0, "127.0.0.1", resolve)); + const address = pages.address(); + if (!address || typeof address === "string") throw new Error("No fixture server"); + const origin = `http://127.0.0.1:${address.port}`; + + const app = await electron.launch({ + args: ["."], + cwd: path.resolve(import.meta.dirname, ".."), + env: { + ...process.env, + RAKAZO_WEB_URL: `data:text/html;charset=utf-8,${encodeURIComponent(fixture)}`, + }, + }); + + try { + const page = await app.firstWindow(); + await expect(page.getByText("App with a browser panel")).toBeVisible(); + + const call = (method: K, ...args: unknown[]) => + page.evaluate( + ([name, values]) => { + const bridge = (window as typeof window & { rakazoDesktop?: RakazoDesktop }).rakazoDesktop + ?.browserPreview as unknown as Record< + string, + (...input: unknown[]) => Promise + >; + return bridge[name as string]!(...(values as unknown[])); + }, + [method, args] as const, + ); + const state = () => call("state") as Promise; + const view = () => + app.evaluate(({ BrowserWindow }) => { + const win = BrowserWindow.getAllWindows()[0]!; + const child = win.contentView.children[0] as + | (Electron.View & { webContents?: Electron.WebContents }) + | undefined; + return child + ? { + count: win.contentView.children.length, + visible: child.getVisible(), + bounds: child.getBounds(), + url: child.webContents?.getURL(), + partition: child.webContents?.session === win.webContents.session, + sandbox: child.webContents?.getLastWebPreferences()?.sandbox, + nodeIntegration: child.webContents?.getLastWebPreferences()?.nodeIntegration, + } + : null; + }); + + // Pages load in a native view in their own session, never the app's. + await call("navigate", `${origin}/`); + await expect.poll(async () => (await state()).title).toBe("First page"); + expect(await view()).toMatchObject({ + count: 1, + visible: false, + url: `${origin}/`, + partition: false, + sandbox: true, + nodeIntegration: false, + }); + + await call("show", { x: 400, y: 50, width: 300, height: 200 }); + expect(await view()).toMatchObject({ + visible: true, + bounds: { x: 400, y: 50, width: 300, height: 200 }, + }); + // Renderer bounds outside the window are clamped to it. + await call("show", { x: -10, y: -10, width: 100_000, height: 100_000 }); + const [width, height] = await app.evaluate(({ BrowserWindow }) => + BrowserWindow.getAllWindows()[0]!.getContentSize(), + ); + expect((await view())?.bounds).toEqual({ x: 0, y: 0, width, height }); + + const contents = () => + app.evaluate(({ BrowserWindow }, script) => { + const win = BrowserWindow.getAllWindows()[0]!; + const child = win.contentView.children[0] as Electron.View & { + webContents: Electron.WebContents; + }; + return child.webContents.executeJavaScript(script, true); + }, "document.getElementById('next').click()"); + await contents(); + await expect.poll(async () => (await state()).title).toBe("Second page"); + expect(await state()).toMatchObject({ canGoBack: true, canGoForward: false, error: null }); + + await call("back"); + await expect.poll(async () => (await state()).title).toBe("First page"); + expect(await state()).toMatchObject({ canGoForward: true }); + await call("forward"); + await expect.poll(async () => (await state()).url).toBe(`${origin}/second`); + + // New-window links stay in the one view; file links never load. + await call("back"); + await expect.poll(async () => (await state()).title).toBe("First page"); + await app.evaluate(({ BrowserWindow }) => { + const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & { + webContents: Electron.WebContents; + }; + return child.webContents.executeJavaScript("document.getElementById('popup').click()", true); + }); + await expect.poll(async () => (await state()).url).toBe(`${origin}/second`); + expect(await app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows().length)).toBe(1); + await call("back"); + await expect.poll(async () => (await state()).title).toBe("First page"); + await app.evaluate(({ BrowserWindow }) => { + const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & { + webContents: Electron.WebContents; + }; + return child.webContents.executeJavaScript("document.getElementById('file').click()", true); + }); + await page.waitForTimeout(300); + expect((await state()).url).toBe(`${origin}/`); + + await expect(call("navigate", "file:///etc/hosts")).rejects.toThrow(/http/); + + // A dev server that is not running yet shows as an error, then recovers on reload. + pages.close(); + await new Promise((resolve) => setTimeout(resolve, 100)); + await call("navigate", `${origin}/gone`); + await expect.poll(async () => (await state()).error).toMatch(/ERR_CONNECTION_REFUSED/); + + await call("hide"); + expect((await view())?.visible).toBe(false); + } finally { + await app.close(); + pages.closeAllConnections(); + pages.close(); + } +}); diff --git a/apps/desktop/e2e/smoke.spec.ts b/apps/desktop/e2e/smoke.spec.ts index dcd8ae4..62ddcf2 100644 --- a/apps/desktop/e2e/smoke.spec.ts +++ b/apps/desktop/e2e/smoke.spec.ts @@ -42,6 +42,7 @@ test("launches with a narrow preload bridge and an isolated renderer", async () }); expect(renderer.bridgeKeys).toEqual([ + "browserPreview", "files", "localSettings", "oauth", diff --git a/apps/desktop/src/browser-preview-policy.test.ts b/apps/desktop/src/browser-preview-policy.test.ts new file mode 100644 index 0000000..b50fb32 --- /dev/null +++ b/apps/desktop/src/browser-preview-policy.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; +import { + previewBounds, + previewLoadError, + previewShortcut, + previewUrl, +} from "./browser-preview-policy.js"; + +describe("previewUrl", () => { + it("accepts only http(s) without credentials", () => { + expect(previewUrl("http://localhost:5173")).toBe("http://localhost:5173/"); + expect(previewUrl("https://example.com/a?b=1")).toBe("https://example.com/a?b=1"); + for (const value of [ + "file:///etc/hosts", + "javascript:alert(1)", + "chrome://settings", + "https://user:fake-password@example.com", + "not a url", + 42, + `https://example.com/${"a".repeat(9000)}`, + ]) { + expect(previewUrl(value)).toBeNull(); + } + }); +}); + +describe("previewBounds", () => { + const content = { width: 1200, height: 800 }; + + it("rounds and applies the app zoom factor", () => { + expect(previewBounds({ x: 800.4, y: 40.6, width: 399.5, height: 700 }, content)).toEqual({ + x: 800, + y: 41, + width: 400, + height: 700, + }); + expect(previewBounds({ x: 100, y: 10, width: 200, height: 100 }, content, 1.25)).toEqual({ + x: 125, + y: 13, + width: 250, + height: 125, + }); + }); + + it("clamps to the window and rejects empty or malformed rectangles", () => { + expect(previewBounds({ x: -50, y: -10, width: 5000, height: 5000 }, content)).toEqual({ + x: 0, + y: 0, + width: 1200, + height: 800, + }); + expect(previewBounds({ x: 1300, y: 0, width: 100, height: 100 }, content)).toBeNull(); + expect(previewBounds({ x: 0, y: 0, width: 0, height: 100 }, content)).toBeNull(); + expect(previewBounds({ x: "1", y: 0, width: 1, height: 1 }, content)).toBeNull(); + expect(previewBounds({ x: Number.NaN, y: 0, width: 1, height: 1 }, content)).toBeNull(); + expect(previewBounds(null, content)).toBeNull(); + }); +}); + +describe("previewShortcut", () => { + const key = (overrides: Partial[0]>) => ({ + type: "keyDown", + key: "r", + control: false, + meta: false, + alt: false, + shift: false, + ...overrides, + }); + + it("maps platform browser shortcuts", () => { + expect(previewShortcut(key({ meta: true }), "darwin")).toBe("reload"); + expect(previewShortcut(key({ meta: true, shift: true, key: "R" }), "darwin")).toBe( + "hard-reload", + ); + expect(previewShortcut(key({ control: true }), "linux")).toBe("reload"); + expect(previewShortcut(key({ meta: true, key: "[" }), "darwin")).toBe("back"); + expect(previewShortcut(key({ alt: true, key: "ArrowRight" }), "win32")).toBe("forward"); + expect(previewShortcut(key({ key: "F12" }), "linux")).toBe("devtools"); + expect(previewShortcut(key({ key: "F5" }), "win32")).toBe("reload"); + }); + + it("leaves typing alone", () => { + expect(previewShortcut(key({}), "darwin")).toBeNull(); + expect(previewShortcut(key({ control: true }), "darwin")).toBeNull(); + expect(previewShortcut(key({ type: "keyUp", meta: true }), "darwin")).toBeNull(); + expect(previewShortcut(key({ key: "ArrowLeft" }), "linux")).toBeNull(); + }); +}); + +describe("previewLoadError", () => { + it("ignores superseded navigations", () => { + expect(previewLoadError(-3, "ERR_ABORTED")).toBeNull(); + expect(previewLoadError(-102, "ERR_CONNECTION_REFUSED")).toBe("ERR_CONNECTION_REFUSED"); + }); +}); diff --git a/apps/desktop/src/browser-preview-policy.ts b/apps/desktop/src/browser-preview-policy.ts new file mode 100644 index 0000000..c598abd --- /dev/null +++ b/apps/desktop/src/browser-preview-policy.ts @@ -0,0 +1,76 @@ +import type { RakazoDesktopBrowserBounds } from "@rakazo/contracts"; + +/** Only plain http(s) pages load in the in-app browser; no files, schemes or URL credentials. */ +export function previewUrl(value: unknown): string | null { + if (typeof value !== "string" || value.length > 8192) return null; + try { + const url = new URL(value); + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username || url.password) return null; + return url.toString(); + } catch { + return null; + } +} + +/** + * Renderer-reported CSS-pixel bounds, converted to window DIPs and clamped to the content area + * so a compromised renderer cannot place the view outside the panel it reserved. + */ +export function previewBounds( + value: unknown, + content: { width: number; height: number }, + zoomFactor = 1, +): RakazoDesktopBrowserBounds | null { + if (!value || typeof value !== "object") return null; + const raw = value as Record; + const numbers = [raw.x, raw.y, raw.width, raw.height]; + if (!numbers.every((n) => typeof n === "number" && Number.isFinite(n))) return null; + const scale = Number.isFinite(zoomFactor) && zoomFactor > 0 ? zoomFactor : 1; + const [x, y, width, height] = (numbers as number[]).map((n) => Math.round(n * scale)) as [ + number, + number, + number, + number, + ]; + const left = Math.min(Math.max(0, x), content.width); + const top = Math.min(Math.max(0, y), content.height); + const right = Math.min(Math.max(left, x + width), content.width); + const bottom = Math.min(Math.max(top, y + height), content.height); + if (right - left < 1 || bottom - top < 1) return null; + return { x: left, y: top, width: right - left, height: bottom - top }; +} + +export type PreviewShortcut = "reload" | "hard-reload" | "back" | "forward" | "devtools"; + +/** Browser keyboard shortcuts while focus is inside the page. */ +export function previewShortcut( + input: { + type: string; + key: string; + control: boolean; + meta: boolean; + alt: boolean; + shift: boolean; + }, + platform: string, +): PreviewShortcut | null { + if (input.type !== "keyDown") return null; + const mod = platform === "darwin" ? input.meta : input.control; + const key = input.key.length === 1 ? input.key.toLowerCase() : input.key; + if (key === "F5") return input.shift ? "hard-reload" : "reload"; + if (key === "F12") return "devtools"; + if (mod && key === "r") return input.shift ? "hard-reload" : "reload"; + if (mod && input.alt && key === "i") return "devtools"; + if (platform === "darwin" ? input.meta && !input.alt : input.alt && !input.control) { + if (key === (platform === "darwin" ? "[" : "ArrowLeft")) return "back"; + if (key === (platform === "darwin" ? "]" : "ArrowRight")) return "forward"; + } + return null; +} + +/** Load failures worth showing; an aborted load (-3) is a superseded navigation, not an error. */ +export function previewLoadError(errorCode: number, description: string): string | null { + if (errorCode === -3 || errorCode === 0) return null; + return description || `Load failed (${errorCode})`; +} diff --git a/apps/desktop/src/browser-preview.ts b/apps/desktop/src/browser-preview.ts new file mode 100644 index 0000000..7d4c908 --- /dev/null +++ b/apps/desktop/src/browser-preview.ts @@ -0,0 +1,189 @@ +import type { RakazoDesktopBrowserPreviewState } from "@rakazo/contracts"; +import { type BrowserWindow, WebContentsView } from "electron"; +import { + previewBounds, + previewLoadError, + previewShortcut, + previewUrl, +} from "./browser-preview-policy.js"; + +/** Pages share one persistent profile, separate from the app's own session and cookies. */ +const PREVIEW_PARTITION = "persist:rakazo-browser"; +export const PREVIEW_STATE_CHANNEL = "desktop.browserPreview.state"; + +/** + * The in-app browser's native page: one WebContentsView laid over the panel the app window + * reserves. It belongs to one app window at a time and is torn down with it. + */ +export class BrowserPreviewController { + private host: BrowserWindow | null = null; + private view: WebContentsView | null = null; + private error: string | null = null; + + state(win: BrowserWindow): RakazoDesktopBrowserPreviewState { + const contents = this.host === win ? this.view?.webContents : undefined; + if (!contents || contents.isDestroyed()) { + return { + url: "", + title: "", + loading: false, + canGoBack: false, + canGoForward: false, + error: null, + }; + } + const url = contents.getURL(); + return { + url, + title: contents.getTitle() === url ? "" : contents.getTitle(), + loading: contents.isLoading(), + canGoBack: contents.navigationHistory.canGoBack(), + canGoForward: contents.navigationHistory.canGoForward(), + error: this.error, + }; + } + + show(win: BrowserWindow, rawBounds: unknown) { + const view = this.viewFor(win); + const [width, height] = win.getContentSize(); + const bounds = previewBounds( + rawBounds, + { width: width ?? 0, height: height ?? 0 }, + win.webContents.getZoomFactor(), + ); + if (!bounds) { + view.setVisible(false); + return; + } + view.setBounds(bounds); + view.setVisible(true); + } + + hide(win: BrowserWindow) { + if (this.host === win) this.view?.setVisible(false); + } + + async navigate(win: BrowserWindow, rawUrl: unknown) { + const url = previewUrl(rawUrl); + if (!url) throw new Error("Only http(s) pages can open here."); + this.error = null; + // Failures surface through did-fail-load as page state, not as a rejected call. + await this.viewFor(win) + .webContents.loadURL(url) + .catch(() => undefined); + } + + back(win: BrowserWindow) { + const history = this.contentsFor(win)?.navigationHistory; + if (history?.canGoBack()) history.goBack(); + } + + forward(win: BrowserWindow) { + const history = this.contentsFor(win)?.navigationHistory; + if (history?.canGoForward()) history.goForward(); + } + + reload(win: BrowserWindow, ignoreCache = false) { + const contents = this.contentsFor(win); + if (!contents?.getURL()) return; + this.error = null; + if (ignoreCache) contents.reloadIgnoringCache(); + else contents.reload(); + } + + stop(win: BrowserWindow) { + this.contentsFor(win)?.stop(); + } + + openDevTools(win: BrowserWindow) { + const contents = this.contentsFor(win); + if (contents?.getURL()) contents.openDevTools({ mode: "detach" }); + } + + private contentsFor(win: BrowserWindow) { + const contents = this.host === win ? this.view?.webContents : undefined; + return contents && !contents.isDestroyed() ? contents : undefined; + } + + private viewFor(win: BrowserWindow): WebContentsView { + if (this.host === win && this.view && !this.view.webContents.isDestroyed()) return this.view; + this.destroy(); + const view = new WebContentsView({ + webPreferences: { + partition: PREVIEW_PARTITION, + sandbox: true, + contextIsolation: true, + nodeIntegration: false, + webSecurity: true, + spellcheck: false, + }, + }); + view.setVisible(false); + const contents = view.webContents; + const publish = () => { + if (!win.isDestroyed()) win.webContents.send(PREVIEW_STATE_CHANNEL, this.state(win)); + }; + // One tab: links that want a new window open here instead. + contents.setWindowOpenHandler(({ url }) => { + const target = previewUrl(url); + if (target) void contents.loadURL(target).catch(() => undefined); + return { action: "deny" }; + }); + contents.on("will-navigate", (event, url) => { + if (!previewUrl(url)) event.preventDefault(); + }); + contents.on("will-redirect", (event, url) => { + if (!previewUrl(url)) event.preventDefault(); + }); + contents.on("did-start-loading", publish); + contents.on("did-stop-loading", publish); + contents.on("did-navigate", () => { + this.error = null; + publish(); + }); + contents.on("did-navigate-in-page", publish); + contents.on("page-title-updated", publish); + contents.on("did-fail-load", (_event, code, description, _url, isMainFrame) => { + if (!isMainFrame) return; + const error = previewLoadError(code, description); + if (error === null) return; + this.error = error; + publish(); + }); + contents.on("render-process-gone", (_event, details) => { + this.error = details.reason === "clean-exit" ? null : "The page stopped responding."; + publish(); + }); + contents.on("before-input-event", (event, input) => { + const shortcut = previewShortcut(input, process.platform); + if (!shortcut) return; + event.preventDefault(); + if (shortcut === "reload" || shortcut === "hard-reload") { + this.reload(win, shortcut === "hard-reload"); + } else if (shortcut === "back") { + this.back(win); + } else if (shortcut === "forward") { + this.forward(win); + } else { + this.openDevTools(win); + } + }); + win.contentView.addChildView(view); + win.once("closed", () => { + if (this.host === win) this.destroy(); + }); + this.host = win; + this.view = view; + return view; + } + + private destroy() { + const { host, view } = this; + this.host = null; + this.view = null; + this.error = null; + if (!view) return; + if (host && !host.isDestroyed()) host.contentView.removeChildView(view); + if (!view.webContents.isDestroyed()) view.webContents.close(); + } +} diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 304e998..052ab36 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -21,6 +21,7 @@ import { LAUNCH_CHECK_DELAY_MS, } from "./auto-update.js"; import { openBrowserAuth } from "./browser-auth.js"; +import { BrowserPreviewController } from "./browser-preview.js"; import { DOCKER_INSTALL_LINKS, isDesktopSetupLink, runDocker } from "./docker-cli.js"; import { requestLocalSettings } from "./local-settings.js"; import { @@ -1130,6 +1131,35 @@ app.whenReady().then(async () => { }, ); ipcMain.handle("desktop.platform", () => process.platform); + const browserPreview = new BrowserPreviewController(); + // The in-app browser answers only the app window's own top frame, never a page inside it. + const previewHost = (event: Electron.IpcMainInvokeEvent) => { + if (!fromMainWindow(event) || event.senderFrame !== event.sender.mainFrame) { + throw new Error("The in-app browser is not available here."); + } + return mainWindow!; + }; + ipcMain.handle("desktop.browserPreview.state", (event) => + browserPreview.state(previewHost(event)), + ); + ipcMain.handle("desktop.browserPreview.show", (event, bounds: unknown) => + browserPreview.show(previewHost(event), bounds), + ); + ipcMain.handle("desktop.browserPreview.hide", (event) => browserPreview.hide(previewHost(event))); + ipcMain.handle("desktop.browserPreview.navigate", (event, url: unknown) => + browserPreview.navigate(previewHost(event), url), + ); + ipcMain.handle("desktop.browserPreview.back", (event) => browserPreview.back(previewHost(event))); + ipcMain.handle("desktop.browserPreview.forward", (event) => + browserPreview.forward(previewHost(event)), + ); + ipcMain.handle("desktop.browserPreview.reload", (event) => + browserPreview.reload(previewHost(event)), + ); + ipcMain.handle("desktop.browserPreview.stop", (event) => browserPreview.stop(previewHost(event))); + ipcMain.handle("desktop.browserPreview.openDevTools", (event) => + browserPreview.openDevTools(previewHost(event)), + ); ipcMain.handle("desktop.files.pickDirectory", async (event) => { if (!fromMainWindow(event)) return null; const result = await dialog.showOpenDialog(windowFrom(event)!, { diff --git a/apps/desktop/src/preload.cjs b/apps/desktop/src/preload.cjs index 14e5c50..7bc2c7b 100644 --- a/apps/desktop/src/preload.cjs +++ b/apps/desktop/src/preload.cjs @@ -21,6 +21,22 @@ contextBridge.exposeInMainWorld("rakazoDesktop", { download: () => ipcRenderer.invoke("desktop.update.download"), install: () => ipcRenderer.invoke("desktop.update.install"), }, + browserPreview: { + state: () => ipcRenderer.invoke("desktop.browserPreview.state"), + show: (bounds) => ipcRenderer.invoke("desktop.browserPreview.show", bounds), + hide: () => ipcRenderer.invoke("desktop.browserPreview.hide"), + navigate: (url) => ipcRenderer.invoke("desktop.browserPreview.navigate", url), + back: () => ipcRenderer.invoke("desktop.browserPreview.back"), + forward: () => ipcRenderer.invoke("desktop.browserPreview.forward"), + reload: () => ipcRenderer.invoke("desktop.browserPreview.reload"), + stop: () => ipcRenderer.invoke("desktop.browserPreview.stop"), + openDevTools: () => ipcRenderer.invoke("desktop.browserPreview.openDevTools"), + onState: (listener) => { + const handler = (_event, state) => listener(state); + ipcRenderer.on("desktop.browserPreview.state", handler); + return () => ipcRenderer.off("desktop.browserPreview.state", handler); + }, + }, oauth: { open: (url) => ipcRenderer.invoke("desktop.oauth.open", url), cancel: (url) => ipcRenderer.invoke("desktop.oauth.cancel", url), diff --git a/apps/desktop/src/preload.test.ts b/apps/desktop/src/preload.test.ts index f5891c0..9e44a20 100644 --- a/apps/desktop/src/preload.test.ts +++ b/apps/desktop/src/preload.test.ts @@ -32,6 +32,7 @@ describe("desktop preload bridge", () => { expect(globalName).toBe("rakazoDesktop"); expect(bridge.platform).toBe("linux"); expect(Object.keys(bridge).sort()).toEqual([ + "browserPreview", "files", "localSettings", "oauth", @@ -46,6 +47,18 @@ describe("desktop preload bridge", () => { "toggleMaximize", ]); expect(Object.keys(bridge.update).sort()).toEqual(["check", "download", "install", "state"]); + expect(Object.keys(bridge.browserPreview ?? {}).sort()).toEqual([ + "back", + "forward", + "hide", + "navigate", + "onState", + "openDevTools", + "reload", + "show", + "state", + "stop", + ]); await bridge.files?.pickDirectory(); await bridge.oauth.open?.("https://provider.example.com/authorize"); @@ -58,6 +71,8 @@ describe("desktop preload bridge", () => { await bridge.update.check(); await bridge.update.download(); await bridge.update.install(); + await bridge.browserPreview?.navigate("http://localhost:5173/"); + await bridge.browserPreview?.show({ x: 1, y: 2, width: 3, height: 4 }); expect(invoke.mock.calls.map(([channel]) => channel)).toEqual([ "desktop.files.pickDirectory", "desktop.oauth.open", @@ -70,6 +85,8 @@ describe("desktop preload bridge", () => { "desktop.update.check", "desktop.update.download", "desktop.update.install", + "desktop.browserPreview.navigate", + "desktop.browserPreview.show", ]); }); @@ -77,6 +94,7 @@ describe("desktop preload bridge", () => { const { exposeInMainWorld } = runPreload("preload.cjs"); const [, bridge] = exposeInMainWorld.mock.calls[0] as [string, Record]; expect(Object.keys(bridge).sort()).toEqual([ + "browserPreview", "files", "localSettings", "oauth", diff --git a/apps/mobile/lib/api.ts b/apps/mobile/lib/api.ts index ac39cf1..2e17b0f 100644 --- a/apps/mobile/lib/api.ts +++ b/apps/mobile/lib/api.ts @@ -1096,7 +1096,7 @@ export function applyMobileThreadEvent( messages: [...remaining, streaming], }; } - if (event.type === "agent.tool.completed") { + if (event.type === "agent.tool.completed" || event.type === "thread.computer") { return { ...prev, cursor: event.seq ?? prev.cursor }; } if (event.type === "thread.subagent") { diff --git a/apps/web/e2e/agent-cursor.spec.ts b/apps/web/e2e/agent-cursor.spec.ts new file mode 100644 index 0000000..052a821 --- /dev/null +++ b/apps/web/e2e/agent-cursor.spec.ts @@ -0,0 +1,53 @@ +import { expect, test } from "@playwright/test"; +import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers"; + +const SCREEN_URL = "https://screen.example/embed.html"; + +test("the bot's cursor glides to where it clicks on its live screen", async ({ + page, +}, testInfo) => { + await signup(page, `agent-cursor-${Date.now()}@rakazo.test`, "password12", "Cursor Watcher"); + await completeOnboarding(page); + const botId = activeBotId(page); + await rpc(page, "computer/boot", { botId }); + // Stand in for the noVNC viewer: a 16:10 desktop with a field in the middle. + await page.route("https://screen.example/**", (route) => + route.fulfill({ + contentType: "text/html", + body: `Desktop + `, + }), + ); + await page.route("**/rpc/computer/screenUrl", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { url: SCREEN_URL } }), + }), + ); + + await page.getByTitle("Agent computer").click(); + const preview = page.getByTestId("computer-preview"); + await expect(preview.locator("iframe")).toHaveAttribute("src", SCREEN_URL); + const cursor = preview.getByTestId("agent-cursor"); + await expect(cursor).toHaveCSS("opacity", "0"); + + // The scripted runtime clicks the screen center (640, 400) and types. + await rpc(page, "threads/send", { botId, text: "use your screen and type cursor-demo" }); + + await expect(cursor).toHaveCSS("opacity", "1"); + await expect(cursor).toHaveAttribute("data-action", /click|type/); + const layer = (await preview.getByTestId("agent-cursor-layer").boundingBox())!; + await expect + .poll(async () => { + const tip = (await cursor.boundingBox())!; + return [ + Math.round(tip.x - layer.x - layer.width / 2), + Math.round(tip.y - layer.y - layer.height / 2), + ]; + }) + .toEqual([0, 0]); + await captureScreenshot(page, testInfo, "agent-cursor-on-screen"); + + // It fades once the bot stops pointing, so it never lingers over the screen. + await expect(cursor).toHaveCSS("opacity", "0", { timeout: 10_000 }); +}); diff --git a/apps/web/e2e/in-app-browser.spec.ts b/apps/web/e2e/in-app-browser.spec.ts new file mode 100644 index 0000000..1862edb --- /dev/null +++ b/apps/web/e2e/in-app-browser.spec.ts @@ -0,0 +1,159 @@ +import { expect, test } from "@playwright/test"; +import { captureScreenshot, completeOnboarding, signup } from "./helpers"; + +test("the in-app browser previews a page and follows it back to the browser", async ({ + page, +}, testInfo) => { + await signup(page, `in-app-browser-${Date.now()}@rakazo.test`, "password12", "Previewer"); + await completeOnboarding(page); + // Context-wide so the "Open in browser" tab is served too. + await page.context().route("https://preview.example/**", (route) => + route.fulfill({ + contentType: "text/html", + body: `Preview +

Live preview

Rendered inside the app.

`, + }), + ); + + await page.getByRole("button", { name: "Browser" }).click(); + const browser = page.getByTestId("in-app-browser"); + const address = browser.getByRole("textbox", { name: "Address" }); + await expect(address).toBeFocused(); + + await address.fill("not an address"); + await address.press("Enter"); + await expect(address).toHaveAttribute("aria-invalid", "true"); + + await address.fill("preview.example/app"); + await address.press("Enter"); + await expect(address).toHaveValue("https://preview.example/app"); + const frame = browser.getByTestId("in-app-browser-frame"); + await expect(frame).toHaveAttribute("src", "https://preview.example/app"); + await expect( + page.frameLocator('[data-testid="in-app-browser-frame"]').getByRole("heading"), + ).toHaveText("Live preview"); + await captureScreenshot(page, testInfo, "in-app-browser"); + + const popup = page.waitForEvent("popup"); + await browser.getByRole("button", { name: "Open in browser" }).click(); + expect((await popup).url()).toBe("https://preview.example/app"); + + await browser.getByRole("button", { name: "Close panel" }).click(); + await expect(browser).toHaveCount(0); + // The last address comes back when the browser reopens. + await page.getByRole("button", { name: "Browser" }).click(); + await expect(page.getByTestId("in-app-browser-frame")).toHaveAttribute( + "src", + "https://preview.example/app", + ); +}); + +test("on desktop the page is a native view glued to the panel", async ({ page }) => { + await page.addInitScript(() => { + const calls: Array<[string, unknown?]> = []; + let listener: ((state: unknown) => void) | undefined; + let state = { + url: "", + title: "", + loading: false, + canGoBack: false, + canGoForward: false, + error: null as string | null, + }; + const record = (name: string) => async (value?: unknown) => { + calls.push([name, value]); + }; + Object.assign(window, { __browserCalls: calls }); + Object.defineProperty(window, "rakazoDesktop", { + value: { + platform: "linux", + window: { + close: async () => {}, + minimize: async () => {}, + toggleMaximize: async () => {}, + state: async () => ({ minimized: false, maximized: false, fullScreen: false }), + }, + oauth: { onCallback: () => () => {} }, + update: { + state: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }), + check: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }), + download: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }), + install: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }), + }, + browserPreview: { + state: async () => state, + onState: (next: (value: unknown) => void) => { + listener = next; + return () => { + listener = undefined; + }; + }, + navigate: async (url: string) => { + calls.push(["navigate", url]); + state = { ...state, url, title: "Dev server", loading: false, canGoBack: true }; + listener?.(state); + }, + show: record("show"), + hide: record("hide"), + back: record("back"), + forward: record("forward"), + reload: record("reload"), + stop: record("stop"), + openDevTools: record("openDevTools"), + }, + }, + }); + }); + await signup(page, `native-browser-${Date.now()}@rakazo.test`, "password12", "Desktop"); + await completeOnboarding(page); + const calls = () => + page.evaluate( + () => (window as unknown as { __browserCalls: Array<[string, unknown?]> }).__browserCalls, + ); + + await page.getByRole("button", { name: "Browser" }).click(); + const browser = page.getByTestId("in-app-browser"); + await browser.getByRole("textbox", { name: "Address" }).fill("localhost:5173"); + await browser.getByRole("textbox", { name: "Address" }).press("Enter"); + await expect.poll(calls).toContainEqual(["navigate", "http://localhost:5173/"]); + await expect(browser.getByRole("button", { name: "Back" })).toBeEnabled(); + await expect(browser.getByRole("button", { name: "Forward" })).toBeDisabled(); + await expect(browser.getByTestId("in-app-browser-frame")).toHaveCount(0); + + // The reserved page area and the native view's bounds stay identical. + const viewport = browser.getByTestId("in-app-browser-viewport"); + await expect + .poll(async () => { + const box = (await viewport.boundingBox())!; + const shown = (await calls()).filter(([name]) => name === "show").at(-1)?.[1]; + return ( + JSON.stringify(shown) === + JSON.stringify({ + x: Math.round(box.x), + y: Math.round(box.y), + width: Math.round(box.width), + height: Math.round(box.height), + }) + ); + }) + .toBe(true); + + await browser.getByRole("button", { name: "Back" }).click(); + await browser.getByRole("button", { name: "Reload" }).click(); + await browser.getByRole("button", { name: "Developer tools" }).click(); + await expect + .poll(async () => (await calls()).map(([name]) => name)) + .toEqual(expect.arrayContaining(["back", "reload", "openDevTools"])); + + // App menus float above the page, so the native view steps aside while one is open. + const before = (await calls()).length; + await page.getByTestId("create-menu-trigger").click(); + await expect + .poll(async () => (await calls()).slice(before).map(([name]) => name)) + .toContain("hide"); + await page.keyboard.press("Escape"); + await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("show"); + + await browser.getByRole("button", { name: "Close panel" }).click(); + await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("hide"); +}); diff --git a/apps/web/e2e/page-browser.spec.ts b/apps/web/e2e/page-browser.spec.ts index 7354efc..2ffb90c 100644 --- a/apps/web/e2e/page-browser.spec.ts +++ b/apps/web/e2e/page-browser.spec.ts @@ -50,6 +50,16 @@ test("live page helper preserves identity, masks passwords, and reports partial expect(snapshot.title).toBe("Fixture"); expect(snapshot.tree).toContain("Readable article content"); expect(JSON.stringify(snapshot)).not.toContain("fake-test-password"); + // Element boxes are screen pixels for the visible bot cursor. + expect(snapshot.screen.width).toBeGreaterThan(0); + expect(snapshot.screen.height).toBeGreaterThan(0); + const saveBox = snapshot.elements.find((element: { name: string }) => element.name === "Save") + .box as number[]; + const saveRect = (await page.locator("#save").boundingBox())!; + expect(saveBox[2]).toBe(Math.round(saveRect.width)); + expect(saveBox[3]).toBe(Math.round(saveRect.height)); + expect(saveBox[0]).toBeGreaterThanOrEqual(Math.round(saveRect.x)); + expect(saveBox[1]).toBeGreaterThanOrEqual(Math.round(saveRect.y)); const ref = (snapshot: { elements: { name: string; ref: string }[] }, name: string) => snapshot.elements.find((element) => element.name === name)!.ref; const result = await command("act", { diff --git a/apps/web/src/components/browser/InAppBrowser.tsx b/apps/web/src/components/browser/InAppBrowser.tsx new file mode 100644 index 0000000..22ac61a --- /dev/null +++ b/apps/web/src/components/browser/InAppBrowser.tsx @@ -0,0 +1,317 @@ +import { useLingui } from "@lingui/react/macro"; +import type { + RakazoDesktopBrowserPreview, + RakazoDesktopBrowserPreviewState, +} from "@rakazo/contracts"; +import { Button, cn, Input } from "@rakazo/ui-web"; +import { ArrowLeft, ArrowRight, Code, ExternalLink, RotateCw, X } from "lucide-react"; +import { type FormEvent, useCallback, useEffect, useRef, useState } from "react"; +import { browserFrameSandbox, normalizeBrowserUrl } from "../../lib/browser-url"; +import { desktopBridge } from "../../lib/desktop"; + +const LAST_URL_KEY = "rk:in-app-browser:url"; + +function readLastUrl(): string { + try { + return window.localStorage.getItem(LAST_URL_KEY) ?? ""; + } catch { + return ""; + } +} + +function rememberUrl(url: string) { + try { + window.localStorage.setItem(LAST_URL_KEY, url); + } catch { + /* A per-viewer convenience only. */ + } +} + +const EMPTY_STATE: RakazoDesktopBrowserPreviewState = { + url: "", + title: "", + loading: false, + canGoBack: false, + canGoForward: false, + error: null, +}; + +/** DOM layers the native desktop view would otherwise paint over. */ +const FLOATING_UI = '[role="dialog"],[role="alertdialog"],[role="menu"],[role="listbox"]'; + +/** + * Address bar plus page. On desktop the page is a native Chromium view laid over the reserved + * area, so dev servers keep hot reload, DevTools and full speed; in a browser it is an iframe. + */ +export function InAppBrowser({ onClose }: { onClose: () => void }) { + const { t } = useLingui(); + const native = desktopBridge()?.browserPreview; + const [state, setState] = useState(() => ({ + ...EMPTY_STATE, + url: native ? "" : readLastUrl(), + })); + const [draft, setDraft] = useState(state.url); + const [invalid, setInvalid] = useState(false); + const [frameKey, setFrameKey] = useState(0); + const editing = useRef(false); + const inputRef = useRef(null); + + useEffect(() => { + if (!native) return; + let live = true; + const apply = (next: RakazoDesktopBrowserPreviewState) => { + if (!live) return; + setState(next); + if (!editing.current) setDraft(next.url); + }; + const unsubscribe = native.onState(apply); + void native + .state() + .then((current) => { + apply(current); + const last = readLastUrl(); + if (!current.url && last) void native.navigate(last).catch(() => undefined); + }) + .catch(() => undefined); + return () => { + live = false; + unsubscribe(); + }; + }, [native]); + + useEffect(() => { + if (!state.url) inputRef.current?.focus(); + }, [state.url]); + + const go = useCallback( + (input: string) => { + const url = normalizeBrowserUrl(input); + setInvalid(!url); + if (!url) return; + editing.current = false; + setDraft(url); + rememberUrl(url); + if (native) { + void native.navigate(url).catch(() => setInvalid(true)); + } else { + setState((current) => ({ ...current, url, loading: true, error: null })); + setFrameKey((key) => key + 1); + } + }, + [native], + ); + + const reload = useCallback(() => { + if (native) { + void (state.loading ? native.stop() : native.reload()).catch(() => undefined); + return; + } + if (!state.url) return; + setState((current) => ({ ...current, loading: true })); + setFrameKey((key) => key + 1); + }, [native, state.loading, state.url]); + + const onSubmit = (event: FormEvent) => { + event.preventDefault(); + go(draft); + inputRef.current?.blur(); + }; + + return ( +
+
+ {native ? ( + <> + + + + ) : null} + + { + editing.current = true; + event.currentTarget.select(); + }} + onBlur={() => { + editing.current = false; + }} + onChange={(event) => { + setDraft(event.target.value); + setInvalid(false); + }} + onKeyDown={(event) => { + if (event.key !== "Escape") return; + setDraft(state.url); + setInvalid(false); + event.currentTarget.blur(); + }} + className="mx-1 h-control-sm flex-1 rounded-full bg-muted/60 px-3 text-ui-body-sm" + /> + {native ? ( + + ) : null} + + +
+
+
+ +
+ {native ? ( + + ) : state.url ? ( +