diff --git a/apps/desktop/e2e/browser-preview.spec.ts b/apps/desktop/e2e/browser-preview.spec.ts
new file mode 100644
index 0000000..b94f901
--- /dev/null
+++ b/apps/desktop/e2e/browser-preview.spec.ts
@@ -0,0 +1,155 @@
+import http from "node:http";
+import path from "node:path";
+import { _electron as electron, expect, test } from "@playwright/test";
+import type { RakazoDesktop, RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
+
+const fixture = `
+
+
Rakazo desktop browser
+ App with a browser panel
+`;
+
+type Bridge = NonNullable;
+
+test("the in-app browser is a native view the app window lays over its panel", async () => {
+ const pages = http.createServer((req, res) => {
+ res.setHeader("content-type", "text/html");
+ if (req.url === "/second") {
+ res.end("Second pagesecond
");
+ return;
+ }
+ res.end(`First page
+ next
+
+ file`);
+ });
+ await new Promise((resolve) => pages.listen(0, "127.0.0.1", resolve));
+ const address = pages.address();
+ if (!address || typeof address === "string") throw new Error("No fixture server");
+ const origin = `http://127.0.0.1:${address.port}`;
+
+ const app = await electron.launch({
+ args: ["."],
+ cwd: path.resolve(import.meta.dirname, ".."),
+ env: {
+ ...process.env,
+ RAKAZO_WEB_URL: `data:text/html;charset=utf-8,${encodeURIComponent(fixture)}`,
+ },
+ });
+
+ try {
+ const page = await app.firstWindow();
+ await expect(page.getByText("App with a browser panel")).toBeVisible();
+
+ const call = (method: K, ...args: unknown[]) =>
+ page.evaluate(
+ ([name, values]) => {
+ const bridge = (window as typeof window & { rakazoDesktop?: RakazoDesktop }).rakazoDesktop
+ ?.browserPreview as unknown as Record<
+ string,
+ (...input: unknown[]) => Promise
+ >;
+ return bridge[name as string]!(...(values as unknown[]));
+ },
+ [method, args] as const,
+ );
+ const state = () => call("state") as Promise;
+ const view = () =>
+ app.evaluate(({ BrowserWindow }) => {
+ const win = BrowserWindow.getAllWindows()[0]!;
+ const child = win.contentView.children[0] as
+ | (Electron.View & { webContents?: Electron.WebContents })
+ | undefined;
+ return child
+ ? {
+ count: win.contentView.children.length,
+ visible: child.getVisible(),
+ bounds: child.getBounds(),
+ url: child.webContents?.getURL(),
+ partition: child.webContents?.session === win.webContents.session,
+ sandbox: child.webContents?.getLastWebPreferences()?.sandbox,
+ nodeIntegration: child.webContents?.getLastWebPreferences()?.nodeIntegration,
+ }
+ : null;
+ });
+
+ // Pages load in a native view in their own session, never the app's.
+ await call("navigate", `${origin}/`);
+ await expect.poll(async () => (await state()).title).toBe("First page");
+ expect(await view()).toMatchObject({
+ count: 1,
+ visible: false,
+ url: `${origin}/`,
+ partition: false,
+ sandbox: true,
+ nodeIntegration: false,
+ });
+
+ await call("show", { x: 400, y: 50, width: 300, height: 200 });
+ expect(await view()).toMatchObject({
+ visible: true,
+ bounds: { x: 400, y: 50, width: 300, height: 200 },
+ });
+ // Renderer bounds outside the window are clamped to it.
+ await call("show", { x: -10, y: -10, width: 100_000, height: 100_000 });
+ const [width, height] = await app.evaluate(({ BrowserWindow }) =>
+ BrowserWindow.getAllWindows()[0]!.getContentSize(),
+ );
+ expect((await view())?.bounds).toEqual({ x: 0, y: 0, width, height });
+
+ const contents = () =>
+ app.evaluate(({ BrowserWindow }, script) => {
+ const win = BrowserWindow.getAllWindows()[0]!;
+ const child = win.contentView.children[0] as Electron.View & {
+ webContents: Electron.WebContents;
+ };
+ return child.webContents.executeJavaScript(script, true);
+ }, "document.getElementById('next').click()");
+ await contents();
+ await expect.poll(async () => (await state()).title).toBe("Second page");
+ expect(await state()).toMatchObject({ canGoBack: true, canGoForward: false, error: null });
+
+ await call("back");
+ await expect.poll(async () => (await state()).title).toBe("First page");
+ expect(await state()).toMatchObject({ canGoForward: true });
+ await call("forward");
+ await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
+
+ // New-window links stay in the one view; file links never load.
+ await call("back");
+ await expect.poll(async () => (await state()).title).toBe("First page");
+ await app.evaluate(({ BrowserWindow }) => {
+ const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
+ webContents: Electron.WebContents;
+ };
+ return child.webContents.executeJavaScript("document.getElementById('popup').click()", true);
+ });
+ await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
+ expect(await app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows().length)).toBe(1);
+ await call("back");
+ await expect.poll(async () => (await state()).title).toBe("First page");
+ await app.evaluate(({ BrowserWindow }) => {
+ const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
+ webContents: Electron.WebContents;
+ };
+ return child.webContents.executeJavaScript("document.getElementById('file').click()", true);
+ });
+ await page.waitForTimeout(300);
+ expect((await state()).url).toBe(`${origin}/`);
+
+ await expect(call("navigate", "file:///etc/hosts")).rejects.toThrow(/http/);
+
+ // A dev server that is not running yet shows as an error, then recovers on reload.
+ pages.close();
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ await call("navigate", `${origin}/gone`);
+ await expect.poll(async () => (await state()).error).toMatch(/ERR_CONNECTION_REFUSED/);
+
+ await call("hide");
+ expect((await view())?.visible).toBe(false);
+ } finally {
+ await app.close();
+ pages.closeAllConnections();
+ pages.close();
+ }
+});
diff --git a/apps/desktop/e2e/smoke.spec.ts b/apps/desktop/e2e/smoke.spec.ts
index dcd8ae4..62ddcf2 100644
--- a/apps/desktop/e2e/smoke.spec.ts
+++ b/apps/desktop/e2e/smoke.spec.ts
@@ -42,6 +42,7 @@ test("launches with a narrow preload bridge and an isolated renderer", async ()
});
expect(renderer.bridgeKeys).toEqual([
+ "browserPreview",
"files",
"localSettings",
"oauth",
diff --git a/apps/desktop/src/browser-preview-policy.test.ts b/apps/desktop/src/browser-preview-policy.test.ts
new file mode 100644
index 0000000..b50fb32
--- /dev/null
+++ b/apps/desktop/src/browser-preview-policy.test.ts
@@ -0,0 +1,96 @@
+import { describe, expect, it } from "vitest";
+import {
+ previewBounds,
+ previewLoadError,
+ previewShortcut,
+ previewUrl,
+} from "./browser-preview-policy.js";
+
+describe("previewUrl", () => {
+ it("accepts only http(s) without credentials", () => {
+ expect(previewUrl("http://localhost:5173")).toBe("http://localhost:5173/");
+ expect(previewUrl("https://example.com/a?b=1")).toBe("https://example.com/a?b=1");
+ for (const value of [
+ "file:///etc/hosts",
+ "javascript:alert(1)",
+ "chrome://settings",
+ "https://user:fake-password@example.com",
+ "not a url",
+ 42,
+ `https://example.com/${"a".repeat(9000)}`,
+ ]) {
+ expect(previewUrl(value)).toBeNull();
+ }
+ });
+});
+
+describe("previewBounds", () => {
+ const content = { width: 1200, height: 800 };
+
+ it("rounds and applies the app zoom factor", () => {
+ expect(previewBounds({ x: 800.4, y: 40.6, width: 399.5, height: 700 }, content)).toEqual({
+ x: 800,
+ y: 41,
+ width: 400,
+ height: 700,
+ });
+ expect(previewBounds({ x: 100, y: 10, width: 200, height: 100 }, content, 1.25)).toEqual({
+ x: 125,
+ y: 13,
+ width: 250,
+ height: 125,
+ });
+ });
+
+ it("clamps to the window and rejects empty or malformed rectangles", () => {
+ expect(previewBounds({ x: -50, y: -10, width: 5000, height: 5000 }, content)).toEqual({
+ x: 0,
+ y: 0,
+ width: 1200,
+ height: 800,
+ });
+ expect(previewBounds({ x: 1300, y: 0, width: 100, height: 100 }, content)).toBeNull();
+ expect(previewBounds({ x: 0, y: 0, width: 0, height: 100 }, content)).toBeNull();
+ expect(previewBounds({ x: "1", y: 0, width: 1, height: 1 }, content)).toBeNull();
+ expect(previewBounds({ x: Number.NaN, y: 0, width: 1, height: 1 }, content)).toBeNull();
+ expect(previewBounds(null, content)).toBeNull();
+ });
+});
+
+describe("previewShortcut", () => {
+ const key = (overrides: Partial[0]>) => ({
+ type: "keyDown",
+ key: "r",
+ control: false,
+ meta: false,
+ alt: false,
+ shift: false,
+ ...overrides,
+ });
+
+ it("maps platform browser shortcuts", () => {
+ expect(previewShortcut(key({ meta: true }), "darwin")).toBe("reload");
+ expect(previewShortcut(key({ meta: true, shift: true, key: "R" }), "darwin")).toBe(
+ "hard-reload",
+ );
+ expect(previewShortcut(key({ control: true }), "linux")).toBe("reload");
+ expect(previewShortcut(key({ meta: true, key: "[" }), "darwin")).toBe("back");
+ expect(previewShortcut(key({ alt: true, key: "ArrowRight" }), "win32")).toBe("forward");
+ expect(previewShortcut(key({ key: "F12" }), "linux")).toBe("devtools");
+ expect(previewShortcut(key({ key: "F5" }), "win32")).toBe("reload");
+ });
+
+ it("leaves typing alone", () => {
+ expect(previewShortcut(key({}), "darwin")).toBeNull();
+ expect(previewShortcut(key({ control: true }), "darwin")).toBeNull();
+ expect(previewShortcut(key({ type: "keyUp", meta: true }), "darwin")).toBeNull();
+ expect(previewShortcut(key({ key: "ArrowLeft" }), "linux")).toBeNull();
+ });
+});
+
+describe("previewLoadError", () => {
+ it("ignores superseded navigations", () => {
+ expect(previewLoadError(-3, "ERR_ABORTED")).toBeNull();
+ expect(previewLoadError(-102, "ERR_CONNECTION_REFUSED")).toBe("ERR_CONNECTION_REFUSED");
+ });
+});
diff --git a/apps/desktop/src/browser-preview-policy.ts b/apps/desktop/src/browser-preview-policy.ts
new file mode 100644
index 0000000..c598abd
--- /dev/null
+++ b/apps/desktop/src/browser-preview-policy.ts
@@ -0,0 +1,76 @@
+import type { RakazoDesktopBrowserBounds } from "@rakazo/contracts";
+
+/** Only plain http(s) pages load in the in-app browser; no files, schemes or URL credentials. */
+export function previewUrl(value: unknown): string | null {
+ if (typeof value !== "string" || value.length > 8192) return null;
+ try {
+ const url = new URL(value);
+ if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+ if (url.username || url.password) return null;
+ return url.toString();
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Renderer-reported CSS-pixel bounds, converted to window DIPs and clamped to the content area
+ * so a compromised renderer cannot place the view outside the panel it reserved.
+ */
+export function previewBounds(
+ value: unknown,
+ content: { width: number; height: number },
+ zoomFactor = 1,
+): RakazoDesktopBrowserBounds | null {
+ if (!value || typeof value !== "object") return null;
+ const raw = value as Record;
+ const numbers = [raw.x, raw.y, raw.width, raw.height];
+ if (!numbers.every((n) => typeof n === "number" && Number.isFinite(n))) return null;
+ const scale = Number.isFinite(zoomFactor) && zoomFactor > 0 ? zoomFactor : 1;
+ const [x, y, width, height] = (numbers as number[]).map((n) => Math.round(n * scale)) as [
+ number,
+ number,
+ number,
+ number,
+ ];
+ const left = Math.min(Math.max(0, x), content.width);
+ const top = Math.min(Math.max(0, y), content.height);
+ const right = Math.min(Math.max(left, x + width), content.width);
+ const bottom = Math.min(Math.max(top, y + height), content.height);
+ if (right - left < 1 || bottom - top < 1) return null;
+ return { x: left, y: top, width: right - left, height: bottom - top };
+}
+
+export type PreviewShortcut = "reload" | "hard-reload" | "back" | "forward" | "devtools";
+
+/** Browser keyboard shortcuts while focus is inside the page. */
+export function previewShortcut(
+ input: {
+ type: string;
+ key: string;
+ control: boolean;
+ meta: boolean;
+ alt: boolean;
+ shift: boolean;
+ },
+ platform: string,
+): PreviewShortcut | null {
+ if (input.type !== "keyDown") return null;
+ const mod = platform === "darwin" ? input.meta : input.control;
+ const key = input.key.length === 1 ? input.key.toLowerCase() : input.key;
+ if (key === "F5") return input.shift ? "hard-reload" : "reload";
+ if (key === "F12") return "devtools";
+ if (mod && key === "r") return input.shift ? "hard-reload" : "reload";
+ if (mod && input.alt && key === "i") return "devtools";
+ if (platform === "darwin" ? input.meta && !input.alt : input.alt && !input.control) {
+ if (key === (platform === "darwin" ? "[" : "ArrowLeft")) return "back";
+ if (key === (platform === "darwin" ? "]" : "ArrowRight")) return "forward";
+ }
+ return null;
+}
+
+/** Load failures worth showing; an aborted load (-3) is a superseded navigation, not an error. */
+export function previewLoadError(errorCode: number, description: string): string | null {
+ if (errorCode === -3 || errorCode === 0) return null;
+ return description || `Load failed (${errorCode})`;
+}
diff --git a/apps/desktop/src/browser-preview.ts b/apps/desktop/src/browser-preview.ts
new file mode 100644
index 0000000..7d4c908
--- /dev/null
+++ b/apps/desktop/src/browser-preview.ts
@@ -0,0 +1,189 @@
+import type { RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
+import { type BrowserWindow, WebContentsView } from "electron";
+import {
+ previewBounds,
+ previewLoadError,
+ previewShortcut,
+ previewUrl,
+} from "./browser-preview-policy.js";
+
+/** Pages share one persistent profile, separate from the app's own session and cookies. */
+const PREVIEW_PARTITION = "persist:rakazo-browser";
+export const PREVIEW_STATE_CHANNEL = "desktop.browserPreview.state";
+
+/**
+ * The in-app browser's native page: one WebContentsView laid over the panel the app window
+ * reserves. It belongs to one app window at a time and is torn down with it.
+ */
+export class BrowserPreviewController {
+ private host: BrowserWindow | null = null;
+ private view: WebContentsView | null = null;
+ private error: string | null = null;
+
+ state(win: BrowserWindow): RakazoDesktopBrowserPreviewState {
+ const contents = this.host === win ? this.view?.webContents : undefined;
+ if (!contents || contents.isDestroyed()) {
+ return {
+ url: "",
+ title: "",
+ loading: false,
+ canGoBack: false,
+ canGoForward: false,
+ error: null,
+ };
+ }
+ const url = contents.getURL();
+ return {
+ url,
+ title: contents.getTitle() === url ? "" : contents.getTitle(),
+ loading: contents.isLoading(),
+ canGoBack: contents.navigationHistory.canGoBack(),
+ canGoForward: contents.navigationHistory.canGoForward(),
+ error: this.error,
+ };
+ }
+
+ show(win: BrowserWindow, rawBounds: unknown) {
+ const view = this.viewFor(win);
+ const [width, height] = win.getContentSize();
+ const bounds = previewBounds(
+ rawBounds,
+ { width: width ?? 0, height: height ?? 0 },
+ win.webContents.getZoomFactor(),
+ );
+ if (!bounds) {
+ view.setVisible(false);
+ return;
+ }
+ view.setBounds(bounds);
+ view.setVisible(true);
+ }
+
+ hide(win: BrowserWindow) {
+ if (this.host === win) this.view?.setVisible(false);
+ }
+
+ async navigate(win: BrowserWindow, rawUrl: unknown) {
+ const url = previewUrl(rawUrl);
+ if (!url) throw new Error("Only http(s) pages can open here.");
+ this.error = null;
+ // Failures surface through did-fail-load as page state, not as a rejected call.
+ await this.viewFor(win)
+ .webContents.loadURL(url)
+ .catch(() => undefined);
+ }
+
+ back(win: BrowserWindow) {
+ const history = this.contentsFor(win)?.navigationHistory;
+ if (history?.canGoBack()) history.goBack();
+ }
+
+ forward(win: BrowserWindow) {
+ const history = this.contentsFor(win)?.navigationHistory;
+ if (history?.canGoForward()) history.goForward();
+ }
+
+ reload(win: BrowserWindow, ignoreCache = false) {
+ const contents = this.contentsFor(win);
+ if (!contents?.getURL()) return;
+ this.error = null;
+ if (ignoreCache) contents.reloadIgnoringCache();
+ else contents.reload();
+ }
+
+ stop(win: BrowserWindow) {
+ this.contentsFor(win)?.stop();
+ }
+
+ openDevTools(win: BrowserWindow) {
+ const contents = this.contentsFor(win);
+ if (contents?.getURL()) contents.openDevTools({ mode: "detach" });
+ }
+
+ private contentsFor(win: BrowserWindow) {
+ const contents = this.host === win ? this.view?.webContents : undefined;
+ return contents && !contents.isDestroyed() ? contents : undefined;
+ }
+
+ private viewFor(win: BrowserWindow): WebContentsView {
+ if (this.host === win && this.view && !this.view.webContents.isDestroyed()) return this.view;
+ this.destroy();
+ const view = new WebContentsView({
+ webPreferences: {
+ partition: PREVIEW_PARTITION,
+ sandbox: true,
+ contextIsolation: true,
+ nodeIntegration: false,
+ webSecurity: true,
+ spellcheck: false,
+ },
+ });
+ view.setVisible(false);
+ const contents = view.webContents;
+ const publish = () => {
+ if (!win.isDestroyed()) win.webContents.send(PREVIEW_STATE_CHANNEL, this.state(win));
+ };
+ // One tab: links that want a new window open here instead.
+ contents.setWindowOpenHandler(({ url }) => {
+ const target = previewUrl(url);
+ if (target) void contents.loadURL(target).catch(() => undefined);
+ return { action: "deny" };
+ });
+ contents.on("will-navigate", (event, url) => {
+ if (!previewUrl(url)) event.preventDefault();
+ });
+ contents.on("will-redirect", (event, url) => {
+ if (!previewUrl(url)) event.preventDefault();
+ });
+ contents.on("did-start-loading", publish);
+ contents.on("did-stop-loading", publish);
+ contents.on("did-navigate", () => {
+ this.error = null;
+ publish();
+ });
+ contents.on("did-navigate-in-page", publish);
+ contents.on("page-title-updated", publish);
+ contents.on("did-fail-load", (_event, code, description, _url, isMainFrame) => {
+ if (!isMainFrame) return;
+ const error = previewLoadError(code, description);
+ if (error === null) return;
+ this.error = error;
+ publish();
+ });
+ contents.on("render-process-gone", (_event, details) => {
+ this.error = details.reason === "clean-exit" ? null : "The page stopped responding.";
+ publish();
+ });
+ contents.on("before-input-event", (event, input) => {
+ const shortcut = previewShortcut(input, process.platform);
+ if (!shortcut) return;
+ event.preventDefault();
+ if (shortcut === "reload" || shortcut === "hard-reload") {
+ this.reload(win, shortcut === "hard-reload");
+ } else if (shortcut === "back") {
+ this.back(win);
+ } else if (shortcut === "forward") {
+ this.forward(win);
+ } else {
+ this.openDevTools(win);
+ }
+ });
+ win.contentView.addChildView(view);
+ win.once("closed", () => {
+ if (this.host === win) this.destroy();
+ });
+ this.host = win;
+ this.view = view;
+ return view;
+ }
+
+ private destroy() {
+ const { host, view } = this;
+ this.host = null;
+ this.view = null;
+ this.error = null;
+ if (!view) return;
+ if (host && !host.isDestroyed()) host.contentView.removeChildView(view);
+ if (!view.webContents.isDestroyed()) view.webContents.close();
+ }
+}
diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts
index 304e998..052ab36 100644
--- a/apps/desktop/src/main.ts
+++ b/apps/desktop/src/main.ts
@@ -21,6 +21,7 @@ import {
LAUNCH_CHECK_DELAY_MS,
} from "./auto-update.js";
import { openBrowserAuth } from "./browser-auth.js";
+import { BrowserPreviewController } from "./browser-preview.js";
import { DOCKER_INSTALL_LINKS, isDesktopSetupLink, runDocker } from "./docker-cli.js";
import { requestLocalSettings } from "./local-settings.js";
import {
@@ -1130,6 +1131,35 @@ app.whenReady().then(async () => {
},
);
ipcMain.handle("desktop.platform", () => process.platform);
+ const browserPreview = new BrowserPreviewController();
+ // The in-app browser answers only the app window's own top frame, never a page inside it.
+ const previewHost = (event: Electron.IpcMainInvokeEvent) => {
+ if (!fromMainWindow(event) || event.senderFrame !== event.sender.mainFrame) {
+ throw new Error("The in-app browser is not available here.");
+ }
+ return mainWindow!;
+ };
+ ipcMain.handle("desktop.browserPreview.state", (event) =>
+ browserPreview.state(previewHost(event)),
+ );
+ ipcMain.handle("desktop.browserPreview.show", (event, bounds: unknown) =>
+ browserPreview.show(previewHost(event), bounds),
+ );
+ ipcMain.handle("desktop.browserPreview.hide", (event) => browserPreview.hide(previewHost(event)));
+ ipcMain.handle("desktop.browserPreview.navigate", (event, url: unknown) =>
+ browserPreview.navigate(previewHost(event), url),
+ );
+ ipcMain.handle("desktop.browserPreview.back", (event) => browserPreview.back(previewHost(event)));
+ ipcMain.handle("desktop.browserPreview.forward", (event) =>
+ browserPreview.forward(previewHost(event)),
+ );
+ ipcMain.handle("desktop.browserPreview.reload", (event) =>
+ browserPreview.reload(previewHost(event)),
+ );
+ ipcMain.handle("desktop.browserPreview.stop", (event) => browserPreview.stop(previewHost(event)));
+ ipcMain.handle("desktop.browserPreview.openDevTools", (event) =>
+ browserPreview.openDevTools(previewHost(event)),
+ );
ipcMain.handle("desktop.files.pickDirectory", async (event) => {
if (!fromMainWindow(event)) return null;
const result = await dialog.showOpenDialog(windowFrom(event)!, {
diff --git a/apps/desktop/src/preload.cjs b/apps/desktop/src/preload.cjs
index 14e5c50..7bc2c7b 100644
--- a/apps/desktop/src/preload.cjs
+++ b/apps/desktop/src/preload.cjs
@@ -21,6 +21,22 @@ contextBridge.exposeInMainWorld("rakazoDesktop", {
download: () => ipcRenderer.invoke("desktop.update.download"),
install: () => ipcRenderer.invoke("desktop.update.install"),
},
+ browserPreview: {
+ state: () => ipcRenderer.invoke("desktop.browserPreview.state"),
+ show: (bounds) => ipcRenderer.invoke("desktop.browserPreview.show", bounds),
+ hide: () => ipcRenderer.invoke("desktop.browserPreview.hide"),
+ navigate: (url) => ipcRenderer.invoke("desktop.browserPreview.navigate", url),
+ back: () => ipcRenderer.invoke("desktop.browserPreview.back"),
+ forward: () => ipcRenderer.invoke("desktop.browserPreview.forward"),
+ reload: () => ipcRenderer.invoke("desktop.browserPreview.reload"),
+ stop: () => ipcRenderer.invoke("desktop.browserPreview.stop"),
+ openDevTools: () => ipcRenderer.invoke("desktop.browserPreview.openDevTools"),
+ onState: (listener) => {
+ const handler = (_event, state) => listener(state);
+ ipcRenderer.on("desktop.browserPreview.state", handler);
+ return () => ipcRenderer.off("desktop.browserPreview.state", handler);
+ },
+ },
oauth: {
open: (url) => ipcRenderer.invoke("desktop.oauth.open", url),
cancel: (url) => ipcRenderer.invoke("desktop.oauth.cancel", url),
diff --git a/apps/desktop/src/preload.test.ts b/apps/desktop/src/preload.test.ts
index f5891c0..9e44a20 100644
--- a/apps/desktop/src/preload.test.ts
+++ b/apps/desktop/src/preload.test.ts
@@ -32,6 +32,7 @@ describe("desktop preload bridge", () => {
expect(globalName).toBe("rakazoDesktop");
expect(bridge.platform).toBe("linux");
expect(Object.keys(bridge).sort()).toEqual([
+ "browserPreview",
"files",
"localSettings",
"oauth",
@@ -46,6 +47,18 @@ describe("desktop preload bridge", () => {
"toggleMaximize",
]);
expect(Object.keys(bridge.update).sort()).toEqual(["check", "download", "install", "state"]);
+ expect(Object.keys(bridge.browserPreview ?? {}).sort()).toEqual([
+ "back",
+ "forward",
+ "hide",
+ "navigate",
+ "onState",
+ "openDevTools",
+ "reload",
+ "show",
+ "state",
+ "stop",
+ ]);
await bridge.files?.pickDirectory();
await bridge.oauth.open?.("https://provider.example.com/authorize");
@@ -58,6 +71,8 @@ describe("desktop preload bridge", () => {
await bridge.update.check();
await bridge.update.download();
await bridge.update.install();
+ await bridge.browserPreview?.navigate("http://localhost:5173/");
+ await bridge.browserPreview?.show({ x: 1, y: 2, width: 3, height: 4 });
expect(invoke.mock.calls.map(([channel]) => channel)).toEqual([
"desktop.files.pickDirectory",
"desktop.oauth.open",
@@ -70,6 +85,8 @@ describe("desktop preload bridge", () => {
"desktop.update.check",
"desktop.update.download",
"desktop.update.install",
+ "desktop.browserPreview.navigate",
+ "desktop.browserPreview.show",
]);
});
@@ -77,6 +94,7 @@ describe("desktop preload bridge", () => {
const { exposeInMainWorld } = runPreload("preload.cjs");
const [, bridge] = exposeInMainWorld.mock.calls[0] as [string, Record];
expect(Object.keys(bridge).sort()).toEqual([
+ "browserPreview",
"files",
"localSettings",
"oauth",
diff --git a/apps/mobile/lib/api.ts b/apps/mobile/lib/api.ts
index ac39cf1..2e17b0f 100644
--- a/apps/mobile/lib/api.ts
+++ b/apps/mobile/lib/api.ts
@@ -1096,7 +1096,7 @@ export function applyMobileThreadEvent(
messages: [...remaining, streaming],
};
}
- if (event.type === "agent.tool.completed") {
+ if (event.type === "agent.tool.completed" || event.type === "thread.computer") {
return { ...prev, cursor: event.seq ?? prev.cursor };
}
if (event.type === "thread.subagent") {
diff --git a/apps/web/e2e/agent-cursor.spec.ts b/apps/web/e2e/agent-cursor.spec.ts
new file mode 100644
index 0000000..052a821
--- /dev/null
+++ b/apps/web/e2e/agent-cursor.spec.ts
@@ -0,0 +1,53 @@
+import { expect, test } from "@playwright/test";
+import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers";
+
+const SCREEN_URL = "https://screen.example/embed.html";
+
+test("the bot's cursor glides to where it clicks on its live screen", async ({
+ page,
+}, testInfo) => {
+ await signup(page, `agent-cursor-${Date.now()}@rakazo.test`, "password12", "Cursor Watcher");
+ await completeOnboarding(page);
+ const botId = activeBotId(page);
+ await rpc(page, "computer/boot", { botId });
+ // Stand in for the noVNC viewer: a 16:10 desktop with a field in the middle.
+ await page.route("https://screen.example/**", (route) =>
+ route.fulfill({
+ contentType: "text/html",
+ body: `Desktop
+ `,
+ }),
+ );
+ await page.route("**/rpc/computer/screenUrl", (route) =>
+ route.fulfill({
+ contentType: "application/json",
+ body: JSON.stringify({ json: { url: SCREEN_URL } }),
+ }),
+ );
+
+ await page.getByTitle("Agent computer").click();
+ const preview = page.getByTestId("computer-preview");
+ await expect(preview.locator("iframe")).toHaveAttribute("src", SCREEN_URL);
+ const cursor = preview.getByTestId("agent-cursor");
+ await expect(cursor).toHaveCSS("opacity", "0");
+
+ // The scripted runtime clicks the screen center (640, 400) and types.
+ await rpc(page, "threads/send", { botId, text: "use your screen and type cursor-demo" });
+
+ await expect(cursor).toHaveCSS("opacity", "1");
+ await expect(cursor).toHaveAttribute("data-action", /click|type/);
+ const layer = (await preview.getByTestId("agent-cursor-layer").boundingBox())!;
+ await expect
+ .poll(async () => {
+ const tip = (await cursor.boundingBox())!;
+ return [
+ Math.round(tip.x - layer.x - layer.width / 2),
+ Math.round(tip.y - layer.y - layer.height / 2),
+ ];
+ })
+ .toEqual([0, 0]);
+ await captureScreenshot(page, testInfo, "agent-cursor-on-screen");
+
+ // It fades once the bot stops pointing, so it never lingers over the screen.
+ await expect(cursor).toHaveCSS("opacity", "0", { timeout: 10_000 });
+});
diff --git a/apps/web/e2e/in-app-browser.spec.ts b/apps/web/e2e/in-app-browser.spec.ts
new file mode 100644
index 0000000..1862edb
--- /dev/null
+++ b/apps/web/e2e/in-app-browser.spec.ts
@@ -0,0 +1,159 @@
+import { expect, test } from "@playwright/test";
+import { captureScreenshot, completeOnboarding, signup } from "./helpers";
+
+test("the in-app browser previews a page and follows it back to the browser", async ({
+ page,
+}, testInfo) => {
+ await signup(page, `in-app-browser-${Date.now()}@rakazo.test`, "password12", "Previewer");
+ await completeOnboarding(page);
+ // Context-wide so the "Open in browser" tab is served too.
+ await page.context().route("https://preview.example/**", (route) =>
+ route.fulfill({
+ contentType: "text/html",
+ body: `Preview
+ Live preview
Rendered inside the app.
`,
+ }),
+ );
+
+ await page.getByRole("button", { name: "Browser" }).click();
+ const browser = page.getByTestId("in-app-browser");
+ const address = browser.getByRole("textbox", { name: "Address" });
+ await expect(address).toBeFocused();
+
+ await address.fill("not an address");
+ await address.press("Enter");
+ await expect(address).toHaveAttribute("aria-invalid", "true");
+
+ await address.fill("preview.example/app");
+ await address.press("Enter");
+ await expect(address).toHaveValue("https://preview.example/app");
+ const frame = browser.getByTestId("in-app-browser-frame");
+ await expect(frame).toHaveAttribute("src", "https://preview.example/app");
+ await expect(
+ page.frameLocator('[data-testid="in-app-browser-frame"]').getByRole("heading"),
+ ).toHaveText("Live preview");
+ await captureScreenshot(page, testInfo, "in-app-browser");
+
+ const popup = page.waitForEvent("popup");
+ await browser.getByRole("button", { name: "Open in browser" }).click();
+ expect((await popup).url()).toBe("https://preview.example/app");
+
+ await browser.getByRole("button", { name: "Close panel" }).click();
+ await expect(browser).toHaveCount(0);
+ // The last address comes back when the browser reopens.
+ await page.getByRole("button", { name: "Browser" }).click();
+ await expect(page.getByTestId("in-app-browser-frame")).toHaveAttribute(
+ "src",
+ "https://preview.example/app",
+ );
+});
+
+test("on desktop the page is a native view glued to the panel", async ({ page }) => {
+ await page.addInitScript(() => {
+ const calls: Array<[string, unknown?]> = [];
+ let listener: ((state: unknown) => void) | undefined;
+ let state = {
+ url: "",
+ title: "",
+ loading: false,
+ canGoBack: false,
+ canGoForward: false,
+ error: null as string | null,
+ };
+ const record = (name: string) => async (value?: unknown) => {
+ calls.push([name, value]);
+ };
+ Object.assign(window, { __browserCalls: calls });
+ Object.defineProperty(window, "rakazoDesktop", {
+ value: {
+ platform: "linux",
+ window: {
+ close: async () => {},
+ minimize: async () => {},
+ toggleMaximize: async () => {},
+ state: async () => ({ minimized: false, maximized: false, fullScreen: false }),
+ },
+ oauth: { onCallback: () => () => {} },
+ update: {
+ state: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
+ check: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
+ download: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
+ install: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
+ },
+ browserPreview: {
+ state: async () => state,
+ onState: (next: (value: unknown) => void) => {
+ listener = next;
+ return () => {
+ listener = undefined;
+ };
+ },
+ navigate: async (url: string) => {
+ calls.push(["navigate", url]);
+ state = { ...state, url, title: "Dev server", loading: false, canGoBack: true };
+ listener?.(state);
+ },
+ show: record("show"),
+ hide: record("hide"),
+ back: record("back"),
+ forward: record("forward"),
+ reload: record("reload"),
+ stop: record("stop"),
+ openDevTools: record("openDevTools"),
+ },
+ },
+ });
+ });
+ await signup(page, `native-browser-${Date.now()}@rakazo.test`, "password12", "Desktop");
+ await completeOnboarding(page);
+ const calls = () =>
+ page.evaluate(
+ () => (window as unknown as { __browserCalls: Array<[string, unknown?]> }).__browserCalls,
+ );
+
+ await page.getByRole("button", { name: "Browser" }).click();
+ const browser = page.getByTestId("in-app-browser");
+ await browser.getByRole("textbox", { name: "Address" }).fill("localhost:5173");
+ await browser.getByRole("textbox", { name: "Address" }).press("Enter");
+ await expect.poll(calls).toContainEqual(["navigate", "http://localhost:5173/"]);
+ await expect(browser.getByRole("button", { name: "Back" })).toBeEnabled();
+ await expect(browser.getByRole("button", { name: "Forward" })).toBeDisabled();
+ await expect(browser.getByTestId("in-app-browser-frame")).toHaveCount(0);
+
+ // The reserved page area and the native view's bounds stay identical.
+ const viewport = browser.getByTestId("in-app-browser-viewport");
+ await expect
+ .poll(async () => {
+ const box = (await viewport.boundingBox())!;
+ const shown = (await calls()).filter(([name]) => name === "show").at(-1)?.[1];
+ return (
+ JSON.stringify(shown) ===
+ JSON.stringify({
+ x: Math.round(box.x),
+ y: Math.round(box.y),
+ width: Math.round(box.width),
+ height: Math.round(box.height),
+ })
+ );
+ })
+ .toBe(true);
+
+ await browser.getByRole("button", { name: "Back" }).click();
+ await browser.getByRole("button", { name: "Reload" }).click();
+ await browser.getByRole("button", { name: "Developer tools" }).click();
+ await expect
+ .poll(async () => (await calls()).map(([name]) => name))
+ .toEqual(expect.arrayContaining(["back", "reload", "openDevTools"]));
+
+ // App menus float above the page, so the native view steps aside while one is open.
+ const before = (await calls()).length;
+ await page.getByTestId("create-menu-trigger").click();
+ await expect
+ .poll(async () => (await calls()).slice(before).map(([name]) => name))
+ .toContain("hide");
+ await page.keyboard.press("Escape");
+ await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("show");
+
+ await browser.getByRole("button", { name: "Close panel" }).click();
+ await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("hide");
+});
diff --git a/apps/web/e2e/page-browser.spec.ts b/apps/web/e2e/page-browser.spec.ts
index 7354efc..2ffb90c 100644
--- a/apps/web/e2e/page-browser.spec.ts
+++ b/apps/web/e2e/page-browser.spec.ts
@@ -50,6 +50,16 @@ test("live page helper preserves identity, masks passwords, and reports partial
expect(snapshot.title).toBe("Fixture");
expect(snapshot.tree).toContain("Readable article content");
expect(JSON.stringify(snapshot)).not.toContain("fake-test-password");
+ // Element boxes are screen pixels for the visible bot cursor.
+ expect(snapshot.screen.width).toBeGreaterThan(0);
+ expect(snapshot.screen.height).toBeGreaterThan(0);
+ const saveBox = snapshot.elements.find((element: { name: string }) => element.name === "Save")
+ .box as number[];
+ const saveRect = (await page.locator("#save").boundingBox())!;
+ expect(saveBox[2]).toBe(Math.round(saveRect.width));
+ expect(saveBox[3]).toBe(Math.round(saveRect.height));
+ expect(saveBox[0]).toBeGreaterThanOrEqual(Math.round(saveRect.x));
+ expect(saveBox[1]).toBeGreaterThanOrEqual(Math.round(saveRect.y));
const ref = (snapshot: { elements: { name: string; ref: string }[] }, name: string) =>
snapshot.elements.find((element) => element.name === name)!.ref;
const result = await command("act", {
diff --git a/apps/web/src/components/browser/InAppBrowser.tsx b/apps/web/src/components/browser/InAppBrowser.tsx
new file mode 100644
index 0000000..22ac61a
--- /dev/null
+++ b/apps/web/src/components/browser/InAppBrowser.tsx
@@ -0,0 +1,317 @@
+import { useLingui } from "@lingui/react/macro";
+import type {
+ RakazoDesktopBrowserPreview,
+ RakazoDesktopBrowserPreviewState,
+} from "@rakazo/contracts";
+import { Button, cn, Input } from "@rakazo/ui-web";
+import { ArrowLeft, ArrowRight, Code, ExternalLink, RotateCw, X } from "lucide-react";
+import { type FormEvent, useCallback, useEffect, useRef, useState } from "react";
+import { browserFrameSandbox, normalizeBrowserUrl } from "../../lib/browser-url";
+import { desktopBridge } from "../../lib/desktop";
+
+const LAST_URL_KEY = "rk:in-app-browser:url";
+
+function readLastUrl(): string {
+ try {
+ return window.localStorage.getItem(LAST_URL_KEY) ?? "";
+ } catch {
+ return "";
+ }
+}
+
+function rememberUrl(url: string) {
+ try {
+ window.localStorage.setItem(LAST_URL_KEY, url);
+ } catch {
+ /* A per-viewer convenience only. */
+ }
+}
+
+const EMPTY_STATE: RakazoDesktopBrowserPreviewState = {
+ url: "",
+ title: "",
+ loading: false,
+ canGoBack: false,
+ canGoForward: false,
+ error: null,
+};
+
+/** DOM layers the native desktop view would otherwise paint over. */
+const FLOATING_UI = '[role="dialog"],[role="alertdialog"],[role="menu"],[role="listbox"]';
+
+/**
+ * Address bar plus page. On desktop the page is a native Chromium view laid over the reserved
+ * area, so dev servers keep hot reload, DevTools and full speed; in a browser it is an iframe.
+ */
+export function InAppBrowser({ onClose }: { onClose: () => void }) {
+ const { t } = useLingui();
+ const native = desktopBridge()?.browserPreview;
+ const [state, setState] = useState(() => ({
+ ...EMPTY_STATE,
+ url: native ? "" : readLastUrl(),
+ }));
+ const [draft, setDraft] = useState(state.url);
+ const [invalid, setInvalid] = useState(false);
+ const [frameKey, setFrameKey] = useState(0);
+ const editing = useRef(false);
+ const inputRef = useRef(null);
+
+ useEffect(() => {
+ if (!native) return;
+ let live = true;
+ const apply = (next: RakazoDesktopBrowserPreviewState) => {
+ if (!live) return;
+ setState(next);
+ if (!editing.current) setDraft(next.url);
+ };
+ const unsubscribe = native.onState(apply);
+ void native
+ .state()
+ .then((current) => {
+ apply(current);
+ const last = readLastUrl();
+ if (!current.url && last) void native.navigate(last).catch(() => undefined);
+ })
+ .catch(() => undefined);
+ return () => {
+ live = false;
+ unsubscribe();
+ };
+ }, [native]);
+
+ useEffect(() => {
+ if (!state.url) inputRef.current?.focus();
+ }, [state.url]);
+
+ const go = useCallback(
+ (input: string) => {
+ const url = normalizeBrowserUrl(input);
+ setInvalid(!url);
+ if (!url) return;
+ editing.current = false;
+ setDraft(url);
+ rememberUrl(url);
+ if (native) {
+ void native.navigate(url).catch(() => setInvalid(true));
+ } else {
+ setState((current) => ({ ...current, url, loading: true, error: null }));
+ setFrameKey((key) => key + 1);
+ }
+ },
+ [native],
+ );
+
+ const reload = useCallback(() => {
+ if (native) {
+ void (state.loading ? native.stop() : native.reload()).catch(() => undefined);
+ return;
+ }
+ if (!state.url) return;
+ setState((current) => ({ ...current, loading: true }));
+ setFrameKey((key) => key + 1);
+ }, [native, state.loading, state.url]);
+
+ const onSubmit = (event: FormEvent) => {
+ event.preventDefault();
+ go(draft);
+ inputRef.current?.blur();
+ };
+
+ return (
+
+
+
+ {native ? (
+
+ ) : state.url ? (
+
+
+ );
+}
+
+/**
+ * Reserves the page area and keeps the native view glued to it while the panel resizes,
+ * hiding it whenever app UI (dialogs, menus) floats where the view would paint over it.
+ */
+function NativeViewport({
+ browser,
+ visible,
+}: {
+ browser: RakazoDesktopBrowserPreview;
+ visible: boolean;
+}) {
+ const ref = useRef(null);
+
+ useEffect(() => {
+ const element = ref.current;
+ if (!element) return;
+ let last = "";
+ let frame = 0;
+ const sync = () => {
+ frame = 0;
+ const rect = element.getBoundingClientRect();
+ const covered = Array.from(document.querySelectorAll(FLOATING_UI)).some(
+ (node) => !element.contains(node),
+ );
+ const show = visible && rect.width >= 1 && rect.height >= 1 && !covered;
+ const bounds = {
+ x: Math.round(rect.left),
+ y: Math.round(rect.top),
+ width: Math.round(rect.width),
+ height: Math.round(rect.height),
+ };
+ const key = show ? JSON.stringify(bounds) : "hidden";
+ if (key === last) return;
+ last = key;
+ void (show ? browser.show(bounds) : browser.hide()).catch(() => undefined);
+ };
+ const schedule = () => {
+ if (!frame) frame = requestAnimationFrame(sync);
+ };
+ const resize = new ResizeObserver(schedule);
+ resize.observe(element);
+ const floating = new MutationObserver(schedule);
+ floating.observe(document.body, { childList: true, subtree: true });
+ window.addEventListener("resize", schedule);
+ // Sibling layout (sidebars, transitions) can move the area without resizing it.
+ const settle = setInterval(schedule, 250);
+ schedule();
+ return () => {
+ cancelAnimationFrame(frame);
+ resize.disconnect();
+ floating.disconnect();
+ window.removeEventListener("resize", schedule);
+ clearInterval(settle);
+ void browser.hide().catch(() => undefined);
+ };
+ }, [browser, visible]);
+
+ return ;
+}
diff --git a/apps/web/src/components/computer/AgentCursor.tsx b/apps/web/src/components/computer/AgentCursor.tsx
new file mode 100644
index 0000000..5df4889
--- /dev/null
+++ b/apps/web/src/components/computer/AgentCursor.tsx
@@ -0,0 +1,165 @@
+import { screenPointInViewer } from "@rakazo/core";
+import { Keyboard } from "lucide-react";
+import { useEffect, useRef, useState } from "react";
+import {
+ agentPointerFeed,
+ POINTER_IDLE_MS,
+ type PointerPoint,
+ type PointerScreen,
+ type PointerSegment,
+ planPointerTimeline,
+ pointerFrameAt,
+} from "../../lib/agent-pointer";
+
+function prefersReducedMotion() {
+ return (
+ typeof window.matchMedia === "function" &&
+ window.matchMedia("(prefers-reduced-motion: reduce)").matches
+ );
+}
+
+/**
+ * Draws a bot's announced pointer over its live screen: it glides to each target, ripples on
+ * clicks and shows when it types. Frames are written straight to the DOM so the chat never
+ * re-renders at animation rate.
+ */
+export function AgentCursor({
+ botId,
+ color,
+ name,
+}: {
+ botId: string;
+ color: string;
+ name: string;
+}) {
+ const layerRef = useRef(null);
+ const cursorRef = useRef(null);
+ const rippleRef = useRef(null);
+ const [typing, setTyping] = useState(false);
+
+ useEffect(() => {
+ const layer = layerRef.current;
+ const cursor = cursorRef.current;
+ const ripple = rippleRef.current;
+ if (!layer || !cursor || !ripple) return;
+ const reducedMotion = prefersReducedMotion();
+ let screen: PointerScreen | null = null;
+ let segments: PointerSegment[] = [];
+ let position: PointerPoint | null = null;
+ let frame = 0;
+ let idleTimer: ReturnType | undefined;
+ let viewer = { width: layer.clientWidth, height: layer.clientHeight };
+ let shownTyping = false;
+
+ const place = (point: PointerPoint, action: string | null, progress: number) => {
+ if (!screen) return;
+ const projected = screenPointInViewer(point, viewer, screen);
+ cursor.style.transform = `translate3d(${projected.x}px, ${projected.y}px, 0)`;
+ cursor.dataset.action = action ?? "move";
+ const pressing = action === "click" || action === "double_click" || action === "right_click";
+ const pulse = action === "double_click" ? (progress * 2) % 1 : progress;
+ ripple.style.opacity = pressing && !reducedMotion ? String(0.55 * (1 - pulse)) : "0";
+ ripple.style.transform = `scale(${0.35 + pulse * 1.15})`;
+ cursor.style.setProperty("--agent-cursor-press", pressing && pulse < 0.35 ? "0.86" : "1");
+ const nextTyping = action === "type" || action === "key";
+ if (nextTyping !== shownTyping) {
+ shownTyping = nextTyping;
+ setTyping(nextTyping);
+ }
+ };
+
+ const rest = () => {
+ clearTimeout(idleTimer);
+ idleTimer = setTimeout(() => {
+ cursor.style.opacity = "0";
+ }, POINTER_IDLE_MS);
+ };
+
+ const tick = () => {
+ frame = 0;
+ const now = performance.now();
+ const state = pointerFrameAt(segments, now, reducedMotion);
+ if (!state) return;
+ position = state.point;
+ place(state.point, state.action?.kind ?? null, state.action?.progress ?? 0);
+ if (state.done) {
+ segments = [];
+ rest();
+ return;
+ }
+ frame = requestAnimationFrame(tick);
+ };
+
+ const unsubscribe = agentPointerFeed.subscribe(botId, (payload) => {
+ const now = performance.now();
+ const current = pointerFrameAt(segments, now, reducedMotion)?.point ?? position;
+ const sameScreen =
+ screen?.width === payload.screen.width && screen?.height === payload.screen.height;
+ screen = payload.screen;
+ // A first announcement starts at its target rather than flying in from a corner.
+ const firstTarget = payload.steps.find((step) => typeof step.x === "number");
+ const from =
+ current && sameScreen
+ ? current
+ : firstTarget
+ ? { x: firstTarget.x ?? 0, y: firstTarget.y ?? 0 }
+ : { x: payload.screen.width / 2, y: payload.screen.height / 2 };
+ segments = planPointerTimeline(from, payload.steps, now);
+ clearTimeout(idleTimer);
+ cursor.style.opacity = "1";
+ if (!frame) frame = requestAnimationFrame(tick);
+ });
+
+ const resize = new ResizeObserver(() => {
+ viewer = { width: layer.clientWidth, height: layer.clientHeight };
+ if (position && !frame) place(position, null, 0);
+ });
+ resize.observe(layer);
+
+ return () => {
+ unsubscribe();
+ resize.disconnect();
+ cancelAnimationFrame(frame);
+ clearTimeout(idleTimer);
+ };
+ }, [botId]);
+
+ return (
+
+
+
+
+
+
+ {name}
+ {typing ? : null}
+
+
+
+ );
+}
diff --git a/apps/web/src/lib/agent-pointer.test.ts b/apps/web/src/lib/agent-pointer.test.ts
new file mode 100644
index 0000000..951eef4
--- /dev/null
+++ b/apps/web/src/lib/agent-pointer.test.ts
@@ -0,0 +1,88 @@
+import { describe, expect, it } from "vitest";
+import {
+ agentPointerFeed,
+ moveDurationMs,
+ planPointerTimeline,
+ pointerFrameAt,
+} from "./agent-pointer";
+
+const payload = {
+ kind: "pointer",
+ screen: { width: 1280, height: 800 },
+ steps: [{ action: "click", x: 100, y: 100 }],
+};
+
+describe("agentPointerFeed", () => {
+ it("delivers valid live announcements only to the bot's viewers", () => {
+ const seen: unknown[] = [];
+ const other: unknown[] = [];
+ const stop = agentPointerFeed.subscribe("bot-a", (value) => seen.push(value));
+ const stopOther = agentPointerFeed.subscribe("bot-b", (value) => other.push(value));
+ try {
+ const now = Date.parse("2026-01-01T00:00:10Z");
+ expect(agentPointerFeed.publish("bot-a", payload, "2026-01-01T00:00:05Z", now)).toBe(true);
+ // Replayed history and malformed payloads never move the cursor.
+ expect(agentPointerFeed.publish("bot-a", payload, "2025-12-31T23:00:00Z", now)).toBe(false);
+ expect(agentPointerFeed.publish("bot-a", { ...payload, steps: [] }, undefined, now)).toBe(
+ false,
+ );
+ expect(agentPointerFeed.publish("bot-a", { kind: "status" }, undefined, now)).toBe(false);
+ expect(seen).toEqual([payload]);
+ expect(other).toEqual([]);
+ } finally {
+ stop();
+ stopOther();
+ }
+ expect(agentPointerFeed.publish("bot-a", payload)).toBe(true);
+ expect(seen).toHaveLength(1);
+ });
+});
+
+describe("pointer timeline", () => {
+ it("travels, acts in place for keyboard steps, then rests", () => {
+ const segments = planPointerTimeline(
+ { x: 0, y: 0 },
+ [{ action: "click", x: 300, y: 400 }, { action: "type" }, { action: "move", x: 300, y: 400 }],
+ 1_000,
+ );
+ expect(segments.map((segment) => segment.to)).toEqual([
+ { x: 300, y: 400 },
+ { x: 300, y: 400 },
+ { x: 300, y: 400 },
+ ]);
+ const [click, type, move] = segments;
+ expect(click!.arrive - click!.start).toBe(moveDurationMs(500));
+ expect(type!.arrive).toBe(type!.start);
+ expect(move!.end).toBe(move!.start);
+
+ const midway = pointerFrameAt(segments, (click!.start + click!.arrive) / 2)!;
+ expect(midway.action).toBeNull();
+ // The bow lifts the cursor off the straight line between the points.
+ const onLine = { x: 150, y: 200 };
+ expect(Math.hypot(midway.point.x - onLine.x, midway.point.y - onLine.y)).toBeGreaterThan(10);
+
+ const clicking = pointerFrameAt(segments, click!.arrive + 1)!;
+ expect(clicking.point).toEqual({ x: 300, y: 400 });
+ expect(clicking.action?.kind).toBe("click");
+
+ const typing = pointerFrameAt(segments, type!.arrive + 10)!;
+ expect(typing.action?.kind).toBe("type");
+
+ expect(pointerFrameAt(segments, move!.end + 1)).toEqual({
+ point: { x: 300, y: 400 },
+ action: null,
+ done: true,
+ });
+ });
+
+ it("jumps straight to the target when motion is reduced", () => {
+ const segments = planPointerTimeline({ x: 0, y: 0 }, [{ action: "move", x: 50, y: 60 }], 0);
+ expect(pointerFrameAt(segments, 1, true)?.point).toEqual({ x: 50, y: 60 });
+ });
+
+ it("bounds travel time so long and short moves both feel responsive", () => {
+ expect(moveDurationMs(0)).toBe(0);
+ expect(moveDurationMs(5)).toBe(180);
+ expect(moveDurationMs(10_000)).toBe(650);
+ });
+});
diff --git a/apps/web/src/lib/agent-pointer.ts b/apps/web/src/lib/agent-pointer.ts
new file mode 100644
index 0000000..fb8619a
--- /dev/null
+++ b/apps/web/src/lib/agent-pointer.ts
@@ -0,0 +1,156 @@
+import {
+ type ComputerPointerAction,
+ ComputerPointerPayload,
+ type ComputerPointerStep,
+} from "@rakazo/contracts";
+
+export type PointerPoint = { x: number; y: number };
+export type PointerScreen = ComputerPointerPayload["screen"];
+
+/** Older announcements describe actions that already finished; replaying them would mislead. */
+export const POINTER_EVENT_MAX_AGE_MS = 30_000;
+export const POINTER_IDLE_MS = 3_500;
+
+type Listener = (payload: ComputerPointerPayload) => void;
+
+/**
+ * Live pointer announcements per bot. The thread subscription publishes `thread.computer`
+ * pointer events here; screen viewers subscribe for the bot they show.
+ */
+class AgentPointerFeed {
+ private readonly listeners = new Map>();
+
+ publish(botId: string, payload: unknown, createdAt?: string, now = Date.now()): boolean {
+ const parsed = ComputerPointerPayload.safeParse(payload);
+ if (!parsed.success) return false;
+ const at = createdAt ? Date.parse(createdAt) : Number.NaN;
+ if (Number.isFinite(at) && now - at > POINTER_EVENT_MAX_AGE_MS) return false;
+ for (const listener of this.listeners.get(botId) ?? []) listener(parsed.data);
+ return true;
+ }
+
+ subscribe(botId: string, listener: Listener): () => void {
+ let set = this.listeners.get(botId);
+ if (!set) {
+ set = new Set();
+ this.listeners.set(botId, set);
+ }
+ set.add(listener);
+ return () => {
+ set.delete(listener);
+ if (set.size === 0) this.listeners.delete(botId);
+ };
+ }
+}
+
+export const agentPointerFeed = new AgentPointerFeed();
+
+export type PointerSegment = {
+ from: PointerPoint;
+ to: PointerPoint;
+ /** Perpendicular bow in screen pixels so long moves read as a hand, not a teleport. */
+ bow: number;
+ start: number;
+ arrive: number;
+ end: number;
+ action: ComputerPointerAction;
+};
+
+const ACTION_DWELL_MS: Record = {
+ move: 0,
+ click: 320,
+ double_click: 440,
+ right_click: 320,
+ down: 160,
+ up: 160,
+ type: 700,
+ key: 280,
+ scroll: 420,
+};
+
+export function moveDurationMs(distance: number): number {
+ return distance < 1 ? 0 : Math.round(Math.min(650, Math.max(180, 140 + distance * 0.45)));
+}
+
+/**
+ * Lays announced steps on a timeline that starts where the cursor currently is. Keyboard
+ * steps carry no coordinates and act in place.
+ */
+export function planPointerTimeline(
+ from: PointerPoint,
+ steps: readonly ComputerPointerStep[],
+ startAt: number,
+): PointerSegment[] {
+ const segments: PointerSegment[] = [];
+ let position = from;
+ let time = startAt;
+ for (const step of steps) {
+ const to =
+ typeof step.x === "number" && typeof step.y === "number"
+ ? { x: step.x, y: step.y }
+ : position;
+ const distance = Math.hypot(to.x - position.x, to.y - position.y);
+ const arrive = time + moveDurationMs(distance);
+ const end = arrive + ACTION_DWELL_MS[step.action];
+ segments.push({
+ from: position,
+ to,
+ bow: Math.min(48, distance * 0.12),
+ start: time,
+ arrive,
+ end,
+ action: step.action,
+ });
+ position = to;
+ time = end;
+ }
+ return segments;
+}
+
+export type PointerFrame = {
+ point: PointerPoint;
+ /** The action being performed at the target, with 0..1 progress; null while travelling. */
+ action: { kind: ComputerPointerAction; progress: number } | null;
+ done: boolean;
+};
+
+function easeInOutCubic(t: number): number {
+ return t < 0.5 ? 4 * t * t * t : 1 - (-2 * t + 2) ** 3 / 2;
+}
+
+export function pointerFrameAt(
+ segments: readonly PointerSegment[],
+ now: number,
+ reducedMotion = false,
+): PointerFrame | null {
+ if (segments.length === 0) return null;
+ const last = segments[segments.length - 1]!;
+ if (now >= last.end) return { point: last.to, action: null, done: true };
+ const segment = segments.find((candidate) => now < candidate.end) ?? last;
+ if (now < segment.arrive) {
+ if (reducedMotion) return { point: segment.to, action: null, done: false };
+ const span = segment.arrive - segment.start;
+ const t = easeInOutCubic(span > 0 ? Math.max(0, (now - segment.start) / span) : 1);
+ const dx = segment.to.x - segment.from.x;
+ const dy = segment.to.y - segment.from.y;
+ const length = Math.hypot(dx, dy) || 1;
+ const lift = Math.sin(Math.PI * t) * segment.bow;
+ return {
+ point: {
+ x: segment.from.x + dx * t - (dy / length) * lift,
+ y: segment.from.y + dy * t + (dx / length) * lift,
+ },
+ action: null,
+ done: false,
+ };
+ }
+ const dwell = segment.end - segment.arrive;
+ return {
+ point: segment.to,
+ action:
+ segment.action === "move"
+ ? null
+ : { kind: segment.action, progress: dwell > 0 ? (now - segment.arrive) / dwell : 1 },
+ done: false,
+ };
+}
diff --git a/apps/web/src/lib/browser-preview-context.test.ts b/apps/web/src/lib/browser-preview-context.test.ts
deleted file mode 100644
index b28d7e2..0000000
--- a/apps/web/src/lib/browser-preview-context.test.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { browserPreviewActivity } from "./browser-preview-context";
-
-describe("browserPreviewActivity", () => {
- it("publishes cursors to subscribers and drops cleared ids", () => {
- for (const id of browserPreviewActivity.snapshot().keys()) browserPreviewActivity.clear(id);
-
- const seen: string[][] = [];
- const unsubscribe = browserPreviewActivity.subscribe((activities) => {
- seen.push([...activities.keys()]);
- });
-
- try {
- browserPreviewActivity.set("bot-a", 50, 40, "Ada");
- browserPreviewActivity.set("bot-a", 50, 40, "Ada");
- browserPreviewActivity.set("bot-b", 10, 20, "Bob");
- expect(browserPreviewActivity.snapshot().get("bot-a")).toEqual({
- x: 50,
- y: 40,
- label: "Ada",
- });
-
- browserPreviewActivity.clear("bot-a");
- browserPreviewActivity.clear("missing");
-
- expect(seen).toEqual([[], ["bot-a"], ["bot-a", "bot-b"], ["bot-b"]]);
- } finally {
- unsubscribe();
- for (const id of [...browserPreviewActivity.snapshot().keys()]) {
- browserPreviewActivity.clear(id);
- }
- }
-
- expect(browserPreviewActivity.snapshot().size).toBe(0);
- });
-});
diff --git a/apps/web/src/lib/browser-preview-context.ts b/apps/web/src/lib/browser-preview-context.ts
deleted file mode 100644
index e9a9621..0000000
--- a/apps/web/src/lib/browser-preview-context.ts
+++ /dev/null
@@ -1,47 +0,0 @@
-export type BrowserPreviewCursor = {
- x: number;
- y: number;
- label: string;
-};
-
-type Listener = (activities: ReadonlyMap) => void;
-
-/**
- * In-memory cursors for the desktop browser-preview panel.
- * The shell publishes working bots here; the panel subscribes.
- */
-class BrowserPreviewActivity {
- private readonly activities = new Map();
- private readonly listeners = new Set();
-
- subscribe(listener: Listener): () => void {
- this.listeners.add(listener);
- listener(this.snapshot());
- return () => {
- this.listeners.delete(listener);
- };
- }
-
- snapshot(): ReadonlyMap {
- return new Map(this.activities);
- }
-
- set(id: string, x: number, y: number, label: string) {
- const current = this.activities.get(id);
- if (current && current.x === x && current.y === y && current.label === label) return;
- this.activities.set(id, { x, y, label });
- this.emit();
- }
-
- clear(id: string) {
- if (!this.activities.delete(id)) return;
- this.emit();
- }
-
- private emit() {
- const snapshot = this.snapshot();
- for (const listener of this.listeners) listener(snapshot);
- }
-}
-
-export const browserPreviewActivity = new BrowserPreviewActivity();
diff --git a/apps/web/src/lib/browser-url.test.ts b/apps/web/src/lib/browser-url.test.ts
new file mode 100644
index 0000000..fa693ec
--- /dev/null
+++ b/apps/web/src/lib/browser-url.test.ts
@@ -0,0 +1,38 @@
+import { describe, expect, it } from "vitest";
+import { browserFrameSandbox, normalizeBrowserUrl } from "./browser-url";
+
+describe("normalizeBrowserUrl", () => {
+ it("defaults local dev servers to http and the web to https", () => {
+ expect(normalizeBrowserUrl("localhost:5173")).toBe("http://localhost:5173/");
+ expect(normalizeBrowserUrl("127.0.0.1:3000/app?x=1")).toBe("http://127.0.0.1:3000/app?x=1");
+ expect(normalizeBrowserUrl("example.com/docs")).toBe("https://example.com/docs");
+ expect(normalizeBrowserUrl(" https://example.com ")).toBe("https://example.com/");
+ expect(normalizeBrowserUrl("http://example.com")).toBe("http://example.com/");
+ });
+
+ it("rejects input that is not an http(s) address", () => {
+ for (const input of [
+ "",
+ "hello world",
+ "hello",
+ "javascript:alert(1)",
+ "file:///etc/passwd",
+ "data:text/html,hi",
+ "ftp://example.com",
+ "https://user:fake-password@example.com",
+ ]) {
+ expect(normalizeBrowserUrl(input), input).toBeNull();
+ }
+ });
+});
+
+describe("browserFrameSandbox", () => {
+ it("never grants same-origin access to the app's own origin", () => {
+ expect(browserFrameSandbox("https://app.test/x", "https://app.test")).not.toContain(
+ "allow-same-origin",
+ );
+ expect(browserFrameSandbox("http://localhost:5173/", "https://app.test")).toContain(
+ "allow-same-origin",
+ );
+ });
+});
diff --git a/apps/web/src/lib/browser-url.ts b/apps/web/src/lib/browser-url.ts
new file mode 100644
index 0000000..f1472a9
--- /dev/null
+++ b/apps/web/src/lib/browser-url.ts
@@ -0,0 +1,39 @@
+const LOCAL_HOST = /^(localhost|127(?:\.\d{1,3}){3}|\[::1\]|0\.0\.0\.0|[\w-]+\.local(?:host)?)$/i;
+
+/**
+ * Turns what someone types in the address bar into an http(s) URL. Local dev servers default to
+ * http, everything else to https. Returns null for input that is not an address.
+ */
+export function normalizeBrowserUrl(input: string): string | null {
+ const value = input.trim();
+ if (!value || /\s/.test(value)) return null;
+ const hasScheme = /^[a-z][a-z\d+.-]*:\/\//i.test(value);
+ if (!hasScheme && /^[a-z][a-z\d+.-]*:(?!\d)/i.test(value)) return null;
+ let candidate = value;
+ if (!hasScheme) {
+ const host = value.split(/[/?#]/, 1)[0]!.replace(/:\d+$/, "");
+ if (!LOCAL_HOST.test(host) && !host.includes(".") && !/^\d+$/.test(host)) return null;
+ candidate = `${LOCAL_HOST.test(host) ? "http" : "https"}://${value}`;
+ }
+ try {
+ const url = new URL(candidate);
+ if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+ if (url.username || url.password || !url.hostname) return null;
+ return url.toString();
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Sandbox for the web fallback. A page on the app's own origin must not get same-origin
+ * access, or it could reach into the app around the sandbox.
+ */
+export function browserFrameSandbox(url: string, appOrigin: string): string {
+ const base = "allow-scripts allow-forms allow-popups allow-modals allow-downloads";
+ try {
+ return new URL(url).origin === appOrigin ? base : `${base} allow-same-origin`;
+ } catch {
+ return base;
+ }
+}
diff --git a/apps/web/src/lib/thread-events.test.ts b/apps/web/src/lib/thread-events.test.ts
index 75c8c2e..ab0da23 100644
--- a/apps/web/src/lib/thread-events.test.ts
+++ b/apps/web/src/lib/thread-events.test.ts
@@ -925,6 +925,24 @@ describe("thread event reduction", () => {
expect(next?.messages).toEqual(initial.messages);
});
+ it("advances past a bot pointer announcement without touching messages", () => {
+ const initial = snapshot([message("m1", [{ kind: "text", text: "hi" }], 3)], 3);
+ const pointer = event({
+ type: "thread.computer",
+ seq: 4,
+ runId: "run-1",
+ payload: {
+ kind: "pointer",
+ screen: { width: 1280, height: 800 },
+ steps: [{ action: "click", x: 10, y: 20 }],
+ },
+ });
+ expect(isThreadSnapshotEvent(pointer)).toBe(true);
+ const next = reduceThreadSnapshot(initial, pointer);
+ expect(next?.cursor).toBe(4);
+ expect(next?.messages).toEqual(initial.messages);
+ });
+
it("holds a tool call that lands mid-sentence until the sentence completes", () => {
const initial = snapshot([]);
diff --git a/apps/web/src/lib/thread-events.ts b/apps/web/src/lib/thread-events.ts
index bf1fb53..5f5edbe 100644
--- a/apps/web/src/lib/thread-events.ts
+++ b/apps/web/src/lib/thread-events.ts
@@ -250,6 +250,7 @@ export function isThreadSnapshotEvent(event: ProductEvent): boolean {
event.type === "thread.cloud_agent" ||
event.type === "agent.tool.called" ||
event.type === "agent.tool.completed" ||
+ event.type === "thread.computer" ||
event.type === "thread.message.created" ||
event.type === "thread.message.updated" ||
event.type === "thread.message.reaction" ||
@@ -440,7 +441,7 @@ export function reduceThreadSnapshot(
};
return { ...prev, cursor: event.seq, messages: [...remaining, next] };
}
- if (event.type === "agent.tool.completed") {
+ if (event.type === "agent.tool.completed" || event.type === "thread.computer") {
return { ...prev, cursor: event.seq };
}
if (event.type === "thread.subagent") {
diff --git a/apps/web/src/pages/Shell.tsx b/apps/web/src/pages/Shell.tsx
index 2ebaee7..f5dc4df 100644
--- a/apps/web/src/pages/Shell.tsx
+++ b/apps/web/src/pages/Shell.tsx
@@ -109,6 +109,7 @@ import {
Copy,
Folder,
Gauge,
+ Globe,
GripVertical,
LayoutGrid,
Lock,
@@ -157,6 +158,7 @@ import { AskCard } from "../components/AskCard";
import { ActiveBotGlyph, CollaborationMarker } from "../components/ai/CollaborationMarker";
import { DelegatedTaskCard, SentMessageCard } from "../components/ai/DelegatedTaskCard";
import { ProjectCreatedCard } from "../components/ai/ProjectCreatedCard";
+import { InAppBrowser } from "../components/browser/InAppBrowser";
import { CloudAgentCard } from "../components/CloudAgentCard";
import { ComputerMaintenanceActions } from "../components/ComputerMaintenanceActions";
import {
@@ -164,6 +166,7 @@ import {
computersAreUnavailable,
} from "../components/ComputersUnavailableHint";
import { ComputerUpdateProgress } from "../components/ComputerUpdateProgress";
+import { AgentCursor } from "../components/computer/AgentCursor";
import { MainInbox } from "../components/MainInbox";
import { MessageHoverMetadata } from "../components/MessageHoverMetadata";
import { MediaCard } from "../components/media/MediaCard";
@@ -177,6 +180,7 @@ import { VoiceAuroraRim } from "../components/voice/VoiceAurora";
import { useSidebarResize } from "../hooks/useSidebarResize";
import { readActivityMode, writeActivityMode } from "../lib/activity-mode";
import { needsYouChipLabel, rosterActivitySubtitle, rosterStatusDot } from "../lib/agent-activity";
+import { agentPointerFeed } from "../lib/agent-pointer";
import type { ArtifactTarget } from "../lib/artifact-open";
import { attachmentMaxBytes, uploadAttachment } from "../lib/artifact-upload";
import { authClient } from "../lib/auth";
@@ -191,8 +195,6 @@ import {
requestBrowserNotificationPermission,
shouldNotifyBrowser,
} from "../lib/browser-notifications";
-import type { BrowserPreviewCursor } from "../lib/browser-preview-context";
-import { browserPreviewActivity } from "../lib/browser-preview-context";
import { loadComputerScreen } from "../lib/computer-screen";
import { desktopBridge } from "../lib/desktop";
import { scheduleFocusPrompt } from "../lib/focus-prompt";
@@ -380,31 +382,6 @@ function readCollapsedSidebarSections(userId: string | null | undefined): Set,
-): Map {
- return new Map(activities);
-}
-
-function sameBrowserPreviewActivities(
- current: ReadonlyMap,
- next: ReadonlyMap,
-): boolean {
- if (current.size !== next.size) return false;
- for (const [id, cursor] of next) {
- const existing = current.get(id);
- if (
- !existing ||
- existing.x !== cursor.x ||
- existing.y !== cursor.y ||
- existing.label !== cursor.label
- ) {
- return false;
- }
- }
- return true;
-}
-
export type NavPillar = "heute" | "team" | "projekte";
function stripSpaceTitlePrefix(title: string | null, spaceName: string): string | null {
@@ -478,16 +455,7 @@ export function ShellPage() {
setBotsSidebarCollapsed(readBotsSidebarCollapsed(userId));
}, [userId]);
- // ── Browser Preview ──────────────────────────────────────────────
- const bridge = desktopBridge();
- const [isDesktop] = useState(Boolean(bridge?.browserPreview));
const [panel, setPanel] = useState(null);
- const [browserUrl, setBrowserUrl] = useState("http://localhost:5173");
- const [devServerRunning, setDevServerRunning] = useState(false);
- const [browserLoading, setBrowserLoading] = useState(false);
- const [agentActivities, setAgentActivities] = useState<
- Map
- >(new Map());
const [showCreateBotModal, setShowCreateBotModal] = useState(false);
@@ -503,60 +471,6 @@ export function ShellPage() {
useSensor(TouchSensor, { activationConstraint: { distance: 8 } }),
);
- // Browser Preview lifecycle
- useEffect(() => {
- const bp = bridge?.browserPreview;
- if (!bp) return;
- const unsubNav = bp.onNavigate((url) => setBrowserUrl(url));
- const unsubLoading = bp.onLoadingProgress((p) => setBrowserLoading(p < 1));
- bp.getState()
- .then((s) => {
- setBrowserUrl(s.url);
- setDevServerRunning(s.devServerRunning);
- })
- .catch(() => {});
- return () => {
- unsubNav();
- unsubLoading();
- };
- }, [bridge]);
-
- const startDevServer = useCallback(async () => {
- try {
- await bridge?.browserPreview?.devServer.start();
- setDevServerRunning(true);
- } catch {
- /* */
- }
- }, [bridge]);
- const stopDevServer = useCallback(async () => {
- try {
- await bridge?.browserPreview?.devServer.stop();
- setDevServerRunning(false);
- } catch {
- /* */
- }
- }, [bridge]);
- const navigateBrowser = useCallback(
- (url: string) => {
- bridge?.browserPreview?.navigate(url).catch(() => {});
- },
- [bridge],
- );
- const reloadBrowser = useCallback(() => {
- bridge?.browserPreview?.reload().catch(() => {});
- }, [bridge]);
-
- // Agent activity → browser preview cursors
- useEffect(() => {
- const unsub = browserPreviewActivity.subscribe((acts) => {
- setAgentActivities((current) =>
- sameBrowserPreviewActivities(current, acts) ? current : copyBrowserPreviewActivities(acts),
- );
- });
- return unsub;
- }, []);
-
const [query, setQuery] = useState("");
const [searchHits, setSearchHits] = useState([]);
const [searchLoading, setSearchLoading] = useState(false);
@@ -1540,6 +1454,8 @@ export function ShellPage() {
void refreshThread(active.id).catch(() => undefined);
} else if (isComputerStatusEvent(event)) {
void refreshComputerScreen(active.id).catch(() => undefined);
+ } else if (event.type === "thread.computer") {
+ agentPointerFeed.publish(event.botId, event.payload, event.createdAt);
}
},
});
@@ -2235,20 +2151,6 @@ export function ShellPage() {
}));
}, [activeSnapshot, enrichStageMember, inGroup, resolveTranscriptBot, workingRuns]);
- // Working bots → browser preview activity cursors
- useEffect(() => {
- if (workingBots.length === 0) return;
- for (const bot of workingBots) {
- if (!bot.botId) continue;
- const label = bot.name ?? bot.botId.slice(0, 8);
- browserPreviewActivity.set(bot.botId, 50, 40, label);
- }
- const activeBotIds = new Set(workingBots.map((b) => b.botId).filter(Boolean));
- for (const id of browserPreviewActivity.snapshot().keys()) {
- if (!activeBotIds.has(id)) browserPreviewActivity.clear(id);
- }
- }, [workingBots]);
-
const encounterMode = Boolean((active || activeGroup) && !showProjectTimeline && !callOpen);
const voiceEncounterActive = encounterMode || callOpen || voiceEntry;
const voiceCallBot = useMemo(() => {
@@ -4486,6 +4388,16 @@ export function ShellPage() {
+
{!inGroup && active ? (
- {panel && (active || activeGroup || panel === "create" || panel === "create-group") ? (
+ {panel === "browser" ? (
+
setPanel(null)} />
+ ) : panel && (active || activeGroup || panel === "create" || panel === "create-group") ? (
{panel !== "routine" &&
panel !== "create" &&
@@ -4749,17 +4663,6 @@ export function ShellPage() {
) : null}
- {!panel && active ? (
-
- ) : null}
) : null}
- {panel === "browser" ? (
-
-
-
-
setBrowserUrl(e.target.value)}
- onBlur={() => navigateBrowser(browserUrl)}
- onKeyDown={(e) => {
- if (e.key === "Enter") navigateBrowser(browserUrl);
- }}
- placeholder="http://localhost:5173"
- className="flex-1 rounded-md bg-muted/60 px-2.5 py-1 text-xs text-foreground placeholder:text-muted-foreground/60 outline-none focus:ring-1 focus:ring-ring/50"
- />
- {isDesktop ? (
- devServerRunning ? (
-
-
- Dev
-
- ) : (
-
- )
- ) : null}
- {browserLoading && (
-
- )}
-
-
-
- {agentActivities.size > 0 && (
-
- {Array.from(agentActivities.entries()).map(([agentId, activity]) => {
- const cx = Math.min(100, Math.max(0, activity.x)) + "%";
- const cy = Math.min(100, Math.max(0, activity.y)) + "%";
- return (
-
-
-
-
- {activity.label}
-
-
-
- );
- })}
-
- )}
-
-
-
- ) : null}
{panel === "computer" && active ? (
) : computer?.state === "running" && embeddedScreenUrl && !computerScreenError ? (
-
+ <>
+
+
+ >
) : (
{computerScreenError ??
@@ -5755,6 +5587,9 @@ export function ShellPage() {
pointerEvents: recordingSkill || !hasControl ? "none" : "auto",
}}
/>
+ {active && !hasControl ? (
+
+ ) : null}
{active ? (
window.innerWidth || cy > window.innerHeight) return undefined;
+ const box = [
+ viewport.x + rect.left * viewport.scale,
+ viewport.y + rect.top * viewport.scale,
+ rect.width * viewport.scale,
+ rect.height * viewport.scale,
+ ].map(Math.round);
+ return box[0] >= 0 && box[1] >= 0 && box[2] > 0 && box[3] > 0 ? box : undefined;
+ }
+ function viewportOnScreen() {
+ const scale = window.devicePixelRatio || 1;
+ // Browser chrome sits above the page; side borders are split evenly.
+ return {
+ scale,
+ x: (window.screenX + Math.max(0, (window.outerWidth - window.innerWidth) / 2)) * scale,
+ y: (window.screenY + Math.max(0, window.outerHeight - window.innerHeight)) * scale,
+ screen: {
+ width: Math.round(window.screen.width * scale),
+ height: Math.round(window.screen.height * scale),
+ },
+ };
+ }
window.__rakazoPageBrowser = {
snapshot() {
refMap.clear();
const elements = [];
+ const viewport = viewportOnScreen();
for (const el of document.querySelectorAll("body *")) {
if (!isInteractive(el) || !visible(el)) continue;
const ref = newRef();
@@ -428,12 +457,14 @@ EVAL_HELPERS = r"""
: ["checkbox", "radio"].includes(el.type) ? (el.checked ? "checked" : "unchecked")
: "value" in el ? String(el.value ?? "").slice(0, 2000) : undefined,
tag: el.tagName.toLowerCase(),
+ box: screenBox(el, viewport),
});
if (elements.length >= 80) break;
}
return {
url: location.href,
title: (document.title || "").slice(0, 200),
+ screen: viewport.screen,
elements,
tree: (document.body?.innerText || "").slice(0, 12000) + "\n" + formatTree(elements)
+ (document.querySelector("iframe,frame") ? "\nEmbedded frames require desktop tools." : ""),
diff --git a/packages/adapter-kit/src/types.ts b/packages/adapter-kit/src/types.ts
index a3c54b6..67669e5 100644
--- a/packages/adapter-kit/src/types.ts
+++ b/packages/adapter-kit/src/types.ts
@@ -778,14 +778,21 @@ export interface BrowserSnapshotNode {
name: string;
value?: string;
tag?: string;
+ /** Screen pixels when the element's center is in view; drives the visible bot cursor only. */
+ box?: BrowserScreenBox;
}
+/** `[x, y, width, height]` in screen pixels of the computer display. */
+export type BrowserScreenBox = [number, number, number, number];
+
export interface BrowserSnapshotResult {
url: string;
title: string;
/** Compact accessibility-style tree for the model. */
tree: string;
elements: BrowserSnapshotNode[];
+ /** Display size the element boxes are relative to. */
+ screen?: { width: number; height: number };
fallback?: "computer_act";
error?: string;
}
@@ -811,6 +818,7 @@ export interface BrowserActResult {
/** Snapshot after the actions when available. */
tree?: string;
elements?: BrowserSnapshotNode[];
+ screen?: { width: number; height: number };
fallback?: "computer_act";
error?: string;
}
diff --git a/packages/adapters/src/browser-tools.ts b/packages/adapters/src/browser-tools.ts
index cbb4f78..adaa101 100644
--- a/packages/adapters/src/browser-tools.ts
+++ b/packages/adapters/src/browser-tools.ts
@@ -5,6 +5,14 @@ import type {
BrowserProvider,
ComputerRef,
} from "@rakazo/adapter-kit";
+import type { ComputerPointerPayload } from "@rakazo/contracts";
+import { type BrowserPointerTargets, withoutBrowserGeometry } from "./computer-pointer.js";
+
+/** Optional hooks that let the visible bot cursor follow page-browser actions. */
+export interface BrowserPointerHooks {
+ targets: BrowserPointerTargets;
+ announce: (payload: ComputerPointerPayload) => Promise;
+}
const MAX_BROWSER_ACTIONS = 24;
@@ -13,6 +21,7 @@ export async function browserNavigateFromTool(
computer: ComputerRef,
context: AdapterContext,
args: Record,
+ pointer?: BrowserPointerHooks,
) {
const url = String(args.url ?? "").trim();
try {
@@ -23,6 +32,7 @@ export async function browserNavigateFromTool(
fallback: "computer_act" as const,
};
}
+ pointer?.targets.forget();
const result = await browser.navigate(computer, { url, signal: context.signal }, context);
return formatBrowserResult(result);
} catch (error) {
@@ -39,11 +49,13 @@ export async function browserSnapshotFromTool(
computer: ComputerRef,
context: AdapterContext,
args: Record,
+ pointer?: BrowserPointerHooks,
) {
void args;
try {
const result = await browser.snapshot(computer, { signal: context.signal }, context);
- return formatBrowserResult(result);
+ pointer?.targets.remember(result);
+ return formatBrowserResult(withoutBrowserGeometry(result));
} catch (error) {
context.signal.throwIfAborted();
return {
@@ -58,11 +70,16 @@ export async function browserActFromTool(
computer: ComputerRef,
context: AdapterContext,
args: Record,
+ pointer?: BrowserPointerHooks,
) {
try {
const actions = parseBrowserActions(args.actions);
+ const announcement = pointer?.targets.payloadFor(actions);
+ if (announcement) await pointer?.announce(announcement);
const result = await browser.act(computer, { actions, signal: context.signal }, context);
- return formatBrowserResult(result);
+ if (result.elements) pointer?.targets.remember(result);
+ else pointer?.targets.forget();
+ return formatBrowserResult(withoutBrowserGeometry(result));
} catch (error) {
context.signal.throwIfAborted();
return {
diff --git a/packages/adapters/src/computer-browser.ts b/packages/adapters/src/computer-browser.ts
index 4faefd3..4f29d8c 100644
--- a/packages/adapters/src/computer-browser.ts
+++ b/packages/adapters/src/computer-browser.ts
@@ -115,6 +115,7 @@ export class ComputerBrowserProvider implements BrowserProvider {
title: typeof live.title === "string" ? live.title : "",
tree: typeof live.tree === "string" ? live.tree : formatTree(elements),
elements,
+ ...(live.screen ? { screen: live.screen } : {}),
};
}
@@ -150,6 +151,7 @@ export class ComputerBrowserProvider implements BrowserProvider {
title: typeof live.title === "string" ? live.title : "",
tree: typeof live.tree === "string" ? live.tree : elements ? formatTree(elements) : undefined,
elements,
+ ...(live.screen ? { screen: live.screen } : {}),
};
}
diff --git a/packages/adapters/src/computer-pointer.test.ts b/packages/adapters/src/computer-pointer.test.ts
new file mode 100644
index 0000000..566449e
--- /dev/null
+++ b/packages/adapters/src/computer-pointer.test.ts
@@ -0,0 +1,164 @@
+import type {
+ AdapterContext,
+ BrowserActRequest,
+ BrowserActResult,
+ BrowserProvider,
+ BrowserSnapshotResult,
+ ComputerRef,
+} from "@rakazo/adapter-kit";
+import { ComputerPointerPayload } from "@rakazo/contracts";
+import { DEFAULT_COMPUTER_SCREEN } from "@rakazo/core";
+import { describe, expect, it } from "vitest";
+import { browserActFromTool, browserSnapshotFromTool } from "./browser-tools.js";
+import {
+ BrowserPointerTargets,
+ computerPointerPayload,
+ computerPointerSteps,
+} from "./computer-pointer.js";
+import { parseComputerActions } from "./computer-tools.js";
+
+const context: AdapterContext = {
+ operationId: "1",
+ traceId: "1",
+ spaceId: "w",
+ userId: "u",
+ signal: new AbortController().signal,
+};
+const computer: ComputerRef = { id: "c1", botId: "b1", kind: "docker", providerRef: "c1" };
+
+describe("computer pointer steps", () => {
+ it("maps a computer_act batch to cursor steps in order", () => {
+ const steps = computerPointerSteps(
+ parseComputerActions([
+ { kind: "move", x: 10, y: 20 },
+ { kind: "click", x: 30, y: 40, double: true },
+ { kind: "click", x: 50, y: 60, button: "right" },
+ { kind: "type", text: "hello" },
+ { kind: "key", key: "Enter" },
+ { kind: "wait", ms: 100 },
+ { kind: "scroll", direction: "down" },
+ { kind: "down", x: 1, y: 2 },
+ { kind: "up", x: 3, y: 4 },
+ ]),
+ );
+ expect(steps).toEqual([
+ { action: "move", x: 10, y: 20 },
+ { action: "double_click", x: 30, y: 40 },
+ { action: "right_click", x: 50, y: 60 },
+ { action: "type" },
+ { action: "key" },
+ { action: "scroll" },
+ { action: "down", x: 1, y: 2 },
+ { action: "up", x: 3, y: 4 },
+ ]);
+ const payload = computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps);
+ expect(ComputerPointerPayload.parse(payload)).toEqual(payload);
+ });
+
+ it("keeps two separate clicks at different points apart", () => {
+ expect(
+ computerPointerSteps(
+ parseComputerActions([
+ { kind: "click", x: 1, y: 1 },
+ { kind: "click", x: 2, y: 2 },
+ ]),
+ ),
+ ).toEqual([
+ { action: "click", x: 1, y: 1 },
+ { action: "click", x: 2, y: 2 },
+ ]);
+ });
+
+ it("announces nothing for a batch without visible steps", () => {
+ const steps = computerPointerSteps(parseComputerActions([{ kind: "wait", ms: 10 }]));
+ expect(computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps)).toBeUndefined();
+ });
+});
+
+describe("browser pointer targets", () => {
+ it("points at element centers and stops at the first unknown ref", () => {
+ const targets = new BrowserPointerTargets();
+ targets.remember({
+ screen: { width: 1280, height: 800 },
+ elements: [
+ { ref: "e1", role: "textbox", name: "Name", box: [100, 200, 40, 20] },
+ { ref: "e2", role: "button", name: "Save", box: [300, 400, 11, 11] },
+ { ref: "e3", role: "button", name: "Hidden" },
+ // Untrusted helper output never becomes a cursor target.
+ { ref: "e4", role: "button", name: "Bad", box: [-5, 0, 10, 10] as never },
+ ],
+ });
+ expect(
+ targets.payloadFor([
+ { kind: "fill", ref: "e1", text: "x" },
+ { kind: "click", ref: "e2" },
+ { kind: "click", ref: "e3" },
+ { kind: "click", ref: "e1" },
+ ]),
+ ).toEqual({
+ kind: "pointer",
+ screen: { width: 1280, height: 800 },
+ steps: [
+ { action: "type", x: 120, y: 210 },
+ { action: "click", x: 306, y: 406 },
+ ],
+ });
+ expect(targets.payloadFor([{ kind: "click", ref: "e4" }])).toBeUndefined();
+ targets.forget();
+ expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
+ });
+
+ it("needs a reported screen size", () => {
+ const targets = new BrowserPointerTargets();
+ targets.remember({ elements: [{ ref: "e1", role: "button", name: "A", box: [1, 1, 2, 2] }] });
+ expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
+ });
+});
+
+describe("page browser tools with a visible cursor", () => {
+ it("announces before acting and keeps geometry away from the model", async () => {
+ const order: string[] = [];
+ const snapshot: BrowserSnapshotResult = {
+ url: "https://example.test/",
+ title: "Example",
+ tree: '- button "Save" [e1]',
+ screen: { width: 1280, height: 800 },
+ elements: [{ ref: "e1", role: "button", name: "Save", box: [10, 20, 30, 40] }],
+ };
+ const browser = {
+ snapshot: async () => snapshot,
+ act: async (_computer: ComputerRef, request: BrowserActRequest) => {
+ order.push(`act:${request.actions.length}`);
+ return { ok: true, completed: 1, ...snapshot } satisfies BrowserActResult;
+ },
+ } as unknown as BrowserProvider;
+ const announced: unknown[] = [];
+ const pointer = {
+ targets: new BrowserPointerTargets(),
+ announce: async (payload: ComputerPointerPayload) => {
+ order.push("announce");
+ announced.push(payload);
+ },
+ };
+
+ const seen = await browserSnapshotFromTool(browser, computer, context, {}, pointer);
+ expect(JSON.stringify(seen)).not.toMatch(/box|screen/);
+
+ const acted = await browserActFromTool(
+ browser,
+ computer,
+ context,
+ { actions: [{ kind: "click", ref: "e1" }] },
+ pointer,
+ );
+ expect(order).toEqual(["announce", "act:1"]);
+ expect(announced).toEqual([
+ {
+ kind: "pointer",
+ screen: { width: 1280, height: 800 },
+ steps: [{ action: "click", x: 25, y: 40 }],
+ },
+ ]);
+ expect(JSON.stringify(acted)).not.toMatch(/box|screen/);
+ });
+});
diff --git a/packages/adapters/src/computer-pointer.ts b/packages/adapters/src/computer-pointer.ts
new file mode 100644
index 0000000..8aa0a67
--- /dev/null
+++ b/packages/adapters/src/computer-pointer.ts
@@ -0,0 +1,130 @@
+import type {
+ BrowserActStep,
+ BrowserScreenBox,
+ BrowserSnapshotNode,
+ ComputerAction,
+} from "@rakazo/adapter-kit";
+import {
+ type ComputerPointerPayload,
+ type ComputerPointerStep,
+ MAX_COMPUTER_POINTER_STEPS,
+} from "@rakazo/contracts";
+
+export type ComputerScreenSize = ComputerPointerPayload["screen"];
+
+/** Pointer steps for a validated computer_act batch, in execution order. */
+export function computerPointerSteps(actions: ComputerAction[]): ComputerPointerStep[] {
+ const steps: ComputerPointerStep[] = [];
+ for (let index = 0; index < actions.length; index += 1) {
+ const action = actions[index]!;
+ if (action.kind === "pointer") {
+ const next = actions[index + 1];
+ // parseComputerActions expands a double click into two identical clicks.
+ if (
+ action.type === "click" &&
+ next?.kind === "pointer" &&
+ next.type === "click" &&
+ next.x === action.x &&
+ next.y === action.y &&
+ next.button === action.button
+ ) {
+ steps.push({ action: "double_click", x: action.x, y: action.y });
+ index += 1;
+ continue;
+ }
+ const kind =
+ action.type === "click"
+ ? action.button === "right"
+ ? "right_click"
+ : "click"
+ : action.type;
+ steps.push({ action: kind, x: action.x, y: action.y });
+ } else if (action.kind === "clipboard") {
+ steps.push({ action: "type" });
+ } else if (action.kind === "key" || action.kind === "scroll") {
+ steps.push({ action: action.kind });
+ }
+ }
+ return steps.slice(0, MAX_COMPUTER_POINTER_STEPS);
+}
+
+export function computerPointerPayload(
+ screen: ComputerScreenSize,
+ steps: ComputerPointerStep[],
+): ComputerPointerPayload | undefined {
+ if (steps.length === 0) return undefined;
+ return { kind: "pointer", screen, steps: steps.slice(0, MAX_COMPUTER_POINTER_STEPS) };
+}
+
+/**
+ * Remembers where the page browser last saw each element on screen, so a browser_act batch
+ * can be announced as pointer steps before it runs. Geometry never reaches the model.
+ */
+export class BrowserPointerTargets {
+ private boxes = new Map();
+ private screen: ComputerScreenSize | undefined;
+
+ remember(result: { elements?: BrowserSnapshotNode[]; screen?: ComputerScreenSize }) {
+ if (!Array.isArray(result.elements)) return;
+ this.boxes = new Map();
+ this.screen = validScreen(result.screen);
+ for (const element of result.elements) {
+ if (validBox(element.box)) this.boxes.set(element.ref, element.box);
+ }
+ }
+
+ forget() {
+ this.boxes = new Map();
+ this.screen = undefined;
+ }
+
+ payloadFor(actions: BrowserActStep[]): ComputerPointerPayload | undefined {
+ if (!this.screen) return undefined;
+ const steps: ComputerPointerStep[] = [];
+ for (const action of actions) {
+ const box = this.boxes.get(action.ref);
+ // Later steps may target elements an earlier step reveals; stop at the first unknown one.
+ if (!box) break;
+ const [x, y, width, height] = box;
+ steps.push({
+ action: action.kind === "click" ? "click" : "type",
+ x: Math.round(x + width / 2),
+ y: Math.round(y + height / 2),
+ });
+ }
+ return computerPointerPayload(this.screen, steps);
+ }
+}
+
+/** Strips screen geometry from page-browser results before they reach the model. */
+export function withoutBrowserGeometry(
+ result: T,
+): Omit {
+ const { screen: _screen, ...rest } = result as T & { screen?: unknown };
+ if (!Array.isArray(rest.elements)) return rest;
+ return {
+ ...rest,
+ elements: rest.elements.map(({ box: _box, ...element }) => element),
+ };
+}
+
+function validScreen(value: unknown): ComputerScreenSize | undefined {
+ if (!value || typeof value !== "object") return undefined;
+ const { width, height } = value as Record;
+ if (!isPixel(width) || !isPixel(height) || width === 0 || height === 0) return undefined;
+ return { width, height };
+}
+
+function validBox(value: unknown): value is BrowserScreenBox {
+ return (
+ Array.isArray(value) &&
+ value.length === 4 &&
+ value.every(isPixel) &&
+ (value[2] as number) > 0 &&
+ (value[3] as number) > 0
+ );
+}
+
+function isPixel(value: unknown): value is number {
+ return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100_000;
+}
diff --git a/packages/adapters/src/executor.ts b/packages/adapters/src/executor.ts
index f5c4142..95a88e2 100644
--- a/packages/adapters/src/executor.ts
+++ b/packages/adapters/src/executor.ts
@@ -25,7 +25,7 @@ import {
routineWakeupJob,
runContinueJob,
} from "@rakazo/adapter-kit";
-import type { ModelTier, RunStatus } from "@rakazo/contracts";
+import type { ComputerPointerPayload, ModelTier, RunStatus } from "@rakazo/contracts";
import {
ATTACHMENT_MAX_BYTES,
BOT_DESCRIPTION_MAX_LENGTH,
@@ -45,6 +45,7 @@ import {
applyJudgeDecision,
assertTransition,
botMessageAllowsSilence,
+ computerScreenSize,
connectorKindFromToolName,
containsSecret,
createStreamingRedactor,
@@ -165,6 +166,7 @@ import {
} from "./bot-secrets.js";
import { createBrowserProvider } from "./browser-provider-factory.js";
import {
+ type BrowserPointerHooks,
browserActFromTool,
browserNavigateFromTool,
browserSnapshotFromTool,
@@ -193,6 +195,11 @@ import {
renewComputerExecutionLease,
screenLeaseIdForRun,
} from "./computer-lifecycle.js";
+import {
+ BrowserPointerTargets,
+ computerPointerPayload,
+ computerPointerSteps,
+} from "./computer-pointer.js";
import { withComputerScreenAvailability } from "./computer-screens.js";
import {
assertTeamShellCommandAllowed,
@@ -1659,6 +1666,27 @@ export function createRunExecutor(deps: ExecutorDeps) {
lastComputerFrameId = observation.frameId;
return result;
};
+ // Best-effort: the visible cursor must never change a tool result or strand the run.
+ const announcePointer = async (payload: ComputerPointerPayload) => {
+ try {
+ await deps.events.append({
+ spaceId: run.spaceId,
+ threadId: thread.id,
+ botId: bot.id,
+ type: "thread.computer",
+ runId,
+ payload,
+ });
+ } catch (error) {
+ getLogger().warn("computer pointer append failed", {
+ error: sanitizeConnectorError(error, runSecrets),
+ });
+ }
+ };
+ const browserPointer: BrowserPointerHooks = {
+ targets: new BrowserPointerTargets(),
+ announce: announcePointer,
+ };
const pauseForApproval = () => {
approvalPausePending = true;
@@ -2214,10 +2242,16 @@ export function createRunExecutor(deps: ExecutorDeps) {
}
workspaceCheckpoint.markDirty();
return computerScreenToolResult(async () => {
+ const actions = parseComputerActions(args.actions);
+ const pointer = computerPointerPayload(
+ computerScreenSize(computer.kind),
+ computerPointerSteps(actions),
+ );
+ if (pointer) await announcePointer(pointer);
const result = await deps.sandbox.act(
computer,
{
- actions: parseComputerActions(args.actions),
+ actions,
observe: args.observe !== false,
settleMs: Number(args.settle_ms ?? 350),
},
@@ -2607,7 +2641,10 @@ export function createRunExecutor(deps: ExecutorDeps) {
: name === "browser_snapshot"
? browserSnapshotFromTool
: browserActFromTool;
- return computerScreenToolResult(() => tool(browser, computer, context, args), finish);
+ return computerScreenToolResult(
+ () => tool(browser, computer, context, args, browserPointer),
+ finish,
+ );
}
if (isMediaToolName(name)) {
diff --git a/packages/adapters/src/scripted-runtime.ts b/packages/adapters/src/scripted-runtime.ts
index 6a19544..32d5164 100644
--- a/packages/adapters/src/scripted-runtime.ts
+++ b/packages/adapters/src/scripted-runtime.ts
@@ -323,7 +323,11 @@ export function inferScript(
{
name: "computer_act",
args: {
- actions: [{ kind: "type", text: typed }],
+ // Click into the focused field first so the visible cursor has a target.
+ actions: [
+ { kind: "click", x: 640, y: 400 },
+ { kind: "type", text: typed },
+ ],
observe: true,
},
},
diff --git a/packages/contracts/src/computer-pointer.ts b/packages/contracts/src/computer-pointer.ts
new file mode 100644
index 0000000..d1299c0
--- /dev/null
+++ b/packages/contracts/src/computer-pointer.ts
@@ -0,0 +1,38 @@
+import * as z from "zod";
+
+export const ComputerPointerAction = z.enum([
+ "move",
+ "click",
+ "double_click",
+ "right_click",
+ "down",
+ "up",
+ "type",
+ "key",
+ "scroll",
+]);
+export type ComputerPointerAction = z.infer;
+
+/** Screen pixels; absent for keyboard steps that act wherever the pointer already is. */
+export const ComputerPointerStep = z.object({
+ action: ComputerPointerAction,
+ x: z.number().int().min(0).max(100_000).optional(),
+ y: z.number().int().min(0).max(100_000).optional(),
+});
+export type ComputerPointerStep = z.infer;
+
+export const MAX_COMPUTER_POINTER_STEPS = 48;
+
+/**
+ * Payload of a `thread.computer` event with `kind: "pointer"`: where a bot is about to point,
+ * click or type on its computer screen, so clients can draw its cursor over the live screen.
+ */
+export const ComputerPointerPayload = z.object({
+ kind: z.literal("pointer"),
+ screen: z.object({
+ width: z.number().int().positive().max(100_000),
+ height: z.number().int().positive().max(100_000),
+ }),
+ steps: z.array(ComputerPointerStep).min(1).max(MAX_COMPUTER_POINTER_STEPS),
+});
+export type ComputerPointerPayload = z.infer;
diff --git a/packages/contracts/src/desktop.ts b/packages/contracts/src/desktop.ts
index 53b78aa..8a601be 100644
--- a/packages/contracts/src/desktop.ts
+++ b/packages/contracts/src/desktop.ts
@@ -38,20 +38,39 @@ export interface RakazoDesktopOAuthCallback {
export interface RakazoDesktopBrowserPreviewState {
url: string;
- devServerRunning: boolean;
+ title: string;
+ loading: boolean;
+ canGoBack: boolean;
+ canGoForward: boolean;
+ /** The last main-frame load failure for `url`, cleared by the next successful load. */
+ error: string | null;
}
-/** Optional desktop bridge for the in-app browser preview panel. Absent in the browser. */
+/** Rectangle in CSS pixels of the app window's content area. */
+export interface RakazoDesktopBrowserBounds {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+}
+
+/**
+ * Optional desktop bridge for the in-app browser: a native Chromium view the main process lays
+ * over the panel the renderer reserves for it. Absent in the browser, where an iframe is used.
+ */
export interface RakazoDesktopBrowserPreview {
- getState: () => Promise;
+ /** Shows the view over these bounds; call again whenever the reserved area moves. */
+ show: (bounds: RakazoDesktopBrowserBounds) => Promise;
+ hide: () => Promise;
+ /** Only http(s) URLs load; anything else is rejected. */
navigate: (url: string) => Promise;
+ back: () => Promise;
+ forward: () => Promise;
reload: () => Promise;
- onNavigate: (listener: (url: string) => void) => () => void;
- onLoadingProgress: (listener: (progress: number) => void) => () => void;
- devServer: {
- start: () => Promise;
- stop: () => Promise;
- };
+ stop: () => Promise;
+ openDevTools: () => Promise;
+ state: () => Promise;
+ onState: (listener: (state: RakazoDesktopBrowserPreviewState) => void) => () => void;
}
export interface RakazoDesktop {
diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts
index 7b8f0a6..4103e68 100644
--- a/packages/contracts/src/index.ts
+++ b/packages/contracts/src/index.ts
@@ -1,6 +1,7 @@
export * from "./ai-consent.js";
export * from "./attachments.js";
export * from "./bot-secrets.js";
+export * from "./computer-pointer.js";
export * from "./desktop.js";
export * from "./domain.js";
export * from "./events.js";
diff --git a/packages/core/src/teach-recording.test.ts b/packages/core/src/teach-recording.test.ts
index 4efbfe2..4a80603 100644
--- a/packages/core/src/teach-recording.test.ts
+++ b/packages/core/src/teach-recording.test.ts
@@ -3,6 +3,7 @@ import {
computerInputForDomKey,
computerScreenSize,
mapTeachPointer,
+ screenPointInViewer,
teachCaptureKey,
} from "./teach-recording.js";
@@ -88,3 +89,21 @@ describe("mapTeachPointer", () => {
});
});
});
+
+describe("screenPointInViewer", () => {
+ it("projects through the same letterbox the viewer draws", () => {
+ const screen = { width: 1280, height: 800 };
+ // A wide viewer pillarboxes the 16:10 screen.
+ expect(screenPointInViewer({ x: 0, y: 0 }, { width: 1000, height: 400 }, screen)).toEqual({
+ x: 180,
+ y: 0,
+ });
+ expect(screenPointInViewer({ x: 1280, y: 800 }, { width: 1000, height: 400 }, screen)).toEqual({
+ x: 820,
+ y: 400,
+ });
+ const rect = { left: 0, top: 0, width: 640, height: 600 };
+ const projected = screenPointInViewer({ x: 640, y: 400 }, rect, screen);
+ expect(mapTeachPointer(projected.x, projected.y, rect, screen)).toEqual({ x: 640, y: 400 });
+ });
+});
diff --git a/packages/core/src/teach-recording.ts b/packages/core/src/teach-recording.ts
index 55e18a0..6cd622e 100644
--- a/packages/core/src/teach-recording.ts
+++ b/packages/core/src/teach-recording.ts
@@ -91,3 +91,18 @@ export function mapTeachPointer(
y: clamp(Math.round((clientY - rect.top - offsetY) / scale), screen.height - 1),
};
}
+
+/** Inverse of {@link mapTeachPointer}: where a screen pixel lands inside the letterboxed viewer. */
+export function screenPointInViewer(
+ point: { x: number; y: number },
+ viewer: { width: number; height: number },
+ screen: { width: number; height: number },
+): { x: number; y: number } {
+ const scale = Math.min(viewer.width / screen.width, viewer.height / screen.height) || 1;
+ const offsetX = (viewer.width - screen.width * scale) / 2;
+ const offsetY = (viewer.height - screen.height * scale) / 2;
+ return {
+ x: offsetX + clamp(point.x, screen.width) * scale,
+ y: offsetY + clamp(point.y, screen.height) * scale,
+ };
+}