evOo-local/apps/desktop/src/browser-preview.ts
cynarAI 49431def83
feat: show the bot's live cursor and make the in-app browser work (#113)
Bots now announce every click and typing step before it runs, and the
web app draws a cursor in the bot's colour that glides to that point
over the live computer screen, ripples on clicks and shows typing.

- contracts: ComputerPointerPayload on the existing thread.computer event
- executor: computer_act and browser_act announce pointer steps
  (best effort); the page-browser helper reports element screen boxes,
  which are stripped before results reach the model
- web: agentPointerFeed + AgentCursor over the side-panel preview and
  the full-window screen, mapped through the noVNC letterbox
- in-app browser: native WebContentsView on desktop (own session, http(s)
  only, clamped bounds, back/forward, reload, DevTools, load errors) and
  a sandboxed iframe on the web; replaces the dead panel, the fake
  50%/40% cursors and the unimplemented devServer bridge


Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi

Co-authored-by: Claude <noreply@anthropic.com>
2026-10-03 19:08:48 +02:00

190 lines
6.0 KiB
TypeScript

import type { RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
import { type BrowserWindow, WebContentsView } from "electron";
import {
previewBounds,
previewLoadError,
previewShortcut,
previewUrl,
} from "./browser-preview-policy.js";
/** Pages share one persistent profile, separate from the app's own session and cookies. */
const PREVIEW_PARTITION = "persist:rakazo-browser";
export const PREVIEW_STATE_CHANNEL = "desktop.browserPreview.state";
/**
* The in-app browser's native page: one WebContentsView laid over the panel the app window
* reserves. It belongs to one app window at a time and is torn down with it.
*/
export class BrowserPreviewController {
private host: BrowserWindow | null = null;
private view: WebContentsView | null = null;
private error: string | null = null;
state(win: BrowserWindow): RakazoDesktopBrowserPreviewState {
const contents = this.host === win ? this.view?.webContents : undefined;
if (!contents || contents.isDestroyed()) {
return {
url: "",
title: "",
loading: false,
canGoBack: false,
canGoForward: false,
error: null,
};
}
const url = contents.getURL();
return {
url,
title: contents.getTitle() === url ? "" : contents.getTitle(),
loading: contents.isLoading(),
canGoBack: contents.navigationHistory.canGoBack(),
canGoForward: contents.navigationHistory.canGoForward(),
error: this.error,
};
}
show(win: BrowserWindow, rawBounds: unknown) {
const view = this.viewFor(win);
const [width, height] = win.getContentSize();
const bounds = previewBounds(
rawBounds,
{ width: width ?? 0, height: height ?? 0 },
win.webContents.getZoomFactor(),
);
if (!bounds) {
view.setVisible(false);
return;
}
view.setBounds(bounds);
view.setVisible(true);
}
hide(win: BrowserWindow) {
if (this.host === win) this.view?.setVisible(false);
}
async navigate(win: BrowserWindow, rawUrl: unknown) {
const url = previewUrl(rawUrl);
if (!url) throw new Error("Only http(s) pages can open here.");
this.error = null;
// Failures surface through did-fail-load as page state, not as a rejected call.
await this.viewFor(win)
.webContents.loadURL(url)
.catch(() => undefined);
}
back(win: BrowserWindow) {
const history = this.contentsFor(win)?.navigationHistory;
if (history?.canGoBack()) history.goBack();
}
forward(win: BrowserWindow) {
const history = this.contentsFor(win)?.navigationHistory;
if (history?.canGoForward()) history.goForward();
}
reload(win: BrowserWindow, ignoreCache = false) {
const contents = this.contentsFor(win);
if (!contents?.getURL()) return;
this.error = null;
if (ignoreCache) contents.reloadIgnoringCache();
else contents.reload();
}
stop(win: BrowserWindow) {
this.contentsFor(win)?.stop();
}
openDevTools(win: BrowserWindow) {
const contents = this.contentsFor(win);
if (contents?.getURL()) contents.openDevTools({ mode: "detach" });
}
private contentsFor(win: BrowserWindow) {
const contents = this.host === win ? this.view?.webContents : undefined;
return contents && !contents.isDestroyed() ? contents : undefined;
}
private viewFor(win: BrowserWindow): WebContentsView {
if (this.host === win && this.view && !this.view.webContents.isDestroyed()) return this.view;
this.destroy();
const view = new WebContentsView({
webPreferences: {
partition: PREVIEW_PARTITION,
sandbox: true,
contextIsolation: true,
nodeIntegration: false,
webSecurity: true,
spellcheck: false,
},
});
view.setVisible(false);
const contents = view.webContents;
const publish = () => {
if (!win.isDestroyed()) win.webContents.send(PREVIEW_STATE_CHANNEL, this.state(win));
};
// One tab: links that want a new window open here instead.
contents.setWindowOpenHandler(({ url }) => {
const target = previewUrl(url);
if (target) void contents.loadURL(target).catch(() => undefined);
return { action: "deny" };
});
contents.on("will-navigate", (event, url) => {
if (!previewUrl(url)) event.preventDefault();
});
contents.on("will-redirect", (event, url) => {
if (!previewUrl(url)) event.preventDefault();
});
contents.on("did-start-loading", publish);
contents.on("did-stop-loading", publish);
contents.on("did-navigate", () => {
this.error = null;
publish();
});
contents.on("did-navigate-in-page", publish);
contents.on("page-title-updated", publish);
contents.on("did-fail-load", (_event, code, description, _url, isMainFrame) => {
if (!isMainFrame) return;
const error = previewLoadError(code, description);
if (error === null) return;
this.error = error;
publish();
});
contents.on("render-process-gone", (_event, details) => {
this.error = details.reason === "clean-exit" ? null : "The page stopped responding.";
publish();
});
contents.on("before-input-event", (event, input) => {
const shortcut = previewShortcut(input, process.platform);
if (!shortcut) return;
event.preventDefault();
if (shortcut === "reload" || shortcut === "hard-reload") {
this.reload(win, shortcut === "hard-reload");
} else if (shortcut === "back") {
this.back(win);
} else if (shortcut === "forward") {
this.forward(win);
} else {
this.openDevTools(win);
}
});
win.contentView.addChildView(view);
win.once("closed", () => {
if (this.host === win) this.destroy();
});
this.host = win;
this.view = view;
return view;
}
private destroy() {
const { host, view } = this;
this.host = null;
this.view = null;
this.error = null;
if (!view) return;
if (host && !host.isDestroyed()) host.contentView.removeChildView(view);
if (!view.webContents.isDestroyed()) view.webContents.close();
}
}