Bots now announce every click and typing step before it runs, and the web app draws a cursor in the bot's colour that glides to that point over the live computer screen, ripples on clicks and shows typing. - contracts: ComputerPointerPayload on the existing thread.computer event - executor: computer_act and browser_act announce pointer steps (best effort); the page-browser helper reports element screen boxes, which are stripped before results reach the model - web: agentPointerFeed + AgentCursor over the side-panel preview and the full-window screen, mapped through the noVNC letterbox - in-app browser: native WebContentsView on desktop (own session, http(s) only, clamped bounds, back/forward, reload, DevTools, load errors) and a sandboxed iframe on the web; replaces the dead panel, the fake 50%/40% cursors and the unimplemented devServer bridge Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi Co-authored-by: Claude <noreply@anthropic.com>
190 lines
6.0 KiB
TypeScript
190 lines
6.0 KiB
TypeScript
import type { RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
|
|
import { type BrowserWindow, WebContentsView } from "electron";
|
|
import {
|
|
previewBounds,
|
|
previewLoadError,
|
|
previewShortcut,
|
|
previewUrl,
|
|
} from "./browser-preview-policy.js";
|
|
|
|
/** Pages share one persistent profile, separate from the app's own session and cookies. */
|
|
const PREVIEW_PARTITION = "persist:rakazo-browser";
|
|
export const PREVIEW_STATE_CHANNEL = "desktop.browserPreview.state";
|
|
|
|
/**
|
|
* The in-app browser's native page: one WebContentsView laid over the panel the app window
|
|
* reserves. It belongs to one app window at a time and is torn down with it.
|
|
*/
|
|
export class BrowserPreviewController {
|
|
private host: BrowserWindow | null = null;
|
|
private view: WebContentsView | null = null;
|
|
private error: string | null = null;
|
|
|
|
state(win: BrowserWindow): RakazoDesktopBrowserPreviewState {
|
|
const contents = this.host === win ? this.view?.webContents : undefined;
|
|
if (!contents || contents.isDestroyed()) {
|
|
return {
|
|
url: "",
|
|
title: "",
|
|
loading: false,
|
|
canGoBack: false,
|
|
canGoForward: false,
|
|
error: null,
|
|
};
|
|
}
|
|
const url = contents.getURL();
|
|
return {
|
|
url,
|
|
title: contents.getTitle() === url ? "" : contents.getTitle(),
|
|
loading: contents.isLoading(),
|
|
canGoBack: contents.navigationHistory.canGoBack(),
|
|
canGoForward: contents.navigationHistory.canGoForward(),
|
|
error: this.error,
|
|
};
|
|
}
|
|
|
|
show(win: BrowserWindow, rawBounds: unknown) {
|
|
const view = this.viewFor(win);
|
|
const [width, height] = win.getContentSize();
|
|
const bounds = previewBounds(
|
|
rawBounds,
|
|
{ width: width ?? 0, height: height ?? 0 },
|
|
win.webContents.getZoomFactor(),
|
|
);
|
|
if (!bounds) {
|
|
view.setVisible(false);
|
|
return;
|
|
}
|
|
view.setBounds(bounds);
|
|
view.setVisible(true);
|
|
}
|
|
|
|
hide(win: BrowserWindow) {
|
|
if (this.host === win) this.view?.setVisible(false);
|
|
}
|
|
|
|
async navigate(win: BrowserWindow, rawUrl: unknown) {
|
|
const url = previewUrl(rawUrl);
|
|
if (!url) throw new Error("Only http(s) pages can open here.");
|
|
this.error = null;
|
|
// Failures surface through did-fail-load as page state, not as a rejected call.
|
|
await this.viewFor(win)
|
|
.webContents.loadURL(url)
|
|
.catch(() => undefined);
|
|
}
|
|
|
|
back(win: BrowserWindow) {
|
|
const history = this.contentsFor(win)?.navigationHistory;
|
|
if (history?.canGoBack()) history.goBack();
|
|
}
|
|
|
|
forward(win: BrowserWindow) {
|
|
const history = this.contentsFor(win)?.navigationHistory;
|
|
if (history?.canGoForward()) history.goForward();
|
|
}
|
|
|
|
reload(win: BrowserWindow, ignoreCache = false) {
|
|
const contents = this.contentsFor(win);
|
|
if (!contents?.getURL()) return;
|
|
this.error = null;
|
|
if (ignoreCache) contents.reloadIgnoringCache();
|
|
else contents.reload();
|
|
}
|
|
|
|
stop(win: BrowserWindow) {
|
|
this.contentsFor(win)?.stop();
|
|
}
|
|
|
|
openDevTools(win: BrowserWindow) {
|
|
const contents = this.contentsFor(win);
|
|
if (contents?.getURL()) contents.openDevTools({ mode: "detach" });
|
|
}
|
|
|
|
private contentsFor(win: BrowserWindow) {
|
|
const contents = this.host === win ? this.view?.webContents : undefined;
|
|
return contents && !contents.isDestroyed() ? contents : undefined;
|
|
}
|
|
|
|
private viewFor(win: BrowserWindow): WebContentsView {
|
|
if (this.host === win && this.view && !this.view.webContents.isDestroyed()) return this.view;
|
|
this.destroy();
|
|
const view = new WebContentsView({
|
|
webPreferences: {
|
|
partition: PREVIEW_PARTITION,
|
|
sandbox: true,
|
|
contextIsolation: true,
|
|
nodeIntegration: false,
|
|
webSecurity: true,
|
|
spellcheck: false,
|
|
},
|
|
});
|
|
view.setVisible(false);
|
|
const contents = view.webContents;
|
|
const publish = () => {
|
|
if (!win.isDestroyed()) win.webContents.send(PREVIEW_STATE_CHANNEL, this.state(win));
|
|
};
|
|
// One tab: links that want a new window open here instead.
|
|
contents.setWindowOpenHandler(({ url }) => {
|
|
const target = previewUrl(url);
|
|
if (target) void contents.loadURL(target).catch(() => undefined);
|
|
return { action: "deny" };
|
|
});
|
|
contents.on("will-navigate", (event, url) => {
|
|
if (!previewUrl(url)) event.preventDefault();
|
|
});
|
|
contents.on("will-redirect", (event, url) => {
|
|
if (!previewUrl(url)) event.preventDefault();
|
|
});
|
|
contents.on("did-start-loading", publish);
|
|
contents.on("did-stop-loading", publish);
|
|
contents.on("did-navigate", () => {
|
|
this.error = null;
|
|
publish();
|
|
});
|
|
contents.on("did-navigate-in-page", publish);
|
|
contents.on("page-title-updated", publish);
|
|
contents.on("did-fail-load", (_event, code, description, _url, isMainFrame) => {
|
|
if (!isMainFrame) return;
|
|
const error = previewLoadError(code, description);
|
|
if (error === null) return;
|
|
this.error = error;
|
|
publish();
|
|
});
|
|
contents.on("render-process-gone", (_event, details) => {
|
|
this.error = details.reason === "clean-exit" ? null : "The page stopped responding.";
|
|
publish();
|
|
});
|
|
contents.on("before-input-event", (event, input) => {
|
|
const shortcut = previewShortcut(input, process.platform);
|
|
if (!shortcut) return;
|
|
event.preventDefault();
|
|
if (shortcut === "reload" || shortcut === "hard-reload") {
|
|
this.reload(win, shortcut === "hard-reload");
|
|
} else if (shortcut === "back") {
|
|
this.back(win);
|
|
} else if (shortcut === "forward") {
|
|
this.forward(win);
|
|
} else {
|
|
this.openDevTools(win);
|
|
}
|
|
});
|
|
win.contentView.addChildView(view);
|
|
win.once("closed", () => {
|
|
if (this.host === win) this.destroy();
|
|
});
|
|
this.host = win;
|
|
this.view = view;
|
|
return view;
|
|
}
|
|
|
|
private destroy() {
|
|
const { host, view } = this;
|
|
this.host = null;
|
|
this.view = null;
|
|
this.error = null;
|
|
if (!view) return;
|
|
if (host && !host.isDestroyed()) host.contentView.removeChildView(view);
|
|
if (!view.webContents.isDestroyed()) view.webContents.close();
|
|
}
|
|
}
|