Merge the updated legacy cleanup (#109, with main) into the Phase 2 integration branch
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
commit
09ed6c2136
155
apps/desktop/e2e/browser-preview.spec.ts
Normal file
155
apps/desktop/e2e/browser-preview.spec.ts
Normal file
@ -0,0 +1,155 @@
|
||||
import http from "node:http";
|
||||
import path from "node:path";
|
||||
import { _electron as electron, expect, test } from "@playwright/test";
|
||||
import type { RakazoDesktop, RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
|
||||
|
||||
const fixture = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>Rakazo desktop browser</title></head>
|
||||
<body><main>App with a browser panel</main></body>
|
||||
</html>`;
|
||||
|
||||
type Bridge = NonNullable<RakazoDesktop["browserPreview"]>;
|
||||
|
||||
test("the in-app browser is a native view the app window lays over its panel", async () => {
|
||||
const pages = http.createServer((req, res) => {
|
||||
res.setHeader("content-type", "text/html");
|
||||
if (req.url === "/second") {
|
||||
res.end("<!doctype html><title>Second page</title><p>second</p>");
|
||||
return;
|
||||
}
|
||||
res.end(`<!doctype html><title>First page</title>
|
||||
<a id="next" href="/second">next</a>
|
||||
<a id="popup" href="/second" target="_blank">popup</a>
|
||||
<a id="file" href="file:///etc/hosts">file</a>`);
|
||||
});
|
||||
await new Promise<void>((resolve) => pages.listen(0, "127.0.0.1", resolve));
|
||||
const address = pages.address();
|
||||
if (!address || typeof address === "string") throw new Error("No fixture server");
|
||||
const origin = `http://127.0.0.1:${address.port}`;
|
||||
|
||||
const app = await electron.launch({
|
||||
args: ["."],
|
||||
cwd: path.resolve(import.meta.dirname, ".."),
|
||||
env: {
|
||||
...process.env,
|
||||
RAKAZO_WEB_URL: `data:text/html;charset=utf-8,${encodeURIComponent(fixture)}`,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const page = await app.firstWindow();
|
||||
await expect(page.getByText("App with a browser panel")).toBeVisible();
|
||||
|
||||
const call = <K extends keyof Bridge>(method: K, ...args: unknown[]) =>
|
||||
page.evaluate(
|
||||
([name, values]) => {
|
||||
const bridge = (window as typeof window & { rakazoDesktop?: RakazoDesktop }).rakazoDesktop
|
||||
?.browserPreview as unknown as Record<
|
||||
string,
|
||||
(...input: unknown[]) => Promise<unknown>
|
||||
>;
|
||||
return bridge[name as string]!(...(values as unknown[]));
|
||||
},
|
||||
[method, args] as const,
|
||||
);
|
||||
const state = () => call("state") as Promise<RakazoDesktopBrowserPreviewState>;
|
||||
const view = () =>
|
||||
app.evaluate(({ BrowserWindow }) => {
|
||||
const win = BrowserWindow.getAllWindows()[0]!;
|
||||
const child = win.contentView.children[0] as
|
||||
| (Electron.View & { webContents?: Electron.WebContents })
|
||||
| undefined;
|
||||
return child
|
||||
? {
|
||||
count: win.contentView.children.length,
|
||||
visible: child.getVisible(),
|
||||
bounds: child.getBounds(),
|
||||
url: child.webContents?.getURL(),
|
||||
partition: child.webContents?.session === win.webContents.session,
|
||||
sandbox: child.webContents?.getLastWebPreferences()?.sandbox,
|
||||
nodeIntegration: child.webContents?.getLastWebPreferences()?.nodeIntegration,
|
||||
}
|
||||
: null;
|
||||
});
|
||||
|
||||
// Pages load in a native view in their own session, never the app's.
|
||||
await call("navigate", `${origin}/`);
|
||||
await expect.poll(async () => (await state()).title).toBe("First page");
|
||||
expect(await view()).toMatchObject({
|
||||
count: 1,
|
||||
visible: false,
|
||||
url: `${origin}/`,
|
||||
partition: false,
|
||||
sandbox: true,
|
||||
nodeIntegration: false,
|
||||
});
|
||||
|
||||
await call("show", { x: 400, y: 50, width: 300, height: 200 });
|
||||
expect(await view()).toMatchObject({
|
||||
visible: true,
|
||||
bounds: { x: 400, y: 50, width: 300, height: 200 },
|
||||
});
|
||||
// Renderer bounds outside the window are clamped to it.
|
||||
await call("show", { x: -10, y: -10, width: 100_000, height: 100_000 });
|
||||
const [width, height] = await app.evaluate(({ BrowserWindow }) =>
|
||||
BrowserWindow.getAllWindows()[0]!.getContentSize(),
|
||||
);
|
||||
expect((await view())?.bounds).toEqual({ x: 0, y: 0, width, height });
|
||||
|
||||
const contents = () =>
|
||||
app.evaluate(({ BrowserWindow }, script) => {
|
||||
const win = BrowserWindow.getAllWindows()[0]!;
|
||||
const child = win.contentView.children[0] as Electron.View & {
|
||||
webContents: Electron.WebContents;
|
||||
};
|
||||
return child.webContents.executeJavaScript(script, true);
|
||||
}, "document.getElementById('next').click()");
|
||||
await contents();
|
||||
await expect.poll(async () => (await state()).title).toBe("Second page");
|
||||
expect(await state()).toMatchObject({ canGoBack: true, canGoForward: false, error: null });
|
||||
|
||||
await call("back");
|
||||
await expect.poll(async () => (await state()).title).toBe("First page");
|
||||
expect(await state()).toMatchObject({ canGoForward: true });
|
||||
await call("forward");
|
||||
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
|
||||
|
||||
// New-window links stay in the one view; file links never load.
|
||||
await call("back");
|
||||
await expect.poll(async () => (await state()).title).toBe("First page");
|
||||
await app.evaluate(({ BrowserWindow }) => {
|
||||
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
|
||||
webContents: Electron.WebContents;
|
||||
};
|
||||
return child.webContents.executeJavaScript("document.getElementById('popup').click()", true);
|
||||
});
|
||||
await expect.poll(async () => (await state()).url).toBe(`${origin}/second`);
|
||||
expect(await app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows().length)).toBe(1);
|
||||
await call("back");
|
||||
await expect.poll(async () => (await state()).title).toBe("First page");
|
||||
await app.evaluate(({ BrowserWindow }) => {
|
||||
const child = BrowserWindow.getAllWindows()[0]!.contentView.children[0] as Electron.View & {
|
||||
webContents: Electron.WebContents;
|
||||
};
|
||||
return child.webContents.executeJavaScript("document.getElementById('file').click()", true);
|
||||
});
|
||||
await page.waitForTimeout(300);
|
||||
expect((await state()).url).toBe(`${origin}/`);
|
||||
|
||||
await expect(call("navigate", "file:///etc/hosts")).rejects.toThrow(/http/);
|
||||
|
||||
// A dev server that is not running yet shows as an error, then recovers on reload.
|
||||
pages.close();
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
await call("navigate", `${origin}/gone`);
|
||||
await expect.poll(async () => (await state()).error).toMatch(/ERR_CONNECTION_REFUSED/);
|
||||
|
||||
await call("hide");
|
||||
expect((await view())?.visible).toBe(false);
|
||||
} finally {
|
||||
await app.close();
|
||||
pages.closeAllConnections();
|
||||
pages.close();
|
||||
}
|
||||
});
|
||||
@ -42,6 +42,7 @@ test("launches with a narrow preload bridge and an isolated renderer", async ()
|
||||
});
|
||||
|
||||
expect(renderer.bridgeKeys).toEqual([
|
||||
"browserPreview",
|
||||
"files",
|
||||
"localSettings",
|
||||
"oauth",
|
||||
|
||||
96
apps/desktop/src/browser-preview-policy.test.ts
Normal file
96
apps/desktop/src/browser-preview-policy.test.ts
Normal file
@ -0,0 +1,96 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
previewBounds,
|
||||
previewLoadError,
|
||||
previewShortcut,
|
||||
previewUrl,
|
||||
} from "./browser-preview-policy.js";
|
||||
|
||||
describe("previewUrl", () => {
|
||||
it("accepts only http(s) without credentials", () => {
|
||||
expect(previewUrl("http://localhost:5173")).toBe("http://localhost:5173/");
|
||||
expect(previewUrl("https://example.com/a?b=1")).toBe("https://example.com/a?b=1");
|
||||
for (const value of [
|
||||
"file:///etc/hosts",
|
||||
"javascript:alert(1)",
|
||||
"chrome://settings",
|
||||
"https://user:fake-password@example.com",
|
||||
"not a url",
|
||||
42,
|
||||
`https://example.com/${"a".repeat(9000)}`,
|
||||
]) {
|
||||
expect(previewUrl(value)).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("previewBounds", () => {
|
||||
const content = { width: 1200, height: 800 };
|
||||
|
||||
it("rounds and applies the app zoom factor", () => {
|
||||
expect(previewBounds({ x: 800.4, y: 40.6, width: 399.5, height: 700 }, content)).toEqual({
|
||||
x: 800,
|
||||
y: 41,
|
||||
width: 400,
|
||||
height: 700,
|
||||
});
|
||||
expect(previewBounds({ x: 100, y: 10, width: 200, height: 100 }, content, 1.25)).toEqual({
|
||||
x: 125,
|
||||
y: 13,
|
||||
width: 250,
|
||||
height: 125,
|
||||
});
|
||||
});
|
||||
|
||||
it("clamps to the window and rejects empty or malformed rectangles", () => {
|
||||
expect(previewBounds({ x: -50, y: -10, width: 5000, height: 5000 }, content)).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 1200,
|
||||
height: 800,
|
||||
});
|
||||
expect(previewBounds({ x: 1300, y: 0, width: 100, height: 100 }, content)).toBeNull();
|
||||
expect(previewBounds({ x: 0, y: 0, width: 0, height: 100 }, content)).toBeNull();
|
||||
expect(previewBounds({ x: "1", y: 0, width: 1, height: 1 }, content)).toBeNull();
|
||||
expect(previewBounds({ x: Number.NaN, y: 0, width: 1, height: 1 }, content)).toBeNull();
|
||||
expect(previewBounds(null, content)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("previewShortcut", () => {
|
||||
const key = (overrides: Partial<Parameters<typeof previewShortcut>[0]>) => ({
|
||||
type: "keyDown",
|
||||
key: "r",
|
||||
control: false,
|
||||
meta: false,
|
||||
alt: false,
|
||||
shift: false,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
it("maps platform browser shortcuts", () => {
|
||||
expect(previewShortcut(key({ meta: true }), "darwin")).toBe("reload");
|
||||
expect(previewShortcut(key({ meta: true, shift: true, key: "R" }), "darwin")).toBe(
|
||||
"hard-reload",
|
||||
);
|
||||
expect(previewShortcut(key({ control: true }), "linux")).toBe("reload");
|
||||
expect(previewShortcut(key({ meta: true, key: "[" }), "darwin")).toBe("back");
|
||||
expect(previewShortcut(key({ alt: true, key: "ArrowRight" }), "win32")).toBe("forward");
|
||||
expect(previewShortcut(key({ key: "F12" }), "linux")).toBe("devtools");
|
||||
expect(previewShortcut(key({ key: "F5" }), "win32")).toBe("reload");
|
||||
});
|
||||
|
||||
it("leaves typing alone", () => {
|
||||
expect(previewShortcut(key({}), "darwin")).toBeNull();
|
||||
expect(previewShortcut(key({ control: true }), "darwin")).toBeNull();
|
||||
expect(previewShortcut(key({ type: "keyUp", meta: true }), "darwin")).toBeNull();
|
||||
expect(previewShortcut(key({ key: "ArrowLeft" }), "linux")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("previewLoadError", () => {
|
||||
it("ignores superseded navigations", () => {
|
||||
expect(previewLoadError(-3, "ERR_ABORTED")).toBeNull();
|
||||
expect(previewLoadError(-102, "ERR_CONNECTION_REFUSED")).toBe("ERR_CONNECTION_REFUSED");
|
||||
});
|
||||
});
|
||||
76
apps/desktop/src/browser-preview-policy.ts
Normal file
76
apps/desktop/src/browser-preview-policy.ts
Normal file
@ -0,0 +1,76 @@
|
||||
import type { RakazoDesktopBrowserBounds } from "@rakazo/contracts";
|
||||
|
||||
/** Only plain http(s) pages load in the in-app browser; no files, schemes or URL credentials. */
|
||||
export function previewUrl(value: unknown): string | null {
|
||||
if (typeof value !== "string" || value.length > 8192) return null;
|
||||
try {
|
||||
const url = new URL(value);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
||||
if (url.username || url.password) return null;
|
||||
return url.toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renderer-reported CSS-pixel bounds, converted to window DIPs and clamped to the content area
|
||||
* so a compromised renderer cannot place the view outside the panel it reserved.
|
||||
*/
|
||||
export function previewBounds(
|
||||
value: unknown,
|
||||
content: { width: number; height: number },
|
||||
zoomFactor = 1,
|
||||
): RakazoDesktopBrowserBounds | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const raw = value as Record<string, unknown>;
|
||||
const numbers = [raw.x, raw.y, raw.width, raw.height];
|
||||
if (!numbers.every((n) => typeof n === "number" && Number.isFinite(n))) return null;
|
||||
const scale = Number.isFinite(zoomFactor) && zoomFactor > 0 ? zoomFactor : 1;
|
||||
const [x, y, width, height] = (numbers as number[]).map((n) => Math.round(n * scale)) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
const left = Math.min(Math.max(0, x), content.width);
|
||||
const top = Math.min(Math.max(0, y), content.height);
|
||||
const right = Math.min(Math.max(left, x + width), content.width);
|
||||
const bottom = Math.min(Math.max(top, y + height), content.height);
|
||||
if (right - left < 1 || bottom - top < 1) return null;
|
||||
return { x: left, y: top, width: right - left, height: bottom - top };
|
||||
}
|
||||
|
||||
export type PreviewShortcut = "reload" | "hard-reload" | "back" | "forward" | "devtools";
|
||||
|
||||
/** Browser keyboard shortcuts while focus is inside the page. */
|
||||
export function previewShortcut(
|
||||
input: {
|
||||
type: string;
|
||||
key: string;
|
||||
control: boolean;
|
||||
meta: boolean;
|
||||
alt: boolean;
|
||||
shift: boolean;
|
||||
},
|
||||
platform: string,
|
||||
): PreviewShortcut | null {
|
||||
if (input.type !== "keyDown") return null;
|
||||
const mod = platform === "darwin" ? input.meta : input.control;
|
||||
const key = input.key.length === 1 ? input.key.toLowerCase() : input.key;
|
||||
if (key === "F5") return input.shift ? "hard-reload" : "reload";
|
||||
if (key === "F12") return "devtools";
|
||||
if (mod && key === "r") return input.shift ? "hard-reload" : "reload";
|
||||
if (mod && input.alt && key === "i") return "devtools";
|
||||
if (platform === "darwin" ? input.meta && !input.alt : input.alt && !input.control) {
|
||||
if (key === (platform === "darwin" ? "[" : "ArrowLeft")) return "back";
|
||||
if (key === (platform === "darwin" ? "]" : "ArrowRight")) return "forward";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Load failures worth showing; an aborted load (-3) is a superseded navigation, not an error. */
|
||||
export function previewLoadError(errorCode: number, description: string): string | null {
|
||||
if (errorCode === -3 || errorCode === 0) return null;
|
||||
return description || `Load failed (${errorCode})`;
|
||||
}
|
||||
189
apps/desktop/src/browser-preview.ts
Normal file
189
apps/desktop/src/browser-preview.ts
Normal file
@ -0,0 +1,189 @@
|
||||
import type { RakazoDesktopBrowserPreviewState } from "@rakazo/contracts";
|
||||
import { type BrowserWindow, WebContentsView } from "electron";
|
||||
import {
|
||||
previewBounds,
|
||||
previewLoadError,
|
||||
previewShortcut,
|
||||
previewUrl,
|
||||
} from "./browser-preview-policy.js";
|
||||
|
||||
/** Pages share one persistent profile, separate from the app's own session and cookies. */
|
||||
const PREVIEW_PARTITION = "persist:rakazo-browser";
|
||||
export const PREVIEW_STATE_CHANNEL = "desktop.browserPreview.state";
|
||||
|
||||
/**
|
||||
* The in-app browser's native page: one WebContentsView laid over the panel the app window
|
||||
* reserves. It belongs to one app window at a time and is torn down with it.
|
||||
*/
|
||||
export class BrowserPreviewController {
|
||||
private host: BrowserWindow | null = null;
|
||||
private view: WebContentsView | null = null;
|
||||
private error: string | null = null;
|
||||
|
||||
state(win: BrowserWindow): RakazoDesktopBrowserPreviewState {
|
||||
const contents = this.host === win ? this.view?.webContents : undefined;
|
||||
if (!contents || contents.isDestroyed()) {
|
||||
return {
|
||||
url: "",
|
||||
title: "",
|
||||
loading: false,
|
||||
canGoBack: false,
|
||||
canGoForward: false,
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
const url = contents.getURL();
|
||||
return {
|
||||
url,
|
||||
title: contents.getTitle() === url ? "" : contents.getTitle(),
|
||||
loading: contents.isLoading(),
|
||||
canGoBack: contents.navigationHistory.canGoBack(),
|
||||
canGoForward: contents.navigationHistory.canGoForward(),
|
||||
error: this.error,
|
||||
};
|
||||
}
|
||||
|
||||
show(win: BrowserWindow, rawBounds: unknown) {
|
||||
const view = this.viewFor(win);
|
||||
const [width, height] = win.getContentSize();
|
||||
const bounds = previewBounds(
|
||||
rawBounds,
|
||||
{ width: width ?? 0, height: height ?? 0 },
|
||||
win.webContents.getZoomFactor(),
|
||||
);
|
||||
if (!bounds) {
|
||||
view.setVisible(false);
|
||||
return;
|
||||
}
|
||||
view.setBounds(bounds);
|
||||
view.setVisible(true);
|
||||
}
|
||||
|
||||
hide(win: BrowserWindow) {
|
||||
if (this.host === win) this.view?.setVisible(false);
|
||||
}
|
||||
|
||||
async navigate(win: BrowserWindow, rawUrl: unknown) {
|
||||
const url = previewUrl(rawUrl);
|
||||
if (!url) throw new Error("Only http(s) pages can open here.");
|
||||
this.error = null;
|
||||
// Failures surface through did-fail-load as page state, not as a rejected call.
|
||||
await this.viewFor(win)
|
||||
.webContents.loadURL(url)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
back(win: BrowserWindow) {
|
||||
const history = this.contentsFor(win)?.navigationHistory;
|
||||
if (history?.canGoBack()) history.goBack();
|
||||
}
|
||||
|
||||
forward(win: BrowserWindow) {
|
||||
const history = this.contentsFor(win)?.navigationHistory;
|
||||
if (history?.canGoForward()) history.goForward();
|
||||
}
|
||||
|
||||
reload(win: BrowserWindow, ignoreCache = false) {
|
||||
const contents = this.contentsFor(win);
|
||||
if (!contents?.getURL()) return;
|
||||
this.error = null;
|
||||
if (ignoreCache) contents.reloadIgnoringCache();
|
||||
else contents.reload();
|
||||
}
|
||||
|
||||
stop(win: BrowserWindow) {
|
||||
this.contentsFor(win)?.stop();
|
||||
}
|
||||
|
||||
openDevTools(win: BrowserWindow) {
|
||||
const contents = this.contentsFor(win);
|
||||
if (contents?.getURL()) contents.openDevTools({ mode: "detach" });
|
||||
}
|
||||
|
||||
private contentsFor(win: BrowserWindow) {
|
||||
const contents = this.host === win ? this.view?.webContents : undefined;
|
||||
return contents && !contents.isDestroyed() ? contents : undefined;
|
||||
}
|
||||
|
||||
private viewFor(win: BrowserWindow): WebContentsView {
|
||||
if (this.host === win && this.view && !this.view.webContents.isDestroyed()) return this.view;
|
||||
this.destroy();
|
||||
const view = new WebContentsView({
|
||||
webPreferences: {
|
||||
partition: PREVIEW_PARTITION,
|
||||
sandbox: true,
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
webSecurity: true,
|
||||
spellcheck: false,
|
||||
},
|
||||
});
|
||||
view.setVisible(false);
|
||||
const contents = view.webContents;
|
||||
const publish = () => {
|
||||
if (!win.isDestroyed()) win.webContents.send(PREVIEW_STATE_CHANNEL, this.state(win));
|
||||
};
|
||||
// One tab: links that want a new window open here instead.
|
||||
contents.setWindowOpenHandler(({ url }) => {
|
||||
const target = previewUrl(url);
|
||||
if (target) void contents.loadURL(target).catch(() => undefined);
|
||||
return { action: "deny" };
|
||||
});
|
||||
contents.on("will-navigate", (event, url) => {
|
||||
if (!previewUrl(url)) event.preventDefault();
|
||||
});
|
||||
contents.on("will-redirect", (event, url) => {
|
||||
if (!previewUrl(url)) event.preventDefault();
|
||||
});
|
||||
contents.on("did-start-loading", publish);
|
||||
contents.on("did-stop-loading", publish);
|
||||
contents.on("did-navigate", () => {
|
||||
this.error = null;
|
||||
publish();
|
||||
});
|
||||
contents.on("did-navigate-in-page", publish);
|
||||
contents.on("page-title-updated", publish);
|
||||
contents.on("did-fail-load", (_event, code, description, _url, isMainFrame) => {
|
||||
if (!isMainFrame) return;
|
||||
const error = previewLoadError(code, description);
|
||||
if (error === null) return;
|
||||
this.error = error;
|
||||
publish();
|
||||
});
|
||||
contents.on("render-process-gone", (_event, details) => {
|
||||
this.error = details.reason === "clean-exit" ? null : "The page stopped responding.";
|
||||
publish();
|
||||
});
|
||||
contents.on("before-input-event", (event, input) => {
|
||||
const shortcut = previewShortcut(input, process.platform);
|
||||
if (!shortcut) return;
|
||||
event.preventDefault();
|
||||
if (shortcut === "reload" || shortcut === "hard-reload") {
|
||||
this.reload(win, shortcut === "hard-reload");
|
||||
} else if (shortcut === "back") {
|
||||
this.back(win);
|
||||
} else if (shortcut === "forward") {
|
||||
this.forward(win);
|
||||
} else {
|
||||
this.openDevTools(win);
|
||||
}
|
||||
});
|
||||
win.contentView.addChildView(view);
|
||||
win.once("closed", () => {
|
||||
if (this.host === win) this.destroy();
|
||||
});
|
||||
this.host = win;
|
||||
this.view = view;
|
||||
return view;
|
||||
}
|
||||
|
||||
private destroy() {
|
||||
const { host, view } = this;
|
||||
this.host = null;
|
||||
this.view = null;
|
||||
this.error = null;
|
||||
if (!view) return;
|
||||
if (host && !host.isDestroyed()) host.contentView.removeChildView(view);
|
||||
if (!view.webContents.isDestroyed()) view.webContents.close();
|
||||
}
|
||||
}
|
||||
@ -21,6 +21,7 @@ import {
|
||||
LAUNCH_CHECK_DELAY_MS,
|
||||
} from "./auto-update.js";
|
||||
import { openBrowserAuth } from "./browser-auth.js";
|
||||
import { BrowserPreviewController } from "./browser-preview.js";
|
||||
import { DOCKER_INSTALL_LINKS, isDesktopSetupLink, runDocker } from "./docker-cli.js";
|
||||
import { requestLocalSettings } from "./local-settings.js";
|
||||
import {
|
||||
@ -1130,6 +1131,35 @@ app.whenReady().then(async () => {
|
||||
},
|
||||
);
|
||||
ipcMain.handle("desktop.platform", () => process.platform);
|
||||
const browserPreview = new BrowserPreviewController();
|
||||
// The in-app browser answers only the app window's own top frame, never a page inside it.
|
||||
const previewHost = (event: Electron.IpcMainInvokeEvent) => {
|
||||
if (!fromMainWindow(event) || event.senderFrame !== event.sender.mainFrame) {
|
||||
throw new Error("The in-app browser is not available here.");
|
||||
}
|
||||
return mainWindow!;
|
||||
};
|
||||
ipcMain.handle("desktop.browserPreview.state", (event) =>
|
||||
browserPreview.state(previewHost(event)),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.show", (event, bounds: unknown) =>
|
||||
browserPreview.show(previewHost(event), bounds),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.hide", (event) => browserPreview.hide(previewHost(event)));
|
||||
ipcMain.handle("desktop.browserPreview.navigate", (event, url: unknown) =>
|
||||
browserPreview.navigate(previewHost(event), url),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.back", (event) => browserPreview.back(previewHost(event)));
|
||||
ipcMain.handle("desktop.browserPreview.forward", (event) =>
|
||||
browserPreview.forward(previewHost(event)),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.reload", (event) =>
|
||||
browserPreview.reload(previewHost(event)),
|
||||
);
|
||||
ipcMain.handle("desktop.browserPreview.stop", (event) => browserPreview.stop(previewHost(event)));
|
||||
ipcMain.handle("desktop.browserPreview.openDevTools", (event) =>
|
||||
browserPreview.openDevTools(previewHost(event)),
|
||||
);
|
||||
ipcMain.handle("desktop.files.pickDirectory", async (event) => {
|
||||
if (!fromMainWindow(event)) return null;
|
||||
const result = await dialog.showOpenDialog(windowFrom(event)!, {
|
||||
|
||||
@ -21,6 +21,22 @@ contextBridge.exposeInMainWorld("rakazoDesktop", {
|
||||
download: () => ipcRenderer.invoke("desktop.update.download"),
|
||||
install: () => ipcRenderer.invoke("desktop.update.install"),
|
||||
},
|
||||
browserPreview: {
|
||||
state: () => ipcRenderer.invoke("desktop.browserPreview.state"),
|
||||
show: (bounds) => ipcRenderer.invoke("desktop.browserPreview.show", bounds),
|
||||
hide: () => ipcRenderer.invoke("desktop.browserPreview.hide"),
|
||||
navigate: (url) => ipcRenderer.invoke("desktop.browserPreview.navigate", url),
|
||||
back: () => ipcRenderer.invoke("desktop.browserPreview.back"),
|
||||
forward: () => ipcRenderer.invoke("desktop.browserPreview.forward"),
|
||||
reload: () => ipcRenderer.invoke("desktop.browserPreview.reload"),
|
||||
stop: () => ipcRenderer.invoke("desktop.browserPreview.stop"),
|
||||
openDevTools: () => ipcRenderer.invoke("desktop.browserPreview.openDevTools"),
|
||||
onState: (listener) => {
|
||||
const handler = (_event, state) => listener(state);
|
||||
ipcRenderer.on("desktop.browserPreview.state", handler);
|
||||
return () => ipcRenderer.off("desktop.browserPreview.state", handler);
|
||||
},
|
||||
},
|
||||
oauth: {
|
||||
open: (url) => ipcRenderer.invoke("desktop.oauth.open", url),
|
||||
cancel: (url) => ipcRenderer.invoke("desktop.oauth.cancel", url),
|
||||
|
||||
@ -32,6 +32,7 @@ describe("desktop preload bridge", () => {
|
||||
expect(globalName).toBe("rakazoDesktop");
|
||||
expect(bridge.platform).toBe("linux");
|
||||
expect(Object.keys(bridge).sort()).toEqual([
|
||||
"browserPreview",
|
||||
"files",
|
||||
"localSettings",
|
||||
"oauth",
|
||||
@ -46,6 +47,18 @@ describe("desktop preload bridge", () => {
|
||||
"toggleMaximize",
|
||||
]);
|
||||
expect(Object.keys(bridge.update).sort()).toEqual(["check", "download", "install", "state"]);
|
||||
expect(Object.keys(bridge.browserPreview ?? {}).sort()).toEqual([
|
||||
"back",
|
||||
"forward",
|
||||
"hide",
|
||||
"navigate",
|
||||
"onState",
|
||||
"openDevTools",
|
||||
"reload",
|
||||
"show",
|
||||
"state",
|
||||
"stop",
|
||||
]);
|
||||
|
||||
await bridge.files?.pickDirectory();
|
||||
await bridge.oauth.open?.("https://provider.example.com/authorize");
|
||||
@ -58,6 +71,8 @@ describe("desktop preload bridge", () => {
|
||||
await bridge.update.check();
|
||||
await bridge.update.download();
|
||||
await bridge.update.install();
|
||||
await bridge.browserPreview?.navigate("http://localhost:5173/");
|
||||
await bridge.browserPreview?.show({ x: 1, y: 2, width: 3, height: 4 });
|
||||
expect(invoke.mock.calls.map(([channel]) => channel)).toEqual([
|
||||
"desktop.files.pickDirectory",
|
||||
"desktop.oauth.open",
|
||||
@ -70,6 +85,8 @@ describe("desktop preload bridge", () => {
|
||||
"desktop.update.check",
|
||||
"desktop.update.download",
|
||||
"desktop.update.install",
|
||||
"desktop.browserPreview.navigate",
|
||||
"desktop.browserPreview.show",
|
||||
]);
|
||||
});
|
||||
|
||||
@ -77,6 +94,7 @@ describe("desktop preload bridge", () => {
|
||||
const { exposeInMainWorld } = runPreload("preload.cjs");
|
||||
const [, bridge] = exposeInMainWorld.mock.calls[0] as [string, Record<string, unknown>];
|
||||
expect(Object.keys(bridge).sort()).toEqual([
|
||||
"browserPreview",
|
||||
"files",
|
||||
"localSettings",
|
||||
"oauth",
|
||||
|
||||
@ -1080,7 +1080,7 @@ export function applyMobileThreadEvent(
|
||||
messages: [...remaining, streaming],
|
||||
};
|
||||
}
|
||||
if (event.type === "agent.tool.completed") {
|
||||
if (event.type === "agent.tool.completed" || event.type === "thread.computer") {
|
||||
return { ...prev, cursor: event.seq ?? prev.cursor };
|
||||
}
|
||||
if (event.type === "thread.subagent") {
|
||||
|
||||
53
apps/web/e2e/agent-cursor.spec.ts
Normal file
53
apps/web/e2e/agent-cursor.spec.ts
Normal file
@ -0,0 +1,53 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers";
|
||||
|
||||
const SCREEN_URL = "https://screen.example/embed.html";
|
||||
|
||||
test("the bot's cursor glides to where it clicks on its live screen", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
await signup(page, `agent-cursor-${Date.now()}@rakazo.test`, "password12", "Cursor Watcher");
|
||||
await completeOnboarding(page);
|
||||
const botId = activeBotId(page);
|
||||
await rpc(page, "computer/boot", { botId });
|
||||
// Stand in for the noVNC viewer: a 16:10 desktop with a field in the middle.
|
||||
await page.route("https://screen.example/**", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "text/html",
|
||||
body: `<!doctype html><title>Desktop</title><body style="margin:0;background:#1d1d20;display:grid;place-items:center;height:100vh">
|
||||
<input style="width:40%;padding:8px;font:16px system-ui" placeholder="Search">`,
|
||||
}),
|
||||
);
|
||||
await page.route("**/rpc/computer/screenUrl", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ json: { url: SCREEN_URL } }),
|
||||
}),
|
||||
);
|
||||
|
||||
await page.getByTitle("Agent computer").click();
|
||||
const preview = page.getByTestId("computer-preview");
|
||||
await expect(preview.locator("iframe")).toHaveAttribute("src", SCREEN_URL);
|
||||
const cursor = preview.getByTestId("agent-cursor");
|
||||
await expect(cursor).toHaveCSS("opacity", "0");
|
||||
|
||||
// The scripted runtime clicks the screen center (640, 400) and types.
|
||||
await rpc(page, "threads/send", { botId, text: "use your screen and type cursor-demo" });
|
||||
|
||||
await expect(cursor).toHaveCSS("opacity", "1");
|
||||
await expect(cursor).toHaveAttribute("data-action", /click|type/);
|
||||
const layer = (await preview.getByTestId("agent-cursor-layer").boundingBox())!;
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const tip = (await cursor.boundingBox())!;
|
||||
return [
|
||||
Math.round(tip.x - layer.x - layer.width / 2),
|
||||
Math.round(tip.y - layer.y - layer.height / 2),
|
||||
];
|
||||
})
|
||||
.toEqual([0, 0]);
|
||||
await captureScreenshot(page, testInfo, "agent-cursor-on-screen");
|
||||
|
||||
// It fades once the bot stops pointing, so it never lingers over the screen.
|
||||
await expect(cursor).toHaveCSS("opacity", "0", { timeout: 10_000 });
|
||||
});
|
||||
159
apps/web/e2e/in-app-browser.spec.ts
Normal file
159
apps/web/e2e/in-app-browser.spec.ts
Normal file
@ -0,0 +1,159 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { captureScreenshot, completeOnboarding, signup } from "./helpers";
|
||||
|
||||
test("the in-app browser previews a page and follows it back to the browser", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
await signup(page, `in-app-browser-${Date.now()}@rakazo.test`, "password12", "Previewer");
|
||||
await completeOnboarding(page);
|
||||
// Context-wide so the "Open in browser" tab is served too.
|
||||
await page.context().route("https://preview.example/**", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "text/html",
|
||||
body: `<!doctype html><title>Preview</title><body style="font:16px system-ui;padding:24px">
|
||||
<h1>Live preview</h1><p>Rendered inside the app.</p>`,
|
||||
}),
|
||||
);
|
||||
|
||||
await page.getByRole("button", { name: "Browser" }).click();
|
||||
const browser = page.getByTestId("in-app-browser");
|
||||
const address = browser.getByRole("textbox", { name: "Address" });
|
||||
await expect(address).toBeFocused();
|
||||
|
||||
await address.fill("not an address");
|
||||
await address.press("Enter");
|
||||
await expect(address).toHaveAttribute("aria-invalid", "true");
|
||||
|
||||
await address.fill("preview.example/app");
|
||||
await address.press("Enter");
|
||||
await expect(address).toHaveValue("https://preview.example/app");
|
||||
const frame = browser.getByTestId("in-app-browser-frame");
|
||||
await expect(frame).toHaveAttribute("src", "https://preview.example/app");
|
||||
await expect(
|
||||
page.frameLocator('[data-testid="in-app-browser-frame"]').getByRole("heading"),
|
||||
).toHaveText("Live preview");
|
||||
await captureScreenshot(page, testInfo, "in-app-browser");
|
||||
|
||||
const popup = page.waitForEvent("popup");
|
||||
await browser.getByRole("button", { name: "Open in browser" }).click();
|
||||
expect((await popup).url()).toBe("https://preview.example/app");
|
||||
|
||||
await browser.getByRole("button", { name: "Close panel" }).click();
|
||||
await expect(browser).toHaveCount(0);
|
||||
// The last address comes back when the browser reopens.
|
||||
await page.getByRole("button", { name: "Browser" }).click();
|
||||
await expect(page.getByTestId("in-app-browser-frame")).toHaveAttribute(
|
||||
"src",
|
||||
"https://preview.example/app",
|
||||
);
|
||||
});
|
||||
|
||||
test("on desktop the page is a native view glued to the panel", async ({ page }) => {
|
||||
await page.addInitScript(() => {
|
||||
const calls: Array<[string, unknown?]> = [];
|
||||
let listener: ((state: unknown) => void) | undefined;
|
||||
let state = {
|
||||
url: "",
|
||||
title: "",
|
||||
loading: false,
|
||||
canGoBack: false,
|
||||
canGoForward: false,
|
||||
error: null as string | null,
|
||||
};
|
||||
const record = (name: string) => async (value?: unknown) => {
|
||||
calls.push([name, value]);
|
||||
};
|
||||
Object.assign(window, { __browserCalls: calls });
|
||||
Object.defineProperty(window, "rakazoDesktop", {
|
||||
value: {
|
||||
platform: "linux",
|
||||
window: {
|
||||
close: async () => {},
|
||||
minimize: async () => {},
|
||||
toggleMaximize: async () => {},
|
||||
state: async () => ({ minimized: false, maximized: false, fullScreen: false }),
|
||||
},
|
||||
oauth: { onCallback: () => () => {} },
|
||||
update: {
|
||||
state: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
||||
check: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
||||
download: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
||||
install: async () => ({ phase: "unsupported", currentVersion: "0.0.0" }),
|
||||
},
|
||||
browserPreview: {
|
||||
state: async () => state,
|
||||
onState: (next: (value: unknown) => void) => {
|
||||
listener = next;
|
||||
return () => {
|
||||
listener = undefined;
|
||||
};
|
||||
},
|
||||
navigate: async (url: string) => {
|
||||
calls.push(["navigate", url]);
|
||||
state = { ...state, url, title: "Dev server", loading: false, canGoBack: true };
|
||||
listener?.(state);
|
||||
},
|
||||
show: record("show"),
|
||||
hide: record("hide"),
|
||||
back: record("back"),
|
||||
forward: record("forward"),
|
||||
reload: record("reload"),
|
||||
stop: record("stop"),
|
||||
openDevTools: record("openDevTools"),
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
await signup(page, `native-browser-${Date.now()}@rakazo.test`, "password12", "Desktop");
|
||||
await completeOnboarding(page);
|
||||
const calls = () =>
|
||||
page.evaluate(
|
||||
() => (window as unknown as { __browserCalls: Array<[string, unknown?]> }).__browserCalls,
|
||||
);
|
||||
|
||||
await page.getByRole("button", { name: "Browser" }).click();
|
||||
const browser = page.getByTestId("in-app-browser");
|
||||
await browser.getByRole("textbox", { name: "Address" }).fill("localhost:5173");
|
||||
await browser.getByRole("textbox", { name: "Address" }).press("Enter");
|
||||
await expect.poll(calls).toContainEqual(["navigate", "http://localhost:5173/"]);
|
||||
await expect(browser.getByRole("button", { name: "Back" })).toBeEnabled();
|
||||
await expect(browser.getByRole("button", { name: "Forward" })).toBeDisabled();
|
||||
await expect(browser.getByTestId("in-app-browser-frame")).toHaveCount(0);
|
||||
|
||||
// The reserved page area and the native view's bounds stay identical.
|
||||
const viewport = browser.getByTestId("in-app-browser-viewport");
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const box = (await viewport.boundingBox())!;
|
||||
const shown = (await calls()).filter(([name]) => name === "show").at(-1)?.[1];
|
||||
return (
|
||||
JSON.stringify(shown) ===
|
||||
JSON.stringify({
|
||||
x: Math.round(box.x),
|
||||
y: Math.round(box.y),
|
||||
width: Math.round(box.width),
|
||||
height: Math.round(box.height),
|
||||
})
|
||||
);
|
||||
})
|
||||
.toBe(true);
|
||||
|
||||
await browser.getByRole("button", { name: "Back" }).click();
|
||||
await browser.getByRole("button", { name: "Reload" }).click();
|
||||
await browser.getByRole("button", { name: "Developer tools" }).click();
|
||||
await expect
|
||||
.poll(async () => (await calls()).map(([name]) => name))
|
||||
.toEqual(expect.arrayContaining(["back", "reload", "openDevTools"]));
|
||||
|
||||
// App menus float above the page, so the native view steps aside while one is open.
|
||||
const before = (await calls()).length;
|
||||
await page.getByTestId("create-menu-trigger").click();
|
||||
await expect
|
||||
.poll(async () => (await calls()).slice(before).map(([name]) => name))
|
||||
.toContain("hide");
|
||||
await page.keyboard.press("Escape");
|
||||
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("show");
|
||||
|
||||
await browser.getByRole("button", { name: "Close panel" }).click();
|
||||
await expect.poll(async () => (await calls()).at(-1)?.[0]).toBe("hide");
|
||||
});
|
||||
@ -50,6 +50,16 @@ test("live page helper preserves identity, masks passwords, and reports partial
|
||||
expect(snapshot.title).toBe("Fixture");
|
||||
expect(snapshot.tree).toContain("Readable article content");
|
||||
expect(JSON.stringify(snapshot)).not.toContain("fake-test-password");
|
||||
// Element boxes are screen pixels for the visible bot cursor.
|
||||
expect(snapshot.screen.width).toBeGreaterThan(0);
|
||||
expect(snapshot.screen.height).toBeGreaterThan(0);
|
||||
const saveBox = snapshot.elements.find((element: { name: string }) => element.name === "Save")
|
||||
.box as number[];
|
||||
const saveRect = (await page.locator("#save").boundingBox())!;
|
||||
expect(saveBox[2]).toBe(Math.round(saveRect.width));
|
||||
expect(saveBox[3]).toBe(Math.round(saveRect.height));
|
||||
expect(saveBox[0]).toBeGreaterThanOrEqual(Math.round(saveRect.x));
|
||||
expect(saveBox[1]).toBeGreaterThanOrEqual(Math.round(saveRect.y));
|
||||
const ref = (snapshot: { elements: { name: string; ref: string }[] }, name: string) =>
|
||||
snapshot.elements.find((element) => element.name === name)!.ref;
|
||||
const result = await command("act", {
|
||||
|
||||
317
apps/web/src/components/browser/InAppBrowser.tsx
Normal file
317
apps/web/src/components/browser/InAppBrowser.tsx
Normal file
@ -0,0 +1,317 @@
|
||||
import { useLingui } from "@lingui/react/macro";
|
||||
import type {
|
||||
RakazoDesktopBrowserPreview,
|
||||
RakazoDesktopBrowserPreviewState,
|
||||
} from "@rakazo/contracts";
|
||||
import { Button, cn, Input } from "@rakazo/ui-web";
|
||||
import { ArrowLeft, ArrowRight, Code, ExternalLink, RotateCw, X } from "lucide-react";
|
||||
import { type FormEvent, useCallback, useEffect, useRef, useState } from "react";
|
||||
import { browserFrameSandbox, normalizeBrowserUrl } from "../../lib/browser-url";
|
||||
import { desktopBridge } from "../../lib/desktop";
|
||||
|
||||
const LAST_URL_KEY = "rk:in-app-browser:url";
|
||||
|
||||
function readLastUrl(): string {
|
||||
try {
|
||||
return window.localStorage.getItem(LAST_URL_KEY) ?? "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function rememberUrl(url: string) {
|
||||
try {
|
||||
window.localStorage.setItem(LAST_URL_KEY, url);
|
||||
} catch {
|
||||
/* A per-viewer convenience only. */
|
||||
}
|
||||
}
|
||||
|
||||
const EMPTY_STATE: RakazoDesktopBrowserPreviewState = {
|
||||
url: "",
|
||||
title: "",
|
||||
loading: false,
|
||||
canGoBack: false,
|
||||
canGoForward: false,
|
||||
error: null,
|
||||
};
|
||||
|
||||
/** DOM layers the native desktop view would otherwise paint over. */
|
||||
const FLOATING_UI = '[role="dialog"],[role="alertdialog"],[role="menu"],[role="listbox"]';
|
||||
|
||||
/**
|
||||
* Address bar plus page. On desktop the page is a native Chromium view laid over the reserved
|
||||
* area, so dev servers keep hot reload, DevTools and full speed; in a browser it is an iframe.
|
||||
*/
|
||||
export function InAppBrowser({ onClose }: { onClose: () => void }) {
|
||||
const { t } = useLingui();
|
||||
const native = desktopBridge()?.browserPreview;
|
||||
const [state, setState] = useState<RakazoDesktopBrowserPreviewState>(() => ({
|
||||
...EMPTY_STATE,
|
||||
url: native ? "" : readLastUrl(),
|
||||
}));
|
||||
const [draft, setDraft] = useState(state.url);
|
||||
const [invalid, setInvalid] = useState(false);
|
||||
const [frameKey, setFrameKey] = useState(0);
|
||||
const editing = useRef(false);
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!native) return;
|
||||
let live = true;
|
||||
const apply = (next: RakazoDesktopBrowserPreviewState) => {
|
||||
if (!live) return;
|
||||
setState(next);
|
||||
if (!editing.current) setDraft(next.url);
|
||||
};
|
||||
const unsubscribe = native.onState(apply);
|
||||
void native
|
||||
.state()
|
||||
.then((current) => {
|
||||
apply(current);
|
||||
const last = readLastUrl();
|
||||
if (!current.url && last) void native.navigate(last).catch(() => undefined);
|
||||
})
|
||||
.catch(() => undefined);
|
||||
return () => {
|
||||
live = false;
|
||||
unsubscribe();
|
||||
};
|
||||
}, [native]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!state.url) inputRef.current?.focus();
|
||||
}, [state.url]);
|
||||
|
||||
const go = useCallback(
|
||||
(input: string) => {
|
||||
const url = normalizeBrowserUrl(input);
|
||||
setInvalid(!url);
|
||||
if (!url) return;
|
||||
editing.current = false;
|
||||
setDraft(url);
|
||||
rememberUrl(url);
|
||||
if (native) {
|
||||
void native.navigate(url).catch(() => setInvalid(true));
|
||||
} else {
|
||||
setState((current) => ({ ...current, url, loading: true, error: null }));
|
||||
setFrameKey((key) => key + 1);
|
||||
}
|
||||
},
|
||||
[native],
|
||||
);
|
||||
|
||||
const reload = useCallback(() => {
|
||||
if (native) {
|
||||
void (state.loading ? native.stop() : native.reload()).catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
if (!state.url) return;
|
||||
setState((current) => ({ ...current, loading: true }));
|
||||
setFrameKey((key) => key + 1);
|
||||
}, [native, state.loading, state.url]);
|
||||
|
||||
const onSubmit = (event: FormEvent) => {
|
||||
event.preventDefault();
|
||||
go(draft);
|
||||
inputRef.current?.blur();
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col" data-testid="in-app-browser">
|
||||
<form onSubmit={onSubmit} className="relative flex items-center gap-0.5 border-b px-2 py-1.5">
|
||||
{native ? (
|
||||
<>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Back`}
|
||||
disabled={!state.canGoBack}
|
||||
onClick={() => void native.back().catch(() => undefined)}
|
||||
>
|
||||
<ArrowLeft strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Forward`}
|
||||
disabled={!state.canGoForward}
|
||||
onClick={() => void native.forward().catch(() => undefined)}
|
||||
>
|
||||
<ArrowRight strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
</>
|
||||
) : null}
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={state.loading ? t`Stop` : t`Reload`}
|
||||
disabled={!state.url}
|
||||
onClick={reload}
|
||||
>
|
||||
{state.loading && native ? (
|
||||
<X strokeWidth={1.8} className="size-icon-md" />
|
||||
) : (
|
||||
<RotateCw strokeWidth={1.7} className="size-icon-md" />
|
||||
)}
|
||||
</Button>
|
||||
<Input
|
||||
ref={inputRef}
|
||||
value={draft}
|
||||
aria-label={t`Address`}
|
||||
aria-invalid={invalid || undefined}
|
||||
inputMode="url"
|
||||
autoComplete="off"
|
||||
autoCapitalize="off"
|
||||
spellCheck={false}
|
||||
placeholder="localhost:5173"
|
||||
onFocus={(event) => {
|
||||
editing.current = true;
|
||||
event.currentTarget.select();
|
||||
}}
|
||||
onBlur={() => {
|
||||
editing.current = false;
|
||||
}}
|
||||
onChange={(event) => {
|
||||
setDraft(event.target.value);
|
||||
setInvalid(false);
|
||||
}}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key !== "Escape") return;
|
||||
setDraft(state.url);
|
||||
setInvalid(false);
|
||||
event.currentTarget.blur();
|
||||
}}
|
||||
className="mx-1 h-control-sm flex-1 rounded-full bg-muted/60 px-3 text-ui-body-sm"
|
||||
/>
|
||||
{native ? (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Developer tools`}
|
||||
disabled={!state.url}
|
||||
onClick={() => void native.openDevTools().catch(() => undefined)}
|
||||
>
|
||||
<Code strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
) : null}
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Open in browser`}
|
||||
disabled={!state.url}
|
||||
onClick={() => window.open(state.url, "_blank", "noopener,noreferrer")}
|
||||
>
|
||||
<ExternalLink strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Close panel`}
|
||||
onClick={onClose}
|
||||
>
|
||||
<X strokeWidth={1.8} className="size-icon-md" />
|
||||
</Button>
|
||||
<div
|
||||
aria-hidden
|
||||
className={cn(
|
||||
"absolute inset-x-0 -bottom-px h-0.5 overflow-hidden transition-opacity duration-300",
|
||||
state.loading ? "opacity-100" : "opacity-0",
|
||||
)}
|
||||
>
|
||||
<div className="rk-browser-progress h-full w-1/3 rounded-full bg-primary/70" />
|
||||
</div>
|
||||
</form>
|
||||
<div className="relative min-h-0 flex-1 bg-background">
|
||||
{native ? (
|
||||
<NativeViewport browser={native} visible={Boolean(state.url) && !state.error} />
|
||||
) : state.url ? (
|
||||
<iframe
|
||||
key={frameKey}
|
||||
title={state.title || t`Page`}
|
||||
src={state.url}
|
||||
sandbox={browserFrameSandbox(state.url, window.location.origin)}
|
||||
allow="clipboard-read; clipboard-write; fullscreen"
|
||||
onLoad={() => setState((current) => ({ ...current, loading: false }))}
|
||||
className="h-full w-full border-0 bg-background"
|
||||
data-testid="in-app-browser-frame"
|
||||
/>
|
||||
) : null}
|
||||
{state.error ? (
|
||||
<div
|
||||
role="alert"
|
||||
className="absolute inset-0 grid place-items-center px-6 text-center text-ui-body-sm text-muted-foreground"
|
||||
>
|
||||
{state.error}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserves the page area and keeps the native view glued to it while the panel resizes,
|
||||
* hiding it whenever app UI (dialogs, menus) floats where the view would paint over it.
|
||||
*/
|
||||
function NativeViewport({
|
||||
browser,
|
||||
visible,
|
||||
}: {
|
||||
browser: RakazoDesktopBrowserPreview;
|
||||
visible: boolean;
|
||||
}) {
|
||||
const ref = useRef<HTMLDivElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const element = ref.current;
|
||||
if (!element) return;
|
||||
let last = "";
|
||||
let frame = 0;
|
||||
const sync = () => {
|
||||
frame = 0;
|
||||
const rect = element.getBoundingClientRect();
|
||||
const covered = Array.from(document.querySelectorAll(FLOATING_UI)).some(
|
||||
(node) => !element.contains(node),
|
||||
);
|
||||
const show = visible && rect.width >= 1 && rect.height >= 1 && !covered;
|
||||
const bounds = {
|
||||
x: Math.round(rect.left),
|
||||
y: Math.round(rect.top),
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
};
|
||||
const key = show ? JSON.stringify(bounds) : "hidden";
|
||||
if (key === last) return;
|
||||
last = key;
|
||||
void (show ? browser.show(bounds) : browser.hide()).catch(() => undefined);
|
||||
};
|
||||
const schedule = () => {
|
||||
if (!frame) frame = requestAnimationFrame(sync);
|
||||
};
|
||||
const resize = new ResizeObserver(schedule);
|
||||
resize.observe(element);
|
||||
const floating = new MutationObserver(schedule);
|
||||
floating.observe(document.body, { childList: true, subtree: true });
|
||||
window.addEventListener("resize", schedule);
|
||||
// Sibling layout (sidebars, transitions) can move the area without resizing it.
|
||||
const settle = setInterval(schedule, 250);
|
||||
schedule();
|
||||
return () => {
|
||||
cancelAnimationFrame(frame);
|
||||
resize.disconnect();
|
||||
floating.disconnect();
|
||||
window.removeEventListener("resize", schedule);
|
||||
clearInterval(settle);
|
||||
void browser.hide().catch(() => undefined);
|
||||
};
|
||||
}, [browser, visible]);
|
||||
|
||||
return <div ref={ref} className="absolute inset-0" data-testid="in-app-browser-viewport" />;
|
||||
}
|
||||
165
apps/web/src/components/computer/AgentCursor.tsx
Normal file
165
apps/web/src/components/computer/AgentCursor.tsx
Normal file
@ -0,0 +1,165 @@
|
||||
import { screenPointInViewer } from "@rakazo/core";
|
||||
import { Keyboard } from "lucide-react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
agentPointerFeed,
|
||||
POINTER_IDLE_MS,
|
||||
type PointerPoint,
|
||||
type PointerScreen,
|
||||
type PointerSegment,
|
||||
planPointerTimeline,
|
||||
pointerFrameAt,
|
||||
} from "../../lib/agent-pointer";
|
||||
|
||||
function prefersReducedMotion() {
|
||||
return (
|
||||
typeof window.matchMedia === "function" &&
|
||||
window.matchMedia("(prefers-reduced-motion: reduce)").matches
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Draws a bot's announced pointer over its live screen: it glides to each target, ripples on
|
||||
* clicks and shows when it types. Frames are written straight to the DOM so the chat never
|
||||
* re-renders at animation rate.
|
||||
*/
|
||||
export function AgentCursor({
|
||||
botId,
|
||||
color,
|
||||
name,
|
||||
}: {
|
||||
botId: string;
|
||||
color: string;
|
||||
name: string;
|
||||
}) {
|
||||
const layerRef = useRef<HTMLDivElement>(null);
|
||||
const cursorRef = useRef<HTMLDivElement>(null);
|
||||
const rippleRef = useRef<HTMLSpanElement>(null);
|
||||
const [typing, setTyping] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const layer = layerRef.current;
|
||||
const cursor = cursorRef.current;
|
||||
const ripple = rippleRef.current;
|
||||
if (!layer || !cursor || !ripple) return;
|
||||
const reducedMotion = prefersReducedMotion();
|
||||
let screen: PointerScreen | null = null;
|
||||
let segments: PointerSegment[] = [];
|
||||
let position: PointerPoint | null = null;
|
||||
let frame = 0;
|
||||
let idleTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let viewer = { width: layer.clientWidth, height: layer.clientHeight };
|
||||
let shownTyping = false;
|
||||
|
||||
const place = (point: PointerPoint, action: string | null, progress: number) => {
|
||||
if (!screen) return;
|
||||
const projected = screenPointInViewer(point, viewer, screen);
|
||||
cursor.style.transform = `translate3d(${projected.x}px, ${projected.y}px, 0)`;
|
||||
cursor.dataset.action = action ?? "move";
|
||||
const pressing = action === "click" || action === "double_click" || action === "right_click";
|
||||
const pulse = action === "double_click" ? (progress * 2) % 1 : progress;
|
||||
ripple.style.opacity = pressing && !reducedMotion ? String(0.55 * (1 - pulse)) : "0";
|
||||
ripple.style.transform = `scale(${0.35 + pulse * 1.15})`;
|
||||
cursor.style.setProperty("--agent-cursor-press", pressing && pulse < 0.35 ? "0.86" : "1");
|
||||
const nextTyping = action === "type" || action === "key";
|
||||
if (nextTyping !== shownTyping) {
|
||||
shownTyping = nextTyping;
|
||||
setTyping(nextTyping);
|
||||
}
|
||||
};
|
||||
|
||||
const rest = () => {
|
||||
clearTimeout(idleTimer);
|
||||
idleTimer = setTimeout(() => {
|
||||
cursor.style.opacity = "0";
|
||||
}, POINTER_IDLE_MS);
|
||||
};
|
||||
|
||||
const tick = () => {
|
||||
frame = 0;
|
||||
const now = performance.now();
|
||||
const state = pointerFrameAt(segments, now, reducedMotion);
|
||||
if (!state) return;
|
||||
position = state.point;
|
||||
place(state.point, state.action?.kind ?? null, state.action?.progress ?? 0);
|
||||
if (state.done) {
|
||||
segments = [];
|
||||
rest();
|
||||
return;
|
||||
}
|
||||
frame = requestAnimationFrame(tick);
|
||||
};
|
||||
|
||||
const unsubscribe = agentPointerFeed.subscribe(botId, (payload) => {
|
||||
const now = performance.now();
|
||||
const current = pointerFrameAt(segments, now, reducedMotion)?.point ?? position;
|
||||
const sameScreen =
|
||||
screen?.width === payload.screen.width && screen?.height === payload.screen.height;
|
||||
screen = payload.screen;
|
||||
// A first announcement starts at its target rather than flying in from a corner.
|
||||
const firstTarget = payload.steps.find((step) => typeof step.x === "number");
|
||||
const from =
|
||||
current && sameScreen
|
||||
? current
|
||||
: firstTarget
|
||||
? { x: firstTarget.x ?? 0, y: firstTarget.y ?? 0 }
|
||||
: { x: payload.screen.width / 2, y: payload.screen.height / 2 };
|
||||
segments = planPointerTimeline(from, payload.steps, now);
|
||||
clearTimeout(idleTimer);
|
||||
cursor.style.opacity = "1";
|
||||
if (!frame) frame = requestAnimationFrame(tick);
|
||||
});
|
||||
|
||||
const resize = new ResizeObserver(() => {
|
||||
viewer = { width: layer.clientWidth, height: layer.clientHeight };
|
||||
if (position && !frame) place(position, null, 0);
|
||||
});
|
||||
resize.observe(layer);
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
resize.disconnect();
|
||||
cancelAnimationFrame(frame);
|
||||
clearTimeout(idleTimer);
|
||||
};
|
||||
}, [botId]);
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={layerRef}
|
||||
aria-hidden
|
||||
data-testid="agent-cursor-layer"
|
||||
className="pointer-events-none absolute inset-0 overflow-hidden"
|
||||
>
|
||||
<div
|
||||
ref={cursorRef}
|
||||
data-testid="agent-cursor"
|
||||
className="absolute top-0 left-0 opacity-0 transition-opacity duration-300 will-change-transform"
|
||||
style={{ color }}
|
||||
>
|
||||
<span
|
||||
ref={rippleRef}
|
||||
className="absolute -top-5 -left-5 size-10 rounded-full border-2 border-current opacity-0"
|
||||
/>
|
||||
<svg
|
||||
viewBox="0 0 20 22"
|
||||
className="relative block size-5 origin-top-left drop-shadow-md transition-transform duration-100"
|
||||
style={{ transform: "scale(var(--agent-cursor-press, 1))" }}
|
||||
>
|
||||
<path
|
||||
d="M1.5 1.5 L1.5 17.5 L6 13.2 L9.2 20.2 L12.4 18.8 L9.3 12 L15.5 12 Z"
|
||||
fill="currentColor"
|
||||
className="stroke-background"
|
||||
strokeWidth="1.6"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
<span className="absolute top-5 left-3.5 flex max-w-40 items-center gap-1 rounded-full bg-popover px-2 py-0.5 text-ui-label font-medium whitespace-nowrap text-popover-foreground shadow-sm ring-1 ring-border">
|
||||
<span className="size-1.5 shrink-0 rounded-full bg-current" style={{ color }} />
|
||||
<span className="truncate">{name}</span>
|
||||
{typing ? <Keyboard strokeWidth={1.8} className="size-3 shrink-0" /> : null}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
88
apps/web/src/lib/agent-pointer.test.ts
Normal file
88
apps/web/src/lib/agent-pointer.test.ts
Normal file
@ -0,0 +1,88 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
agentPointerFeed,
|
||||
moveDurationMs,
|
||||
planPointerTimeline,
|
||||
pointerFrameAt,
|
||||
} from "./agent-pointer";
|
||||
|
||||
const payload = {
|
||||
kind: "pointer",
|
||||
screen: { width: 1280, height: 800 },
|
||||
steps: [{ action: "click", x: 100, y: 100 }],
|
||||
};
|
||||
|
||||
describe("agentPointerFeed", () => {
|
||||
it("delivers valid live announcements only to the bot's viewers", () => {
|
||||
const seen: unknown[] = [];
|
||||
const other: unknown[] = [];
|
||||
const stop = agentPointerFeed.subscribe("bot-a", (value) => seen.push(value));
|
||||
const stopOther = agentPointerFeed.subscribe("bot-b", (value) => other.push(value));
|
||||
try {
|
||||
const now = Date.parse("2026-01-01T00:00:10Z");
|
||||
expect(agentPointerFeed.publish("bot-a", payload, "2026-01-01T00:00:05Z", now)).toBe(true);
|
||||
// Replayed history and malformed payloads never move the cursor.
|
||||
expect(agentPointerFeed.publish("bot-a", payload, "2025-12-31T23:00:00Z", now)).toBe(false);
|
||||
expect(agentPointerFeed.publish("bot-a", { ...payload, steps: [] }, undefined, now)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(agentPointerFeed.publish("bot-a", { kind: "status" }, undefined, now)).toBe(false);
|
||||
expect(seen).toEqual([payload]);
|
||||
expect(other).toEqual([]);
|
||||
} finally {
|
||||
stop();
|
||||
stopOther();
|
||||
}
|
||||
expect(agentPointerFeed.publish("bot-a", payload)).toBe(true);
|
||||
expect(seen).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pointer timeline", () => {
|
||||
it("travels, acts in place for keyboard steps, then rests", () => {
|
||||
const segments = planPointerTimeline(
|
||||
{ x: 0, y: 0 },
|
||||
[{ action: "click", x: 300, y: 400 }, { action: "type" }, { action: "move", x: 300, y: 400 }],
|
||||
1_000,
|
||||
);
|
||||
expect(segments.map((segment) => segment.to)).toEqual([
|
||||
{ x: 300, y: 400 },
|
||||
{ x: 300, y: 400 },
|
||||
{ x: 300, y: 400 },
|
||||
]);
|
||||
const [click, type, move] = segments;
|
||||
expect(click!.arrive - click!.start).toBe(moveDurationMs(500));
|
||||
expect(type!.arrive).toBe(type!.start);
|
||||
expect(move!.end).toBe(move!.start);
|
||||
|
||||
const midway = pointerFrameAt(segments, (click!.start + click!.arrive) / 2)!;
|
||||
expect(midway.action).toBeNull();
|
||||
// The bow lifts the cursor off the straight line between the points.
|
||||
const onLine = { x: 150, y: 200 };
|
||||
expect(Math.hypot(midway.point.x - onLine.x, midway.point.y - onLine.y)).toBeGreaterThan(10);
|
||||
|
||||
const clicking = pointerFrameAt(segments, click!.arrive + 1)!;
|
||||
expect(clicking.point).toEqual({ x: 300, y: 400 });
|
||||
expect(clicking.action?.kind).toBe("click");
|
||||
|
||||
const typing = pointerFrameAt(segments, type!.arrive + 10)!;
|
||||
expect(typing.action?.kind).toBe("type");
|
||||
|
||||
expect(pointerFrameAt(segments, move!.end + 1)).toEqual({
|
||||
point: { x: 300, y: 400 },
|
||||
action: null,
|
||||
done: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("jumps straight to the target when motion is reduced", () => {
|
||||
const segments = planPointerTimeline({ x: 0, y: 0 }, [{ action: "move", x: 50, y: 60 }], 0);
|
||||
expect(pointerFrameAt(segments, 1, true)?.point).toEqual({ x: 50, y: 60 });
|
||||
});
|
||||
|
||||
it("bounds travel time so long and short moves both feel responsive", () => {
|
||||
expect(moveDurationMs(0)).toBe(0);
|
||||
expect(moveDurationMs(5)).toBe(180);
|
||||
expect(moveDurationMs(10_000)).toBe(650);
|
||||
});
|
||||
});
|
||||
156
apps/web/src/lib/agent-pointer.ts
Normal file
156
apps/web/src/lib/agent-pointer.ts
Normal file
@ -0,0 +1,156 @@
|
||||
import {
|
||||
type ComputerPointerAction,
|
||||
ComputerPointerPayload,
|
||||
type ComputerPointerStep,
|
||||
} from "@rakazo/contracts";
|
||||
|
||||
export type PointerPoint = { x: number; y: number };
|
||||
export type PointerScreen = ComputerPointerPayload["screen"];
|
||||
|
||||
/** Older announcements describe actions that already finished; replaying them would mislead. */
|
||||
export const POINTER_EVENT_MAX_AGE_MS = 30_000;
|
||||
export const POINTER_IDLE_MS = 3_500;
|
||||
|
||||
type Listener = (payload: ComputerPointerPayload) => void;
|
||||
|
||||
/**
|
||||
* Live pointer announcements per bot. The thread subscription publishes `thread.computer`
|
||||
* pointer events here; screen viewers subscribe for the bot they show.
|
||||
*/
|
||||
class AgentPointerFeed {
|
||||
private readonly listeners = new Map<string, Set<Listener>>();
|
||||
|
||||
publish(botId: string, payload: unknown, createdAt?: string, now = Date.now()): boolean {
|
||||
const parsed = ComputerPointerPayload.safeParse(payload);
|
||||
if (!parsed.success) return false;
|
||||
const at = createdAt ? Date.parse(createdAt) : Number.NaN;
|
||||
if (Number.isFinite(at) && now - at > POINTER_EVENT_MAX_AGE_MS) return false;
|
||||
for (const listener of this.listeners.get(botId) ?? []) listener(parsed.data);
|
||||
return true;
|
||||
}
|
||||
|
||||
subscribe(botId: string, listener: Listener): () => void {
|
||||
let set = this.listeners.get(botId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
this.listeners.set(botId, set);
|
||||
}
|
||||
set.add(listener);
|
||||
return () => {
|
||||
set.delete(listener);
|
||||
if (set.size === 0) this.listeners.delete(botId);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const agentPointerFeed = new AgentPointerFeed();
|
||||
|
||||
export type PointerSegment = {
|
||||
from: PointerPoint;
|
||||
to: PointerPoint;
|
||||
/** Perpendicular bow in screen pixels so long moves read as a hand, not a teleport. */
|
||||
bow: number;
|
||||
start: number;
|
||||
arrive: number;
|
||||
end: number;
|
||||
action: ComputerPointerAction;
|
||||
};
|
||||
|
||||
const ACTION_DWELL_MS: Record<ComputerPointerAction, number> = {
|
||||
move: 0,
|
||||
click: 320,
|
||||
double_click: 440,
|
||||
right_click: 320,
|
||||
down: 160,
|
||||
up: 160,
|
||||
type: 700,
|
||||
key: 280,
|
||||
scroll: 420,
|
||||
};
|
||||
|
||||
export function moveDurationMs(distance: number): number {
|
||||
return distance < 1 ? 0 : Math.round(Math.min(650, Math.max(180, 140 + distance * 0.45)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Lays announced steps on a timeline that starts where the cursor currently is. Keyboard
|
||||
* steps carry no coordinates and act in place.
|
||||
*/
|
||||
export function planPointerTimeline(
|
||||
from: PointerPoint,
|
||||
steps: readonly ComputerPointerStep[],
|
||||
startAt: number,
|
||||
): PointerSegment[] {
|
||||
const segments: PointerSegment[] = [];
|
||||
let position = from;
|
||||
let time = startAt;
|
||||
for (const step of steps) {
|
||||
const to =
|
||||
typeof step.x === "number" && typeof step.y === "number"
|
||||
? { x: step.x, y: step.y }
|
||||
: position;
|
||||
const distance = Math.hypot(to.x - position.x, to.y - position.y);
|
||||
const arrive = time + moveDurationMs(distance);
|
||||
const end = arrive + ACTION_DWELL_MS[step.action];
|
||||
segments.push({
|
||||
from: position,
|
||||
to,
|
||||
bow: Math.min(48, distance * 0.12),
|
||||
start: time,
|
||||
arrive,
|
||||
end,
|
||||
action: step.action,
|
||||
});
|
||||
position = to;
|
||||
time = end;
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
|
||||
export type PointerFrame = {
|
||||
point: PointerPoint;
|
||||
/** The action being performed at the target, with 0..1 progress; null while travelling. */
|
||||
action: { kind: ComputerPointerAction; progress: number } | null;
|
||||
done: boolean;
|
||||
};
|
||||
|
||||
function easeInOutCubic(t: number): number {
|
||||
return t < 0.5 ? 4 * t * t * t : 1 - (-2 * t + 2) ** 3 / 2;
|
||||
}
|
||||
|
||||
export function pointerFrameAt(
|
||||
segments: readonly PointerSegment[],
|
||||
now: number,
|
||||
reducedMotion = false,
|
||||
): PointerFrame | null {
|
||||
if (segments.length === 0) return null;
|
||||
const last = segments[segments.length - 1]!;
|
||||
if (now >= last.end) return { point: last.to, action: null, done: true };
|
||||
const segment = segments.find((candidate) => now < candidate.end) ?? last;
|
||||
if (now < segment.arrive) {
|
||||
if (reducedMotion) return { point: segment.to, action: null, done: false };
|
||||
const span = segment.arrive - segment.start;
|
||||
const t = easeInOutCubic(span > 0 ? Math.max(0, (now - segment.start) / span) : 1);
|
||||
const dx = segment.to.x - segment.from.x;
|
||||
const dy = segment.to.y - segment.from.y;
|
||||
const length = Math.hypot(dx, dy) || 1;
|
||||
const lift = Math.sin(Math.PI * t) * segment.bow;
|
||||
return {
|
||||
point: {
|
||||
x: segment.from.x + dx * t - (dy / length) * lift,
|
||||
y: segment.from.y + dy * t + (dx / length) * lift,
|
||||
},
|
||||
action: null,
|
||||
done: false,
|
||||
};
|
||||
}
|
||||
const dwell = segment.end - segment.arrive;
|
||||
return {
|
||||
point: segment.to,
|
||||
action:
|
||||
segment.action === "move"
|
||||
? null
|
||||
: { kind: segment.action, progress: dwell > 0 ? (now - segment.arrive) / dwell : 1 },
|
||||
done: false,
|
||||
};
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { browserPreviewActivity } from "./browser-preview-context";
|
||||
|
||||
describe("browserPreviewActivity", () => {
|
||||
it("publishes cursors to subscribers and drops cleared ids", () => {
|
||||
for (const id of browserPreviewActivity.snapshot().keys()) browserPreviewActivity.clear(id);
|
||||
|
||||
const seen: string[][] = [];
|
||||
const unsubscribe = browserPreviewActivity.subscribe((activities) => {
|
||||
seen.push([...activities.keys()]);
|
||||
});
|
||||
|
||||
try {
|
||||
browserPreviewActivity.set("bot-a", 50, 40, "Ada");
|
||||
browserPreviewActivity.set("bot-a", 50, 40, "Ada");
|
||||
browserPreviewActivity.set("bot-b", 10, 20, "Bob");
|
||||
expect(browserPreviewActivity.snapshot().get("bot-a")).toEqual({
|
||||
x: 50,
|
||||
y: 40,
|
||||
label: "Ada",
|
||||
});
|
||||
|
||||
browserPreviewActivity.clear("bot-a");
|
||||
browserPreviewActivity.clear("missing");
|
||||
|
||||
expect(seen).toEqual([[], ["bot-a"], ["bot-a", "bot-b"], ["bot-b"]]);
|
||||
} finally {
|
||||
unsubscribe();
|
||||
for (const id of [...browserPreviewActivity.snapshot().keys()]) {
|
||||
browserPreviewActivity.clear(id);
|
||||
}
|
||||
}
|
||||
|
||||
expect(browserPreviewActivity.snapshot().size).toBe(0);
|
||||
});
|
||||
});
|
||||
@ -1,47 +0,0 @@
|
||||
export type BrowserPreviewCursor = {
|
||||
x: number;
|
||||
y: number;
|
||||
label: string;
|
||||
};
|
||||
|
||||
type Listener = (activities: ReadonlyMap<string, BrowserPreviewCursor>) => void;
|
||||
|
||||
/**
|
||||
* In-memory cursors for the desktop browser-preview panel.
|
||||
* The shell publishes working bots here; the panel subscribes.
|
||||
*/
|
||||
class BrowserPreviewActivity {
|
||||
private readonly activities = new Map<string, BrowserPreviewCursor>();
|
||||
private readonly listeners = new Set<Listener>();
|
||||
|
||||
subscribe(listener: Listener): () => void {
|
||||
this.listeners.add(listener);
|
||||
listener(this.snapshot());
|
||||
return () => {
|
||||
this.listeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): ReadonlyMap<string, BrowserPreviewCursor> {
|
||||
return new Map(this.activities);
|
||||
}
|
||||
|
||||
set(id: string, x: number, y: number, label: string) {
|
||||
const current = this.activities.get(id);
|
||||
if (current && current.x === x && current.y === y && current.label === label) return;
|
||||
this.activities.set(id, { x, y, label });
|
||||
this.emit();
|
||||
}
|
||||
|
||||
clear(id: string) {
|
||||
if (!this.activities.delete(id)) return;
|
||||
this.emit();
|
||||
}
|
||||
|
||||
private emit() {
|
||||
const snapshot = this.snapshot();
|
||||
for (const listener of this.listeners) listener(snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
export const browserPreviewActivity = new BrowserPreviewActivity();
|
||||
38
apps/web/src/lib/browser-url.test.ts
Normal file
38
apps/web/src/lib/browser-url.test.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { browserFrameSandbox, normalizeBrowserUrl } from "./browser-url";
|
||||
|
||||
describe("normalizeBrowserUrl", () => {
|
||||
it("defaults local dev servers to http and the web to https", () => {
|
||||
expect(normalizeBrowserUrl("localhost:5173")).toBe("http://localhost:5173/");
|
||||
expect(normalizeBrowserUrl("127.0.0.1:3000/app?x=1")).toBe("http://127.0.0.1:3000/app?x=1");
|
||||
expect(normalizeBrowserUrl("example.com/docs")).toBe("https://example.com/docs");
|
||||
expect(normalizeBrowserUrl(" https://example.com ")).toBe("https://example.com/");
|
||||
expect(normalizeBrowserUrl("http://example.com")).toBe("http://example.com/");
|
||||
});
|
||||
|
||||
it("rejects input that is not an http(s) address", () => {
|
||||
for (const input of [
|
||||
"",
|
||||
"hello world",
|
||||
"hello",
|
||||
"javascript:alert(1)",
|
||||
"file:///etc/passwd",
|
||||
"data:text/html,hi",
|
||||
"ftp://example.com",
|
||||
"https://user:fake-password@example.com",
|
||||
]) {
|
||||
expect(normalizeBrowserUrl(input), input).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("browserFrameSandbox", () => {
|
||||
it("never grants same-origin access to the app's own origin", () => {
|
||||
expect(browserFrameSandbox("https://app.test/x", "https://app.test")).not.toContain(
|
||||
"allow-same-origin",
|
||||
);
|
||||
expect(browserFrameSandbox("http://localhost:5173/", "https://app.test")).toContain(
|
||||
"allow-same-origin",
|
||||
);
|
||||
});
|
||||
});
|
||||
39
apps/web/src/lib/browser-url.ts
Normal file
39
apps/web/src/lib/browser-url.ts
Normal file
@ -0,0 +1,39 @@
|
||||
const LOCAL_HOST = /^(localhost|127(?:\.\d{1,3}){3}|\[::1\]|0\.0\.0\.0|[\w-]+\.local(?:host)?)$/i;
|
||||
|
||||
/**
|
||||
* Turns what someone types in the address bar into an http(s) URL. Local dev servers default to
|
||||
* http, everything else to https. Returns null for input that is not an address.
|
||||
*/
|
||||
export function normalizeBrowserUrl(input: string): string | null {
|
||||
const value = input.trim();
|
||||
if (!value || /\s/.test(value)) return null;
|
||||
const hasScheme = /^[a-z][a-z\d+.-]*:\/\//i.test(value);
|
||||
if (!hasScheme && /^[a-z][a-z\d+.-]*:(?!\d)/i.test(value)) return null;
|
||||
let candidate = value;
|
||||
if (!hasScheme) {
|
||||
const host = value.split(/[/?#]/, 1)[0]!.replace(/:\d+$/, "");
|
||||
if (!LOCAL_HOST.test(host) && !host.includes(".") && !/^\d+$/.test(host)) return null;
|
||||
candidate = `${LOCAL_HOST.test(host) ? "http" : "https"}://${value}`;
|
||||
}
|
||||
try {
|
||||
const url = new URL(candidate);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
||||
if (url.username || url.password || !url.hostname) return null;
|
||||
return url.toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sandbox for the web fallback. A page on the app's own origin must not get same-origin
|
||||
* access, or it could reach into the app around the sandbox.
|
||||
*/
|
||||
export function browserFrameSandbox(url: string, appOrigin: string): string {
|
||||
const base = "allow-scripts allow-forms allow-popups allow-modals allow-downloads";
|
||||
try {
|
||||
return new URL(url).origin === appOrigin ? base : `${base} allow-same-origin`;
|
||||
} catch {
|
||||
return base;
|
||||
}
|
||||
}
|
||||
@ -925,6 +925,24 @@ describe("thread event reduction", () => {
|
||||
expect(next?.messages).toEqual(initial.messages);
|
||||
});
|
||||
|
||||
it("advances past a bot pointer announcement without touching messages", () => {
|
||||
const initial = snapshot([message("m1", [{ kind: "text", text: "hi" }], 3)], 3);
|
||||
const pointer = event({
|
||||
type: "thread.computer",
|
||||
seq: 4,
|
||||
runId: "run-1",
|
||||
payload: {
|
||||
kind: "pointer",
|
||||
screen: { width: 1280, height: 800 },
|
||||
steps: [{ action: "click", x: 10, y: 20 }],
|
||||
},
|
||||
});
|
||||
expect(isThreadSnapshotEvent(pointer)).toBe(true);
|
||||
const next = reduceThreadSnapshot(initial, pointer);
|
||||
expect(next?.cursor).toBe(4);
|
||||
expect(next?.messages).toEqual(initial.messages);
|
||||
});
|
||||
|
||||
it("holds a tool call that lands mid-sentence until the sentence completes", () => {
|
||||
const initial = snapshot([]);
|
||||
|
||||
|
||||
@ -249,6 +249,7 @@ export function isThreadSnapshotEvent(event: ProductEvent): boolean {
|
||||
event.type === "thread.cloud_agent" ||
|
||||
event.type === "agent.tool.called" ||
|
||||
event.type === "agent.tool.completed" ||
|
||||
event.type === "thread.computer" ||
|
||||
event.type === "thread.message.created" ||
|
||||
event.type === "thread.message.updated" ||
|
||||
event.type === "thread.message.reaction" ||
|
||||
@ -439,7 +440,7 @@ export function reduceThreadSnapshot(
|
||||
};
|
||||
return { ...prev, cursor: event.seq, messages: [...remaining, next] };
|
||||
}
|
||||
if (event.type === "agent.tool.completed") {
|
||||
if (event.type === "agent.tool.completed" || event.type === "thread.computer") {
|
||||
return { ...prev, cursor: event.seq };
|
||||
}
|
||||
if (event.type === "thread.subagent") {
|
||||
|
||||
@ -110,6 +110,7 @@ import {
|
||||
Copy,
|
||||
Folder,
|
||||
Gauge,
|
||||
Globe,
|
||||
GripVertical,
|
||||
LayoutGrid,
|
||||
Lock,
|
||||
@ -158,6 +159,7 @@ import { AskCard } from "../components/AskCard";
|
||||
import { ActiveBotGlyph, CollaborationMarker } from "../components/ai/CollaborationMarker";
|
||||
import { DelegatedTaskCard, SentMessageCard } from "../components/ai/DelegatedTaskCard";
|
||||
import { ProjectCreatedCard } from "../components/ai/ProjectCreatedCard";
|
||||
import { InAppBrowser } from "../components/browser/InAppBrowser";
|
||||
import { CloudAgentCard } from "../components/CloudAgentCard";
|
||||
import { ComputerMaintenanceActions } from "../components/ComputerMaintenanceActions";
|
||||
import {
|
||||
@ -165,6 +167,7 @@ import {
|
||||
computersAreUnavailable,
|
||||
} from "../components/ComputersUnavailableHint";
|
||||
import { ComputerUpdateProgress } from "../components/ComputerUpdateProgress";
|
||||
import { AgentCursor } from "../components/computer/AgentCursor";
|
||||
import { MainInbox } from "../components/MainInbox";
|
||||
import { MessageHoverMetadata } from "../components/MessageHoverMetadata";
|
||||
import { OoGroupStage, OoStage } from "../components/stage/OoStage";
|
||||
@ -176,6 +179,7 @@ import { VoiceAuroraRim } from "../components/voice/VoiceAurora";
|
||||
import { useSidebarResize } from "../hooks/useSidebarResize";
|
||||
import { readActivityMode, writeActivityMode } from "../lib/activity-mode";
|
||||
import { needsYouChipLabel, rosterActivitySubtitle, rosterStatusDot } from "../lib/agent-activity";
|
||||
import { agentPointerFeed } from "../lib/agent-pointer";
|
||||
import type { ArtifactTarget } from "../lib/artifact-open";
|
||||
import { authClient } from "../lib/auth";
|
||||
import { takeInitialBootstrap } from "../lib/bootstrap";
|
||||
@ -189,8 +193,6 @@ import {
|
||||
requestBrowserNotificationPermission,
|
||||
shouldNotifyBrowser,
|
||||
} from "../lib/browser-notifications";
|
||||
import type { BrowserPreviewCursor } from "../lib/browser-preview-context";
|
||||
import { browserPreviewActivity } from "../lib/browser-preview-context";
|
||||
import { loadComputerScreen } from "../lib/computer-screen";
|
||||
import { desktopBridge } from "../lib/desktop";
|
||||
import { scheduleFocusPrompt } from "../lib/focus-prompt";
|
||||
@ -380,31 +382,6 @@ function readCollapsedSidebarSections(userId: string | null | undefined): Set<st
|
||||
}
|
||||
}
|
||||
|
||||
function copyBrowserPreviewActivities(
|
||||
activities: ReadonlyMap<string, BrowserPreviewCursor>,
|
||||
): Map<string, BrowserPreviewCursor> {
|
||||
return new Map(activities);
|
||||
}
|
||||
|
||||
function sameBrowserPreviewActivities(
|
||||
current: ReadonlyMap<string, BrowserPreviewCursor>,
|
||||
next: ReadonlyMap<string, BrowserPreviewCursor>,
|
||||
): boolean {
|
||||
if (current.size !== next.size) return false;
|
||||
for (const [id, cursor] of next) {
|
||||
const existing = current.get(id);
|
||||
if (
|
||||
!existing ||
|
||||
existing.x !== cursor.x ||
|
||||
existing.y !== cursor.y ||
|
||||
existing.label !== cursor.label
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export type NavPillar = "heute" | "team" | "projekte";
|
||||
|
||||
function stripSpaceTitlePrefix(title: string | null, spaceName: string): string | null {
|
||||
@ -478,16 +455,7 @@ export function ShellPage() {
|
||||
setBotsSidebarCollapsed(readBotsSidebarCollapsed(userId));
|
||||
}, [userId]);
|
||||
|
||||
// ── Browser Preview ──────────────────────────────────────────────
|
||||
const bridge = desktopBridge();
|
||||
const [isDesktop] = useState(Boolean(bridge?.browserPreview));
|
||||
const [panel, setPanel] = useState<Panel>(null);
|
||||
const [browserUrl, setBrowserUrl] = useState("http://localhost:5173");
|
||||
const [devServerRunning, setDevServerRunning] = useState(false);
|
||||
const [browserLoading, setBrowserLoading] = useState(false);
|
||||
const [agentActivities, setAgentActivities] = useState<
|
||||
Map<string, { x: number; y: number; label: string }>
|
||||
>(new Map());
|
||||
|
||||
const [showCreateBotModal, setShowCreateBotModal] = useState(false);
|
||||
|
||||
@ -503,60 +471,6 @@ export function ShellPage() {
|
||||
useSensor(TouchSensor, { activationConstraint: { distance: 8 } }),
|
||||
);
|
||||
|
||||
// Browser Preview lifecycle
|
||||
useEffect(() => {
|
||||
const bp = bridge?.browserPreview;
|
||||
if (!bp) return;
|
||||
const unsubNav = bp.onNavigate((url) => setBrowserUrl(url));
|
||||
const unsubLoading = bp.onLoadingProgress((p) => setBrowserLoading(p < 1));
|
||||
bp.getState()
|
||||
.then((s) => {
|
||||
setBrowserUrl(s.url);
|
||||
setDevServerRunning(s.devServerRunning);
|
||||
})
|
||||
.catch(() => {});
|
||||
return () => {
|
||||
unsubNav();
|
||||
unsubLoading();
|
||||
};
|
||||
}, [bridge]);
|
||||
|
||||
const startDevServer = useCallback(async () => {
|
||||
try {
|
||||
await bridge?.browserPreview?.devServer.start();
|
||||
setDevServerRunning(true);
|
||||
} catch {
|
||||
/* */
|
||||
}
|
||||
}, [bridge]);
|
||||
const stopDevServer = useCallback(async () => {
|
||||
try {
|
||||
await bridge?.browserPreview?.devServer.stop();
|
||||
setDevServerRunning(false);
|
||||
} catch {
|
||||
/* */
|
||||
}
|
||||
}, [bridge]);
|
||||
const navigateBrowser = useCallback(
|
||||
(url: string) => {
|
||||
bridge?.browserPreview?.navigate(url).catch(() => {});
|
||||
},
|
||||
[bridge],
|
||||
);
|
||||
const reloadBrowser = useCallback(() => {
|
||||
bridge?.browserPreview?.reload().catch(() => {});
|
||||
}, [bridge]);
|
||||
|
||||
// Agent activity → browser preview cursors
|
||||
useEffect(() => {
|
||||
const unsub = browserPreviewActivity.subscribe((acts) => {
|
||||
setAgentActivities((current) =>
|
||||
sameBrowserPreviewActivities(current, acts) ? current : copyBrowserPreviewActivities(acts),
|
||||
);
|
||||
});
|
||||
return unsub;
|
||||
}, []);
|
||||
|
||||
const [query, setQuery] = useState("");
|
||||
const [searchHits, setSearchHits] = useState<SearchHit[]>([]);
|
||||
const [searchLoading, setSearchLoading] = useState(false);
|
||||
@ -1540,6 +1454,8 @@ export function ShellPage() {
|
||||
void refreshThread(active.id).catch(() => undefined);
|
||||
} else if (isComputerStatusEvent(event)) {
|
||||
void refreshComputerScreen(active.id).catch(() => undefined);
|
||||
} else if (event.type === "thread.computer") {
|
||||
agentPointerFeed.publish(event.botId, event.payload, event.createdAt);
|
||||
}
|
||||
},
|
||||
});
|
||||
@ -2235,20 +2151,6 @@ export function ShellPage() {
|
||||
}));
|
||||
}, [activeSnapshot, enrichStageMember, inGroup, resolveTranscriptBot, workingRuns]);
|
||||
|
||||
// Working bots → browser preview activity cursors
|
||||
useEffect(() => {
|
||||
if (workingBots.length === 0) return;
|
||||
for (const bot of workingBots) {
|
||||
if (!bot.botId) continue;
|
||||
const label = bot.name ?? bot.botId.slice(0, 8);
|
||||
browserPreviewActivity.set(bot.botId, 50, 40, label);
|
||||
}
|
||||
const activeBotIds = new Set(workingBots.map((b) => b.botId).filter(Boolean));
|
||||
for (const id of browserPreviewActivity.snapshot().keys()) {
|
||||
if (!activeBotIds.has(id)) browserPreviewActivity.clear(id);
|
||||
}
|
||||
}, [workingBots]);
|
||||
|
||||
const encounterMode = Boolean((active || activeGroup) && !showProjectTimeline && !callOpen);
|
||||
const voiceEncounterActive = encounterMode || callOpen || voiceEntry;
|
||||
const voiceCallBot = useMemo(() => {
|
||||
@ -4490,6 +4392,16 @@ export function ShellPage() {
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
title={t`Browser`}
|
||||
aria-label={t`Browser`}
|
||||
onClick={() => setPanel(panel === "browser" ? null : "browser")}
|
||||
data-active={panel === "browser" ? "" : undefined}
|
||||
className="app-no-drag grid h-[30px] w-[34px] place-items-center rounded-[9px] hover:bg-accent data-active:bg-accent"
|
||||
>
|
||||
<Globe strokeWidth={1.7} className="text-foreground/75 size-icon-lg" />
|
||||
</button>
|
||||
{!inGroup && active ? (
|
||||
<button
|
||||
type="button"
|
||||
@ -4718,7 +4630,9 @@ export function ShellPage() {
|
||||
onPointerDown={startResizingRight}
|
||||
className="absolute -left-2 top-0 bottom-0 w-4 cursor-col-resize z-50 hidden md:block"
|
||||
/>
|
||||
{panel && (active || activeGroup || panel === "create" || panel === "create-group") ? (
|
||||
{panel === "browser" ? (
|
||||
<InAppBrowser onClose={() => setPanel(null)} />
|
||||
) : panel && (active || activeGroup || panel === "create" || panel === "create-group") ? (
|
||||
<div className="rk-scroll h-full w-full overflow-y-auto px-5 lg:px-7 2xl:px-9 py-[17px]">
|
||||
{panel !== "routine" &&
|
||||
panel !== "create" &&
|
||||
@ -4758,17 +4672,6 @@ export function ShellPage() {
|
||||
<Settings strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
) : null}
|
||||
{!panel && active ? (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
aria-label={t`Browser preview`}
|
||||
onClick={() => setPanel("browser")}
|
||||
className="text-muted-foreground"
|
||||
>
|
||||
<PanelRight strokeWidth={1.7} className="size-icon-md" />
|
||||
</Button>
|
||||
) : null}
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
@ -4780,80 +4683,6 @@ export function ShellPage() {
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
{panel === "browser" ? (
|
||||
<div className="flex h-full flex-col">
|
||||
<div className="flex items-center gap-1.5 border-b px-3 py-2">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
onClick={reloadBrowser}
|
||||
aria-label={t`Reload`}
|
||||
className="size-icon-sm text-muted-foreground"
|
||||
>
|
||||
<RefreshCw strokeWidth={1.7} className="size-4" />
|
||||
</Button>
|
||||
<input
|
||||
type="url"
|
||||
value={browserUrl}
|
||||
onChange={(e) => setBrowserUrl(e.target.value)}
|
||||
onBlur={() => navigateBrowser(browserUrl)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter") navigateBrowser(browserUrl);
|
||||
}}
|
||||
placeholder="http://localhost:5173"
|
||||
className="flex-1 rounded-md bg-muted/60 px-2.5 py-1 text-xs text-foreground placeholder:text-muted-foreground/60 outline-none focus:ring-1 focus:ring-ring/50"
|
||||
/>
|
||||
{isDesktop ? (
|
||||
devServerRunning ? (
|
||||
<span className="inline-flex items-center gap-1 rounded-full bg-muted px-2 py-0.5 text-ui-label font-medium text-foreground">
|
||||
<span className="size-1.5 rounded-full bg-success" />
|
||||
<Trans>Dev</Trans>
|
||||
</span>
|
||||
) : (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
onClick={startDevServer}
|
||||
className="text-ui-label text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<Trans>Start Dev</Trans>
|
||||
</Button>
|
||||
)
|
||||
) : null}
|
||||
{browserLoading && (
|
||||
<div className="h-0.5 w-full rounded-full bg-muted/30">
|
||||
<div className="h-full w-2/3 animate-pulse rounded-full bg-primary/60" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="relative flex-1 overflow-hidden bg-background">
|
||||
<div className="relative h-full w-full" data-testid="browser-preview">
|
||||
{agentActivities.size > 0 && (
|
||||
<div className="pointer-events-none absolute inset-0">
|
||||
{Array.from(agentActivities.entries()).map(([agentId, activity]) => {
|
||||
const cx = Math.min(100, Math.max(0, activity.x)) + "%";
|
||||
const cy = Math.min(100, Math.max(0, activity.y)) + "%";
|
||||
return (
|
||||
<div
|
||||
key={agentId}
|
||||
className="absolute transition-[left,top] duration-300 ease-out"
|
||||
style={{ left: cx, top: cy }}
|
||||
>
|
||||
<div className="relative">
|
||||
<div className="size-3 rounded-full bg-primary/60 ring-2 ring-primary" />
|
||||
<div className="absolute -right-1 -top-4 whitespace-nowrap rounded bg-overlay px-1.5 py-0.5 text-ui-label text-foreground shadow-sm">
|
||||
{activity.label}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
{panel === "computer" && active ? (
|
||||
<div>
|
||||
<div
|
||||
@ -4867,14 +4696,17 @@ export function ShellPage() {
|
||||
) : computer?.kind === "desktop" ? (
|
||||
<DesktopKindEmptyState className="grid h-full place-items-center px-6 text-center text-sm text-muted-foreground/80" />
|
||||
) : computer?.state === "running" && embeddedScreenUrl && !computerScreenError ? (
|
||||
<iframe
|
||||
title={t`Bot screen preview`}
|
||||
src={embeddedScreenUrl}
|
||||
sandbox={screenIframeSandbox(embeddedScreenUrl)}
|
||||
className="h-full w-full border-0 bg-black"
|
||||
allow="clipboard-read; clipboard-write"
|
||||
style={{ pointerEvents: "none" }}
|
||||
/>
|
||||
<>
|
||||
<iframe
|
||||
title={t`Bot screen preview`}
|
||||
src={embeddedScreenUrl}
|
||||
sandbox={screenIframeSandbox(embeddedScreenUrl)}
|
||||
className="h-full w-full border-0 bg-black"
|
||||
allow="clipboard-read; clipboard-write"
|
||||
style={{ pointerEvents: "none" }}
|
||||
/>
|
||||
<AgentCursor botId={active.id} color={active.color} name={active.name} />
|
||||
</>
|
||||
) : (
|
||||
<div className="grid h-full place-items-center px-6 text-center text-sm text-muted-foreground/80">
|
||||
{computerScreenError ??
|
||||
@ -5764,6 +5596,9 @@ export function ShellPage() {
|
||||
pointerEvents: recordingSkill || !hasControl ? "none" : "auto",
|
||||
}}
|
||||
/>
|
||||
{active && !hasControl ? (
|
||||
<AgentCursor botId={active.id} color={active.color} name={active.name} />
|
||||
) : null}
|
||||
{active ? (
|
||||
<TeachCaptureOverlay
|
||||
botId={active.id}
|
||||
|
||||
@ -240,6 +240,26 @@ body,
|
||||
-webkit-app-region: no-drag;
|
||||
}
|
||||
|
||||
/* In-app browser load indicator: a short bar sweeping under the address bar. */
|
||||
.rk-browser-progress {
|
||||
animation: rkBrowserProgress 1.1s cubic-bezier(0.65, 0, 0.35, 1) infinite;
|
||||
}
|
||||
@keyframes rkBrowserProgress {
|
||||
from {
|
||||
transform: translateX(-100%);
|
||||
}
|
||||
to {
|
||||
transform: translateX(300%);
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.rk-browser-progress {
|
||||
animation: none;
|
||||
width: 100%;
|
||||
opacity: 0.6;
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes rkPulse {
|
||||
0%,
|
||||
100% {
|
||||
|
||||
@ -412,10 +412,39 @@ EVAL_HELPERS = r"""
|
||||
})
|
||||
.join("\n");
|
||||
}
|
||||
// Screen pixels of the element center's box, only while that center is in the viewport.
|
||||
// Drives the visible bot cursor; the model never sees it.
|
||||
function screenBox(el, viewport) {
|
||||
const rect = el.getBoundingClientRect();
|
||||
const cx = rect.left + rect.width / 2;
|
||||
const cy = rect.top + rect.height / 2;
|
||||
if (cx < 0 || cy < 0 || cx > window.innerWidth || cy > window.innerHeight) return undefined;
|
||||
const box = [
|
||||
viewport.x + rect.left * viewport.scale,
|
||||
viewport.y + rect.top * viewport.scale,
|
||||
rect.width * viewport.scale,
|
||||
rect.height * viewport.scale,
|
||||
].map(Math.round);
|
||||
return box[0] >= 0 && box[1] >= 0 && box[2] > 0 && box[3] > 0 ? box : undefined;
|
||||
}
|
||||
function viewportOnScreen() {
|
||||
const scale = window.devicePixelRatio || 1;
|
||||
// Browser chrome sits above the page; side borders are split evenly.
|
||||
return {
|
||||
scale,
|
||||
x: (window.screenX + Math.max(0, (window.outerWidth - window.innerWidth) / 2)) * scale,
|
||||
y: (window.screenY + Math.max(0, window.outerHeight - window.innerHeight)) * scale,
|
||||
screen: {
|
||||
width: Math.round(window.screen.width * scale),
|
||||
height: Math.round(window.screen.height * scale),
|
||||
},
|
||||
};
|
||||
}
|
||||
window.__rakazoPageBrowser = {
|
||||
snapshot() {
|
||||
refMap.clear();
|
||||
const elements = [];
|
||||
const viewport = viewportOnScreen();
|
||||
for (const el of document.querySelectorAll("body *")) {
|
||||
if (!isInteractive(el) || !visible(el)) continue;
|
||||
const ref = newRef();
|
||||
@ -428,12 +457,14 @@ EVAL_HELPERS = r"""
|
||||
: ["checkbox", "radio"].includes(el.type) ? (el.checked ? "checked" : "unchecked")
|
||||
: "value" in el ? String(el.value ?? "").slice(0, 2000) : undefined,
|
||||
tag: el.tagName.toLowerCase(),
|
||||
box: screenBox(el, viewport),
|
||||
});
|
||||
if (elements.length >= 80) break;
|
||||
}
|
||||
return {
|
||||
url: location.href,
|
||||
title: (document.title || "").slice(0, 200),
|
||||
screen: viewport.screen,
|
||||
elements,
|
||||
tree: (document.body?.innerText || "").slice(0, 12000) + "\n" + formatTree(elements)
|
||||
+ (document.querySelector("iframe,frame") ? "\nEmbedded frames require desktop tools." : ""),
|
||||
|
||||
@ -748,14 +748,21 @@ export interface BrowserSnapshotNode {
|
||||
name: string;
|
||||
value?: string;
|
||||
tag?: string;
|
||||
/** Screen pixels when the element's center is in view; drives the visible bot cursor only. */
|
||||
box?: BrowserScreenBox;
|
||||
}
|
||||
|
||||
/** `[x, y, width, height]` in screen pixels of the computer display. */
|
||||
export type BrowserScreenBox = [number, number, number, number];
|
||||
|
||||
export interface BrowserSnapshotResult {
|
||||
url: string;
|
||||
title: string;
|
||||
/** Compact accessibility-style tree for the model. */
|
||||
tree: string;
|
||||
elements: BrowserSnapshotNode[];
|
||||
/** Display size the element boxes are relative to. */
|
||||
screen?: { width: number; height: number };
|
||||
fallback?: "computer_act";
|
||||
error?: string;
|
||||
}
|
||||
@ -781,6 +788,7 @@ export interface BrowserActResult {
|
||||
/** Snapshot after the actions when available. */
|
||||
tree?: string;
|
||||
elements?: BrowserSnapshotNode[];
|
||||
screen?: { width: number; height: number };
|
||||
fallback?: "computer_act";
|
||||
error?: string;
|
||||
}
|
||||
|
||||
@ -5,6 +5,14 @@ import type {
|
||||
BrowserProvider,
|
||||
ComputerRef,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { ComputerPointerPayload } from "@rakazo/contracts";
|
||||
import { type BrowserPointerTargets, withoutBrowserGeometry } from "./computer-pointer.js";
|
||||
|
||||
/** Optional hooks that let the visible bot cursor follow page-browser actions. */
|
||||
export interface BrowserPointerHooks {
|
||||
targets: BrowserPointerTargets;
|
||||
announce: (payload: ComputerPointerPayload) => Promise<void>;
|
||||
}
|
||||
|
||||
const MAX_BROWSER_ACTIONS = 24;
|
||||
|
||||
@ -13,6 +21,7 @@ export async function browserNavigateFromTool(
|
||||
computer: ComputerRef,
|
||||
context: AdapterContext,
|
||||
args: Record<string, unknown>,
|
||||
pointer?: BrowserPointerHooks,
|
||||
) {
|
||||
const url = String(args.url ?? "").trim();
|
||||
try {
|
||||
@ -23,6 +32,7 @@ export async function browserNavigateFromTool(
|
||||
fallback: "computer_act" as const,
|
||||
};
|
||||
}
|
||||
pointer?.targets.forget();
|
||||
const result = await browser.navigate(computer, { url, signal: context.signal }, context);
|
||||
return formatBrowserResult(result);
|
||||
} catch (error) {
|
||||
@ -39,11 +49,13 @@ export async function browserSnapshotFromTool(
|
||||
computer: ComputerRef,
|
||||
context: AdapterContext,
|
||||
args: Record<string, unknown>,
|
||||
pointer?: BrowserPointerHooks,
|
||||
) {
|
||||
void args;
|
||||
try {
|
||||
const result = await browser.snapshot(computer, { signal: context.signal }, context);
|
||||
return formatBrowserResult(result);
|
||||
pointer?.targets.remember(result);
|
||||
return formatBrowserResult(withoutBrowserGeometry(result));
|
||||
} catch (error) {
|
||||
context.signal.throwIfAborted();
|
||||
return {
|
||||
@ -58,11 +70,16 @@ export async function browserActFromTool(
|
||||
computer: ComputerRef,
|
||||
context: AdapterContext,
|
||||
args: Record<string, unknown>,
|
||||
pointer?: BrowserPointerHooks,
|
||||
) {
|
||||
try {
|
||||
const actions = parseBrowserActions(args.actions);
|
||||
const announcement = pointer?.targets.payloadFor(actions);
|
||||
if (announcement) await pointer?.announce(announcement);
|
||||
const result = await browser.act(computer, { actions, signal: context.signal }, context);
|
||||
return formatBrowserResult(result);
|
||||
if (result.elements) pointer?.targets.remember(result);
|
||||
else pointer?.targets.forget();
|
||||
return formatBrowserResult(withoutBrowserGeometry(result));
|
||||
} catch (error) {
|
||||
context.signal.throwIfAborted();
|
||||
return {
|
||||
|
||||
@ -115,6 +115,7 @@ export class ComputerBrowserProvider implements BrowserProvider {
|
||||
title: typeof live.title === "string" ? live.title : "",
|
||||
tree: typeof live.tree === "string" ? live.tree : formatTree(elements),
|
||||
elements,
|
||||
...(live.screen ? { screen: live.screen } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@ -150,6 +151,7 @@ export class ComputerBrowserProvider implements BrowserProvider {
|
||||
title: typeof live.title === "string" ? live.title : "",
|
||||
tree: typeof live.tree === "string" ? live.tree : elements ? formatTree(elements) : undefined,
|
||||
elements,
|
||||
...(live.screen ? { screen: live.screen } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
164
packages/adapters/src/computer-pointer.test.ts
Normal file
164
packages/adapters/src/computer-pointer.test.ts
Normal file
@ -0,0 +1,164 @@
|
||||
import type {
|
||||
AdapterContext,
|
||||
BrowserActRequest,
|
||||
BrowserActResult,
|
||||
BrowserProvider,
|
||||
BrowserSnapshotResult,
|
||||
ComputerRef,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import { ComputerPointerPayload } from "@rakazo/contracts";
|
||||
import { DEFAULT_COMPUTER_SCREEN } from "@rakazo/core";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { browserActFromTool, browserSnapshotFromTool } from "./browser-tools.js";
|
||||
import {
|
||||
BrowserPointerTargets,
|
||||
computerPointerPayload,
|
||||
computerPointerSteps,
|
||||
} from "./computer-pointer.js";
|
||||
import { parseComputerActions } from "./computer-tools.js";
|
||||
|
||||
const context: AdapterContext = {
|
||||
operationId: "1",
|
||||
traceId: "1",
|
||||
spaceId: "w",
|
||||
userId: "u",
|
||||
signal: new AbortController().signal,
|
||||
};
|
||||
const computer: ComputerRef = { id: "c1", botId: "b1", kind: "docker", providerRef: "c1" };
|
||||
|
||||
describe("computer pointer steps", () => {
|
||||
it("maps a computer_act batch to cursor steps in order", () => {
|
||||
const steps = computerPointerSteps(
|
||||
parseComputerActions([
|
||||
{ kind: "move", x: 10, y: 20 },
|
||||
{ kind: "click", x: 30, y: 40, double: true },
|
||||
{ kind: "click", x: 50, y: 60, button: "right" },
|
||||
{ kind: "type", text: "hello" },
|
||||
{ kind: "key", key: "Enter" },
|
||||
{ kind: "wait", ms: 100 },
|
||||
{ kind: "scroll", direction: "down" },
|
||||
{ kind: "down", x: 1, y: 2 },
|
||||
{ kind: "up", x: 3, y: 4 },
|
||||
]),
|
||||
);
|
||||
expect(steps).toEqual([
|
||||
{ action: "move", x: 10, y: 20 },
|
||||
{ action: "double_click", x: 30, y: 40 },
|
||||
{ action: "right_click", x: 50, y: 60 },
|
||||
{ action: "type" },
|
||||
{ action: "key" },
|
||||
{ action: "scroll" },
|
||||
{ action: "down", x: 1, y: 2 },
|
||||
{ action: "up", x: 3, y: 4 },
|
||||
]);
|
||||
const payload = computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps);
|
||||
expect(ComputerPointerPayload.parse(payload)).toEqual(payload);
|
||||
});
|
||||
|
||||
it("keeps two separate clicks at different points apart", () => {
|
||||
expect(
|
||||
computerPointerSteps(
|
||||
parseComputerActions([
|
||||
{ kind: "click", x: 1, y: 1 },
|
||||
{ kind: "click", x: 2, y: 2 },
|
||||
]),
|
||||
),
|
||||
).toEqual([
|
||||
{ action: "click", x: 1, y: 1 },
|
||||
{ action: "click", x: 2, y: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("announces nothing for a batch without visible steps", () => {
|
||||
const steps = computerPointerSteps(parseComputerActions([{ kind: "wait", ms: 10 }]));
|
||||
expect(computerPointerPayload(DEFAULT_COMPUTER_SCREEN, steps)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser pointer targets", () => {
|
||||
it("points at element centers and stops at the first unknown ref", () => {
|
||||
const targets = new BrowserPointerTargets();
|
||||
targets.remember({
|
||||
screen: { width: 1280, height: 800 },
|
||||
elements: [
|
||||
{ ref: "e1", role: "textbox", name: "Name", box: [100, 200, 40, 20] },
|
||||
{ ref: "e2", role: "button", name: "Save", box: [300, 400, 11, 11] },
|
||||
{ ref: "e3", role: "button", name: "Hidden" },
|
||||
// Untrusted helper output never becomes a cursor target.
|
||||
{ ref: "e4", role: "button", name: "Bad", box: [-5, 0, 10, 10] as never },
|
||||
],
|
||||
});
|
||||
expect(
|
||||
targets.payloadFor([
|
||||
{ kind: "fill", ref: "e1", text: "x" },
|
||||
{ kind: "click", ref: "e2" },
|
||||
{ kind: "click", ref: "e3" },
|
||||
{ kind: "click", ref: "e1" },
|
||||
]),
|
||||
).toEqual({
|
||||
kind: "pointer",
|
||||
screen: { width: 1280, height: 800 },
|
||||
steps: [
|
||||
{ action: "type", x: 120, y: 210 },
|
||||
{ action: "click", x: 306, y: 406 },
|
||||
],
|
||||
});
|
||||
expect(targets.payloadFor([{ kind: "click", ref: "e4" }])).toBeUndefined();
|
||||
targets.forget();
|
||||
expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
|
||||
});
|
||||
|
||||
it("needs a reported screen size", () => {
|
||||
const targets = new BrowserPointerTargets();
|
||||
targets.remember({ elements: [{ ref: "e1", role: "button", name: "A", box: [1, 1, 2, 2] }] });
|
||||
expect(targets.payloadFor([{ kind: "click", ref: "e1" }])).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("page browser tools with a visible cursor", () => {
|
||||
it("announces before acting and keeps geometry away from the model", async () => {
|
||||
const order: string[] = [];
|
||||
const snapshot: BrowserSnapshotResult = {
|
||||
url: "https://example.test/",
|
||||
title: "Example",
|
||||
tree: '- button "Save" [e1]',
|
||||
screen: { width: 1280, height: 800 },
|
||||
elements: [{ ref: "e1", role: "button", name: "Save", box: [10, 20, 30, 40] }],
|
||||
};
|
||||
const browser = {
|
||||
snapshot: async () => snapshot,
|
||||
act: async (_computer: ComputerRef, request: BrowserActRequest) => {
|
||||
order.push(`act:${request.actions.length}`);
|
||||
return { ok: true, completed: 1, ...snapshot } satisfies BrowserActResult;
|
||||
},
|
||||
} as unknown as BrowserProvider;
|
||||
const announced: unknown[] = [];
|
||||
const pointer = {
|
||||
targets: new BrowserPointerTargets(),
|
||||
announce: async (payload: ComputerPointerPayload) => {
|
||||
order.push("announce");
|
||||
announced.push(payload);
|
||||
},
|
||||
};
|
||||
|
||||
const seen = await browserSnapshotFromTool(browser, computer, context, {}, pointer);
|
||||
expect(JSON.stringify(seen)).not.toMatch(/box|screen/);
|
||||
|
||||
const acted = await browserActFromTool(
|
||||
browser,
|
||||
computer,
|
||||
context,
|
||||
{ actions: [{ kind: "click", ref: "e1" }] },
|
||||
pointer,
|
||||
);
|
||||
expect(order).toEqual(["announce", "act:1"]);
|
||||
expect(announced).toEqual([
|
||||
{
|
||||
kind: "pointer",
|
||||
screen: { width: 1280, height: 800 },
|
||||
steps: [{ action: "click", x: 25, y: 40 }],
|
||||
},
|
||||
]);
|
||||
expect(JSON.stringify(acted)).not.toMatch(/box|screen/);
|
||||
});
|
||||
});
|
||||
130
packages/adapters/src/computer-pointer.ts
Normal file
130
packages/adapters/src/computer-pointer.ts
Normal file
@ -0,0 +1,130 @@
|
||||
import type {
|
||||
BrowserActStep,
|
||||
BrowserScreenBox,
|
||||
BrowserSnapshotNode,
|
||||
ComputerAction,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import {
|
||||
type ComputerPointerPayload,
|
||||
type ComputerPointerStep,
|
||||
MAX_COMPUTER_POINTER_STEPS,
|
||||
} from "@rakazo/contracts";
|
||||
|
||||
export type ComputerScreenSize = ComputerPointerPayload["screen"];
|
||||
|
||||
/** Pointer steps for a validated computer_act batch, in execution order. */
|
||||
export function computerPointerSteps(actions: ComputerAction[]): ComputerPointerStep[] {
|
||||
const steps: ComputerPointerStep[] = [];
|
||||
for (let index = 0; index < actions.length; index += 1) {
|
||||
const action = actions[index]!;
|
||||
if (action.kind === "pointer") {
|
||||
const next = actions[index + 1];
|
||||
// parseComputerActions expands a double click into two identical clicks.
|
||||
if (
|
||||
action.type === "click" &&
|
||||
next?.kind === "pointer" &&
|
||||
next.type === "click" &&
|
||||
next.x === action.x &&
|
||||
next.y === action.y &&
|
||||
next.button === action.button
|
||||
) {
|
||||
steps.push({ action: "double_click", x: action.x, y: action.y });
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
const kind =
|
||||
action.type === "click"
|
||||
? action.button === "right"
|
||||
? "right_click"
|
||||
: "click"
|
||||
: action.type;
|
||||
steps.push({ action: kind, x: action.x, y: action.y });
|
||||
} else if (action.kind === "clipboard") {
|
||||
steps.push({ action: "type" });
|
||||
} else if (action.kind === "key" || action.kind === "scroll") {
|
||||
steps.push({ action: action.kind });
|
||||
}
|
||||
}
|
||||
return steps.slice(0, MAX_COMPUTER_POINTER_STEPS);
|
||||
}
|
||||
|
||||
export function computerPointerPayload(
|
||||
screen: ComputerScreenSize,
|
||||
steps: ComputerPointerStep[],
|
||||
): ComputerPointerPayload | undefined {
|
||||
if (steps.length === 0) return undefined;
|
||||
return { kind: "pointer", screen, steps: steps.slice(0, MAX_COMPUTER_POINTER_STEPS) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Remembers where the page browser last saw each element on screen, so a browser_act batch
|
||||
* can be announced as pointer steps before it runs. Geometry never reaches the model.
|
||||
*/
|
||||
export class BrowserPointerTargets {
|
||||
private boxes = new Map<string, BrowserScreenBox>();
|
||||
private screen: ComputerScreenSize | undefined;
|
||||
|
||||
remember(result: { elements?: BrowserSnapshotNode[]; screen?: ComputerScreenSize }) {
|
||||
if (!Array.isArray(result.elements)) return;
|
||||
this.boxes = new Map();
|
||||
this.screen = validScreen(result.screen);
|
||||
for (const element of result.elements) {
|
||||
if (validBox(element.box)) this.boxes.set(element.ref, element.box);
|
||||
}
|
||||
}
|
||||
|
||||
forget() {
|
||||
this.boxes = new Map();
|
||||
this.screen = undefined;
|
||||
}
|
||||
|
||||
payloadFor(actions: BrowserActStep[]): ComputerPointerPayload | undefined {
|
||||
if (!this.screen) return undefined;
|
||||
const steps: ComputerPointerStep[] = [];
|
||||
for (const action of actions) {
|
||||
const box = this.boxes.get(action.ref);
|
||||
// Later steps may target elements an earlier step reveals; stop at the first unknown one.
|
||||
if (!box) break;
|
||||
const [x, y, width, height] = box;
|
||||
steps.push({
|
||||
action: action.kind === "click" ? "click" : "type",
|
||||
x: Math.round(x + width / 2),
|
||||
y: Math.round(y + height / 2),
|
||||
});
|
||||
}
|
||||
return computerPointerPayload(this.screen, steps);
|
||||
}
|
||||
}
|
||||
|
||||
/** Strips screen geometry from page-browser results before they reach the model. */
|
||||
export function withoutBrowserGeometry<T extends { elements?: BrowserSnapshotNode[] }>(
|
||||
result: T,
|
||||
): Omit<T, "screen"> {
|
||||
const { screen: _screen, ...rest } = result as T & { screen?: unknown };
|
||||
if (!Array.isArray(rest.elements)) return rest;
|
||||
return {
|
||||
...rest,
|
||||
elements: rest.elements.map(({ box: _box, ...element }) => element),
|
||||
};
|
||||
}
|
||||
|
||||
function validScreen(value: unknown): ComputerScreenSize | undefined {
|
||||
if (!value || typeof value !== "object") return undefined;
|
||||
const { width, height } = value as Record<string, unknown>;
|
||||
if (!isPixel(width) || !isPixel(height) || width === 0 || height === 0) return undefined;
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function validBox(value: unknown): value is BrowserScreenBox {
|
||||
return (
|
||||
Array.isArray(value) &&
|
||||
value.length === 4 &&
|
||||
value.every(isPixel) &&
|
||||
(value[2] as number) > 0 &&
|
||||
(value[3] as number) > 0
|
||||
);
|
||||
}
|
||||
|
||||
function isPixel(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100_000;
|
||||
}
|
||||
@ -25,7 +25,7 @@ import {
|
||||
routineWakeupJob,
|
||||
runContinueJob,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { ModelTier, RunStatus } from "@rakazo/contracts";
|
||||
import type { ComputerPointerPayload, ModelTier, RunStatus } from "@rakazo/contracts";
|
||||
import {
|
||||
ATTACHMENT_MAX_BYTES,
|
||||
BOT_DESCRIPTION_MAX_LENGTH,
|
||||
@ -45,6 +45,7 @@ import {
|
||||
applyJudgeDecision,
|
||||
assertTransition,
|
||||
botMessageAllowsSilence,
|
||||
computerScreenSize,
|
||||
connectorKindFromToolName,
|
||||
containsSecret,
|
||||
createStreamingRedactor,
|
||||
@ -163,6 +164,7 @@ import {
|
||||
} from "./bot-secrets.js";
|
||||
import { createBrowserProvider } from "./browser-provider-factory.js";
|
||||
import {
|
||||
type BrowserPointerHooks,
|
||||
browserActFromTool,
|
||||
browserNavigateFromTool,
|
||||
browserSnapshotFromTool,
|
||||
@ -192,6 +194,11 @@ import {
|
||||
renewComputerExecutionLease,
|
||||
screenLeaseIdForRun,
|
||||
} from "./computer-lifecycle.js";
|
||||
import {
|
||||
BrowserPointerTargets,
|
||||
computerPointerPayload,
|
||||
computerPointerSteps,
|
||||
} from "./computer-pointer.js";
|
||||
import { withComputerScreenAvailability } from "./computer-screens.js";
|
||||
import {
|
||||
assertTeamShellCommandAllowed,
|
||||
@ -1655,6 +1662,27 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
lastComputerFrameId = observation.frameId;
|
||||
return result;
|
||||
};
|
||||
// Best-effort: the visible cursor must never change a tool result or strand the run.
|
||||
const announcePointer = async (payload: ComputerPointerPayload) => {
|
||||
try {
|
||||
await deps.events.append({
|
||||
spaceId: run.spaceId,
|
||||
threadId: thread.id,
|
||||
botId: bot.id,
|
||||
type: "thread.computer",
|
||||
runId,
|
||||
payload,
|
||||
});
|
||||
} catch (error) {
|
||||
getLogger().warn("computer pointer append failed", {
|
||||
error: sanitizeConnectorError(error, runSecrets),
|
||||
});
|
||||
}
|
||||
};
|
||||
const browserPointer: BrowserPointerHooks = {
|
||||
targets: new BrowserPointerTargets(),
|
||||
announce: announcePointer,
|
||||
};
|
||||
|
||||
const pauseForApproval = () => {
|
||||
approvalPausePending = true;
|
||||
@ -2210,10 +2238,16 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
}
|
||||
workspaceCheckpoint.markDirty();
|
||||
return computerScreenToolResult(async () => {
|
||||
const actions = parseComputerActions(args.actions);
|
||||
const pointer = computerPointerPayload(
|
||||
computerScreenSize(computer.kind),
|
||||
computerPointerSteps(actions),
|
||||
);
|
||||
if (pointer) await announcePointer(pointer);
|
||||
const result = await deps.sandbox.act(
|
||||
computer,
|
||||
{
|
||||
actions: parseComputerActions(args.actions),
|
||||
actions,
|
||||
observe: args.observe !== false,
|
||||
settleMs: Number(args.settle_ms ?? 350),
|
||||
},
|
||||
@ -2608,7 +2642,10 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
: name === "browser_snapshot"
|
||||
? browserSnapshotFromTool
|
||||
: browserActFromTool;
|
||||
return computerScreenToolResult(() => tool(browser, computer, context, args), finish);
|
||||
return computerScreenToolResult(
|
||||
() => tool(browser, computer, context, args, browserPointer),
|
||||
finish,
|
||||
);
|
||||
}
|
||||
|
||||
if (name.startsWith("cloud_agent_")) {
|
||||
|
||||
@ -376,7 +376,11 @@ export function inferScript(
|
||||
{
|
||||
name: "computer_act",
|
||||
args: {
|
||||
actions: [{ kind: "type", text: typed }],
|
||||
// Click into the focused field first so the visible cursor has a target.
|
||||
actions: [
|
||||
{ kind: "click", x: 640, y: 400 },
|
||||
{ kind: "type", text: typed },
|
||||
],
|
||||
observe: true,
|
||||
},
|
||||
},
|
||||
|
||||
38
packages/contracts/src/computer-pointer.ts
Normal file
38
packages/contracts/src/computer-pointer.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import * as z from "zod";
|
||||
|
||||
export const ComputerPointerAction = z.enum([
|
||||
"move",
|
||||
"click",
|
||||
"double_click",
|
||||
"right_click",
|
||||
"down",
|
||||
"up",
|
||||
"type",
|
||||
"key",
|
||||
"scroll",
|
||||
]);
|
||||
export type ComputerPointerAction = z.infer<typeof ComputerPointerAction>;
|
||||
|
||||
/** Screen pixels; absent for keyboard steps that act wherever the pointer already is. */
|
||||
export const ComputerPointerStep = z.object({
|
||||
action: ComputerPointerAction,
|
||||
x: z.number().int().min(0).max(100_000).optional(),
|
||||
y: z.number().int().min(0).max(100_000).optional(),
|
||||
});
|
||||
export type ComputerPointerStep = z.infer<typeof ComputerPointerStep>;
|
||||
|
||||
export const MAX_COMPUTER_POINTER_STEPS = 48;
|
||||
|
||||
/**
|
||||
* Payload of a `thread.computer` event with `kind: "pointer"`: where a bot is about to point,
|
||||
* click or type on its computer screen, so clients can draw its cursor over the live screen.
|
||||
*/
|
||||
export const ComputerPointerPayload = z.object({
|
||||
kind: z.literal("pointer"),
|
||||
screen: z.object({
|
||||
width: z.number().int().positive().max(100_000),
|
||||
height: z.number().int().positive().max(100_000),
|
||||
}),
|
||||
steps: z.array(ComputerPointerStep).min(1).max(MAX_COMPUTER_POINTER_STEPS),
|
||||
});
|
||||
export type ComputerPointerPayload = z.infer<typeof ComputerPointerPayload>;
|
||||
@ -38,20 +38,39 @@ export interface RakazoDesktopOAuthCallback {
|
||||
|
||||
export interface RakazoDesktopBrowserPreviewState {
|
||||
url: string;
|
||||
devServerRunning: boolean;
|
||||
title: string;
|
||||
loading: boolean;
|
||||
canGoBack: boolean;
|
||||
canGoForward: boolean;
|
||||
/** The last main-frame load failure for `url`, cleared by the next successful load. */
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** Optional desktop bridge for the in-app browser preview panel. Absent in the browser. */
|
||||
/** Rectangle in CSS pixels of the app window's content area. */
|
||||
export interface RakazoDesktopBrowserBounds {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional desktop bridge for the in-app browser: a native Chromium view the main process lays
|
||||
* over the panel the renderer reserves for it. Absent in the browser, where an iframe is used.
|
||||
*/
|
||||
export interface RakazoDesktopBrowserPreview {
|
||||
getState: () => Promise<RakazoDesktopBrowserPreviewState>;
|
||||
/** Shows the view over these bounds; call again whenever the reserved area moves. */
|
||||
show: (bounds: RakazoDesktopBrowserBounds) => Promise<void>;
|
||||
hide: () => Promise<void>;
|
||||
/** Only http(s) URLs load; anything else is rejected. */
|
||||
navigate: (url: string) => Promise<void>;
|
||||
back: () => Promise<void>;
|
||||
forward: () => Promise<void>;
|
||||
reload: () => Promise<void>;
|
||||
onNavigate: (listener: (url: string) => void) => () => void;
|
||||
onLoadingProgress: (listener: (progress: number) => void) => () => void;
|
||||
devServer: {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
};
|
||||
stop: () => Promise<void>;
|
||||
openDevTools: () => Promise<void>;
|
||||
state: () => Promise<RakazoDesktopBrowserPreviewState>;
|
||||
onState: (listener: (state: RakazoDesktopBrowserPreviewState) => void) => () => void;
|
||||
}
|
||||
|
||||
export interface RakazoDesktop {
|
||||
|
||||
@ -3,6 +3,7 @@ export * from "./attachments.js";
|
||||
export * from "./bot-secrets.js";
|
||||
export * from "./clock-time.js";
|
||||
export * from "./coach.js";
|
||||
export * from "./computer-pointer.js";
|
||||
export * from "./desktop.js";
|
||||
export * from "./domain.js";
|
||||
export * from "./events.js";
|
||||
|
||||
@ -3,6 +3,7 @@ import {
|
||||
computerInputForDomKey,
|
||||
computerScreenSize,
|
||||
mapTeachPointer,
|
||||
screenPointInViewer,
|
||||
teachCaptureKey,
|
||||
} from "./teach-recording.js";
|
||||
|
||||
@ -88,3 +89,21 @@ describe("mapTeachPointer", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("screenPointInViewer", () => {
|
||||
it("projects through the same letterbox the viewer draws", () => {
|
||||
const screen = { width: 1280, height: 800 };
|
||||
// A wide viewer pillarboxes the 16:10 screen.
|
||||
expect(screenPointInViewer({ x: 0, y: 0 }, { width: 1000, height: 400 }, screen)).toEqual({
|
||||
x: 180,
|
||||
y: 0,
|
||||
});
|
||||
expect(screenPointInViewer({ x: 1280, y: 800 }, { width: 1000, height: 400 }, screen)).toEqual({
|
||||
x: 820,
|
||||
y: 400,
|
||||
});
|
||||
const rect = { left: 0, top: 0, width: 640, height: 600 };
|
||||
const projected = screenPointInViewer({ x: 640, y: 400 }, rect, screen);
|
||||
expect(mapTeachPointer(projected.x, projected.y, rect, screen)).toEqual({ x: 640, y: 400 });
|
||||
});
|
||||
});
|
||||
|
||||
@ -91,3 +91,18 @@ export function mapTeachPointer(
|
||||
y: clamp(Math.round((clientY - rect.top - offsetY) / scale), screen.height - 1),
|
||||
};
|
||||
}
|
||||
|
||||
/** Inverse of {@link mapTeachPointer}: where a screen pixel lands inside the letterboxed viewer. */
|
||||
export function screenPointInViewer(
|
||||
point: { x: number; y: number },
|
||||
viewer: { width: number; height: number },
|
||||
screen: { width: number; height: number },
|
||||
): { x: number; y: number } {
|
||||
const scale = Math.min(viewer.width / screen.width, viewer.height / screen.height) || 1;
|
||||
const offsetX = (viewer.width - screen.width * scale) / 2;
|
||||
const offsetY = (viewer.height - screen.height * scale) / 2;
|
||||
return {
|
||||
x: offsetX + clamp(point.x, screen.width) * scale,
|
||||
y: offsetY + clamp(point.y, screen.height) * scale,
|
||||
};
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user