From f07d17e927b4cdfd691c58ef89773abb8862772c Mon Sep 17 00:00:00 2001 From: cynarAI <155274210+cynarAI@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:10:25 +0200 Subject: [PATCH] Oo computer browser: live Chromium, cowork, AgentCursor (#179) * feat(web): Oo computer browser panel with cowork and local preview Wire the in-app browser panel to the active Oo's live Chromium via the computer screen stream, page-browser navigation, and interactive cowork input with AgentCursor. User browser actions become system meta lines in thread context; localhost preview remains as Lokale Vorschau. Co-authored-by: cynarAI * fix(i18n): sync Lingui catalogs for Oo browser panel Co-authored-by: cynarAI * Fix the Oo browser's lint and E2E failures The shared-browser overlay is a focusable input surface, so it gets the application role and a translated label. The local preview focuses the address field again when no page is open, which the split out of InAppBrowser had dropped. The Oo browser spec opens the panel by its exact name, because the signed-up user's name also contains "Browser", and reads the user action from the RPC body's json envelope. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs --------- Co-authored-by: Cursor Agent Co-authored-by: cynarAI Co-authored-by: Claude --- apps/api/src/router.ts | 123 ++++++ apps/web/e2e/in-app-browser.spec.ts | 3 + apps/web/e2e/oo-computer-browser.spec.ts | 88 +++++ .../browser/BrowserCoworkOverlay.tsx | 181 +++++++++ .../src/components/browser/InAppBrowser.tsx | 360 ++++-------------- .../browser/LocalBrowserPreview.tsx | 280 ++++++++++++++ .../components/browser/OoComputerBrowser.tsx | 258 +++++++++++++ apps/web/src/locales/de/messages.po | 56 ++- apps/web/src/locales/en/messages.po | 56 ++- apps/web/src/locales/es/messages.po | 56 ++- apps/web/src/locales/hi/messages.po | 56 ++- apps/web/src/locales/ko/messages.po | 56 ++- apps/web/src/locales/pt-BR/messages.po | 56 ++- apps/web/src/locales/ru/messages.po | 56 ++- apps/web/src/locales/tr/messages.po | 56 ++- apps/web/src/locales/zh-CN/messages.po | 56 ++- apps/web/src/pages/Shell.tsx | 22 +- infra/sandboxes/computer/rakazo-page-browser | 67 ++++ infra/sandboxes/supervisor/src/index.ts | 4 + packages/adapter-kit/src/types.ts | 24 +- packages/adapters/src/browser-cowork.ts | 225 +++++++++++ packages/adapters/src/computer-browser.ts | 56 ++- packages/adapters/src/index.ts | 1 + packages/adapters/src/page-browser-session.ts | 49 +++ packages/contracts/src/rpc.ts | 62 +++ packages/core/src/browser-cowork.test.ts | 16 + packages/core/src/browser-cowork.ts | 35 ++ packages/core/src/index.ts | 1 + packages/testkit/src/authorization.test.ts | 17 + 29 files changed, 2007 insertions(+), 369 deletions(-) create mode 100644 apps/web/e2e/oo-computer-browser.spec.ts create mode 100644 apps/web/src/components/browser/BrowserCoworkOverlay.tsx create mode 100644 apps/web/src/components/browser/LocalBrowserPreview.tsx create mode 100644 apps/web/src/components/browser/OoComputerBrowser.tsx create mode 100644 packages/adapters/src/browser-cowork.ts create mode 100644 packages/core/src/browser-cowork.test.ts create mode 100644 packages/core/src/browser-cowork.ts diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 9165644..960972f 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -45,10 +45,12 @@ import { clearInactiveUserComputerControl, computerSupportsUpdate, computerUpdateView, + createBrowserProvider, createVoiceProvider, deploymentAutoReviewDefault, destroyBot, displayBotWorkspacePath, + engageBrowserCowork, enqueueTakeoverContinuation, expireComputerControl, filterTeamWorkspaceEntries, @@ -73,6 +75,7 @@ import { queueComputerUpdate, RECOMMENDED_MODEL, readProjectWorkspaceFile, + recordBrowserUserAction, releaseComputerExecutionLease, replaceComputer, resolveAutoReviewChecker, @@ -2682,6 +2685,77 @@ export function createRouter(deps: RouterDeps) { } return { ok: true as const }; }), + browserChromeState: authed.computer.browserChromeState.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + return readBrowserChrome(deps, bot, "state", context); + }), + browserNavigate: authed.computer.browserNavigate.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + const browser = createBrowserProvider(undefined, { sandbox: deps.sandbox }); + const result = await browser.navigate( + toComputerRef(bot.computer), + { url: input.url }, + computerContext(context.actor, bot.id, "browser.navigate"), + ); + if (result.fallback) { + throw new ORPCError("CONFLICT", { message: result.error ?? "Browser unavailable" }); + } + await noteBrowserNavigate(deps, context.actor, bot, input.url); + return readBrowserChrome(deps, bot, "state", context); + }), + browserBack: authed.computer.browserBack.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + return readBrowserChrome(deps, bot, "back", context); + }), + browserForward: authed.computer.browserForward.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + return readBrowserChrome(deps, bot, "forward", context); + }), + browserReload: authed.computer.browserReload.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + return readBrowserChrome(deps, bot, "reload", context); + }), + browserCoworkEngage: authed.computer.browserCoworkEngage.handler( + async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer) throw new IsolationError(); + const engaged = await engageBrowserCowork( + deps, + context.actor, + bot, + computerContext(context.actor, bot.id, "browser.cowork"), + ); + return engaged; + }, + ), + browserUserAction: authed.computer.browserUserAction.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + const threadId = bot.thread?.id; + if (!threadId) throw new IsolationError(); + await recordBrowserUserAction(deps, context.actor, { + botId: bot.id, + threadId, + kind: input.kind, + detail: input.detail, + passwordField: input.passwordField, + }); + return { ok: true as const }; + }), }, memory: { list: authed.memory.list.handler(async ({ context, input }) => { @@ -5842,6 +5916,55 @@ async function persistModelCredential( return modelCredentialDto(cred, input.plaintext); } +async function readBrowserChrome( + deps: RouterDeps, + bot: { + id: string; + computer: { homeKey: string; kind: string; providerRef: string | null } | null; + }, + command: "state" | "back" | "forward" | "reload", + context: { actor: Actor }, +) { + if (!bot.computer?.providerRef) { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + const browser = createBrowserProvider(undefined, { sandbox: deps.sandbox }); + const chrome = browser as InstanceType; + const result = await chrome.chrome( + toComputerRef(bot.computer), + command, + computerContext(context.actor, bot.id, `browser.${command}`), + ); + if (!result.ok) { + throw new ORPCError("CONFLICT", { + message: "error" in result && result.error ? result.error : "Browser unavailable", + }); + } + return { + url: result.url, + title: result.title, + loading: result.loading, + canGoBack: result.canGoBack, + canGoForward: result.canGoForward, + }; +} + +async function noteBrowserNavigate( + deps: RouterDeps, + actor: Actor, + bot: { id: string; thread: { id: string } | null }, + url: string, +) { + const threadId = bot.thread?.id; + if (!threadId) return; + await recordBrowserUserAction(deps, actor, { + botId: bot.id, + threadId, + kind: "navigate", + detail: url, + }); +} + function throwIfAborted(signal?: AbortSignal) { if (signal?.aborted) throw signal.reason ?? new Error("Request cancelled"); } diff --git a/apps/web/e2e/in-app-browser.spec.ts b/apps/web/e2e/in-app-browser.spec.ts index 1862edb..e8f4c98 100644 --- a/apps/web/e2e/in-app-browser.spec.ts +++ b/apps/web/e2e/in-app-browser.spec.ts @@ -17,6 +17,7 @@ test("the in-app browser previews a page and follows it back to the browser", as await page.getByRole("button", { name: "Browser" }).click(); const browser = page.getByTestId("in-app-browser"); + await browser.getByTestId("in-app-browser-mode-local").click(); const address = browser.getByRole("textbox", { name: "Address" }); await expect(address).toBeFocused(); @@ -42,6 +43,7 @@ test("the in-app browser previews a page and follows it back to the browser", as await expect(browser).toHaveCount(0); // The last address comes back when the browser reopens. await page.getByRole("button", { name: "Browser" }).click(); + await page.getByTestId("in-app-browser-mode-local").click(); await expect(page.getByTestId("in-app-browser-frame")).toHaveAttribute( "src", "https://preview.example/app", @@ -113,6 +115,7 @@ test("on desktop the page is a native view glued to the panel", async ({ page }) await page.getByRole("button", { name: "Browser" }).click(); const browser = page.getByTestId("in-app-browser"); + await browser.getByTestId("in-app-browser-mode-local").click(); await browser.getByRole("textbox", { name: "Address" }).fill("localhost:5173"); await browser.getByRole("textbox", { name: "Address" }).press("Enter"); await expect.poll(calls).toContainEqual(["navigate", "http://localhost:5173/"]); diff --git a/apps/web/e2e/oo-computer-browser.spec.ts b/apps/web/e2e/oo-computer-browser.spec.ts new file mode 100644 index 0000000..949b7c4 --- /dev/null +++ b/apps/web/e2e/oo-computer-browser.spec.ts @@ -0,0 +1,88 @@ +import { expect, test } from "@playwright/test"; +import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers"; + +const SCREEN_URL = "https://screen.example/embed.html"; + +test("Oo browser shows live screen, cursor, cowork click, and user context", async ({ + page, +}, testInfo) => { + await signup(page, `oo-browser-${Date.now()}@rakazo.test`, "password12", "Browser Cowork"); + await completeOnboarding(page); + const botId = activeBotId(page); + await rpc(page, "computer/boot", { botId }); + + await page.route("https://screen.example/**", (route) => + route.fulfill({ + contentType: "text/html", + body: `Chrome`, + }), + ); + await page.route("**/rpc/computer/screenUrl", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { url: SCREEN_URL } }), + }), + ); + + let coworkInput = 0; + await page.route("**/rpc/computer/browserCoworkEngage", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ + json: { + leaseId: "lease-test", + expiresAt: new Date(Date.now() + 60_000).toISOString(), + interrupted: true, + }, + }), + }), + ); + await page.route("**/rpc/computer/input", (route) => { + coworkInput += 1; + return route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { ok: true } }), + }); + }); + await page.route("**/rpc/computer/browserChromeState", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ + json: { + url: "https://example.com/", + title: "Example", + loading: false, + canGoBack: false, + canGoForward: false, + }, + }), + }), + ); + + let userActionDetail = ""; + await page.route("**/rpc/computer/browserUserAction", (route) => { + const body = route.request().postDataJSON() as { json?: { detail?: string } }; + userActionDetail = body.json?.detail ?? ""; + return route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { ok: true } }), + }); + }); + + await page.getByRole("button", { name: "Browser", exact: true }).click(); + const browser = page.getByTestId("in-app-browser"); + await expect(browser.getByTestId("oo-computer-browser-screen")).toHaveAttribute( + "src", + SCREEN_URL, + ); + + await rpc(page, "threads/send", { botId, text: "use your screen and click center" }); + const cursor = browser.getByTestId("agent-cursor"); + await expect(cursor).toHaveCSS("opacity", "1", { timeout: 15_000 }); + + await browser.getByTestId("browser-cowork-overlay").click({ position: { x: 120, y: 80 } }); + await expect.poll(() => coworkInput).toBeGreaterThan(0); + await expect.poll(() => userActionDetail).toContain("("); + + await captureScreenshot(page, testInfo, "oo-computer-browser"); +}); diff --git a/apps/web/src/components/browser/BrowserCoworkOverlay.tsx b/apps/web/src/components/browser/BrowserCoworkOverlay.tsx new file mode 100644 index 0000000..b86a6a7 --- /dev/null +++ b/apps/web/src/components/browser/BrowserCoworkOverlay.tsx @@ -0,0 +1,181 @@ +import { useLingui } from "@lingui/react/macro"; +import type { ComputerStatus } from "@rakazo/contracts"; +import { DEFAULT_COMPUTER_SCREEN, mapTeachPointer } from "@rakazo/core"; +import { useEffect, useRef } from "react"; +import { rpc } from "../../lib/rpc"; + +/** Forwards pointer, scroll, and key input to the Oo's live computer after cowork engage. */ +export function BrowserCoworkOverlay({ + botId, + computer, + enabled, + onEngaged, + onUserAction, +}: { + botId: string; + computer: ComputerStatus | null; + enabled: boolean; + onEngaged: () => void; + onUserAction: (detail: { kind: "click" | "scroll" | "type"; detail: string }) => void; +}) { + const { t } = useLingui(); + const rootRef = useRef(null); + const engagedRef = useRef(false); + const inputChainRef = useRef(Promise.resolve()); + const width = computer?.screenWidth ?? DEFAULT_COMPUTER_SCREEN.width; + const height = computer?.screenHeight ?? DEFAULT_COMPUTER_SCREEN.height; + + useEffect(() => { + engagedRef.current = false; + }, [botId]); + + useEffect(() => { + if (!enabled) return; + const overlay = rootRef.current; + if (!overlay) return; + + function enqueueInput(task: () => Promise) { + inputChainRef.current = inputChainRef.current.then(task).catch(() => undefined); + } + + async function ensureEngaged() { + if (engagedRef.current) return; + await rpc.computer.browserCoworkEngage({ botId }); + engagedRef.current = true; + onEngaged(); + } + + function pointerAt(event: PointerEvent) { + const box = overlay!.getBoundingClientRect(); + return mapTeachPointer(event.clientX, event.clientY, box, { + width, + height, + }); + } + + async function sendPointer( + type: "move" | "down" | "up" | "click", + x: number, + y: number, + button: "left" | "right" = "left", + ) { + await rpc.computer.input({ + botId, + kind: "pointer", + payload: { x, y, button, type }, + }); + } + + async function sendKey(key: string) { + await rpc.computer.input({ botId, kind: "key", payload: { key } }); + } + + async function sendScroll(direction: "up" | "down", amount: number) { + await rpc.computer.input({ + botId, + kind: "scroll", + payload: { direction, amount }, + }); + } + + function buttonFor(event: PointerEvent): "left" | "right" { + return event.button === 2 ? "right" : "left"; + } + + function onPointerDown(event: PointerEvent) { + event.preventDefault(); + overlay!.setPointerCapture(event.pointerId); + const { x, y } = pointerAt(event); + enqueueInput(async () => { + await ensureEngaged(); + await sendPointer("down", x, y, buttonFor(event)); + onUserAction({ kind: "click", detail: `(${x}, ${y})` }); + }); + } + + let pendingMove: { x: number; y: number; button: "left" | "right" } | null = null; + let moveInFlight = false; + + function pumpMove() { + if (moveInFlight || !pendingMove) return; + const move = pendingMove; + pendingMove = null; + moveInFlight = true; + enqueueInput(async () => { + try { + await ensureEngaged(); + await sendPointer("move", move.x, move.y, move.button); + } finally { + moveInFlight = false; + pumpMove(); + } + }); + } + + function onPointerMove(event: PointerEvent) { + if (!event.buttons) return; + event.preventDefault(); + pendingMove = { ...pointerAt(event), button: buttonFor(event) }; + pumpMove(); + } + + function onPointerUp(event: PointerEvent) { + event.preventDefault(); + const { x, y } = pointerAt(event); + enqueueInput(async () => { + await ensureEngaged(); + await sendPointer("up", x, y, buttonFor(event)); + await sendPointer("click", x, y, buttonFor(event)); + }); + } + + function onWheel(event: WheelEvent) { + event.preventDefault(); + const direction = event.deltaY < 0 ? "up" : "down"; + const amount = Math.min(12, Math.max(1, Math.round(Math.abs(event.deltaY) / 40))); + enqueueInput(async () => { + await ensureEngaged(); + await sendScroll(direction, amount); + onUserAction({ kind: "scroll", detail: direction }); + }); + } + + function onKeyDown(event: KeyboardEvent) { + if (event.metaKey || event.ctrlKey || event.altKey) return; + event.preventDefault(); + enqueueInput(async () => { + await ensureEngaged(); + await sendKey(event.key.length === 1 ? event.key : event.code); + onUserAction({ + kind: "type", + detail: event.key.length === 1 ? event.key : event.code, + }); + }); + } + + overlay.addEventListener("pointerdown", onPointerDown); + overlay.addEventListener("pointermove", onPointerMove); + overlay.addEventListener("pointerup", onPointerUp); + overlay.addEventListener("wheel", onWheel, { passive: false }); + overlay.addEventListener("keydown", onKeyDown); + overlay.tabIndex = 0; + return () => { + overlay.removeEventListener("pointerdown", onPointerDown); + overlay.removeEventListener("pointermove", onPointerMove); + overlay.removeEventListener("pointerup", onPointerUp); + overlay.removeEventListener("wheel", onWheel); + overlay.removeEventListener("keydown", onKeyDown); + }; + }, [botId, enabled, height, onEngaged, onUserAction, width]); + + if (!enabled) return null; + return ( +
+ ); +} diff --git a/apps/web/src/components/browser/InAppBrowser.tsx b/apps/web/src/components/browser/InAppBrowser.tsx index 22ac61a..f50b2bd 100644 --- a/apps/web/src/components/browser/InAppBrowser.tsx +++ b/apps/web/src/components/browser/InAppBrowser.tsx @@ -1,214 +1,63 @@ -import { useLingui } from "@lingui/react/macro"; -import type { - RakazoDesktopBrowserPreview, - RakazoDesktopBrowserPreviewState, -} from "@rakazo/contracts"; -import { Button, cn, Input } from "@rakazo/ui-web"; -import { ArrowLeft, ArrowRight, Code, ExternalLink, RotateCw, X } from "lucide-react"; -import { type FormEvent, useCallback, useEffect, useRef, useState } from "react"; -import { browserFrameSandbox, normalizeBrowserUrl } from "../../lib/browser-url"; -import { desktopBridge } from "../../lib/desktop"; +import { Trans, useLingui } from "@lingui/react/macro"; +import type { Bot, ComputerStatus } from "@rakazo/contracts"; +import { Button, cn } from "@rakazo/ui-web"; +import { X } from "lucide-react"; +import { useState } from "react"; +import { LocalBrowserPreview } from "./LocalBrowserPreview"; +import { OoComputerBrowser } from "./OoComputerBrowser"; -const LAST_URL_KEY = "rk:in-app-browser:url"; +export type InAppBrowserMode = "oo" | "local"; -function readLastUrl(): string { - try { - return window.localStorage.getItem(LAST_URL_KEY) ?? ""; - } catch { - return ""; - } -} - -function rememberUrl(url: string) { - try { - window.localStorage.setItem(LAST_URL_KEY, url); - } catch { - /* A per-viewer convenience only. */ - } -} - -const EMPTY_STATE: RakazoDesktopBrowserPreviewState = { - url: "", - title: "", - loading: false, - canGoBack: false, - canGoForward: false, - error: null, -}; - -/** DOM layers the native desktop view would otherwise paint over. */ -const FLOATING_UI = '[role="dialog"],[role="alertdialog"],[role="menu"],[role="listbox"]'; - -/** - * Address bar plus page. On desktop the page is a native Chromium view laid over the reserved - * area, so dev servers keep hot reload, DevTools and full speed; in a browser it is an iframe. - */ -export function InAppBrowser({ onClose }: { onClose: () => void }) { +export function InAppBrowser({ + onClose, + active, + computer, + embeddedScreenUrl, + computerScreenError, + sandboxProvider, + booting, + onBoot, + onRefreshScreen, +}: { + onClose: () => void; + active: Bot | null; + computer: ComputerStatus | null; + embeddedScreenUrl: string | null; + computerScreenError: React.ReactNode; + sandboxProvider: string | undefined; + booting: boolean; + onBoot: () => Promise; + onRefreshScreen: () => Promise; +}) { const { t } = useLingui(); - const native = desktopBridge()?.browserPreview; - const [state, setState] = useState(() => ({ - ...EMPTY_STATE, - url: native ? "" : readLastUrl(), - })); - const [draft, setDraft] = useState(state.url); - const [invalid, setInvalid] = useState(false); - const [frameKey, setFrameKey] = useState(0); - const editing = useRef(false); - const inputRef = useRef(null); - - useEffect(() => { - if (!native) return; - let live = true; - const apply = (next: RakazoDesktopBrowserPreviewState) => { - if (!live) return; - setState(next); - if (!editing.current) setDraft(next.url); - }; - const unsubscribe = native.onState(apply); - void native - .state() - .then((current) => { - apply(current); - const last = readLastUrl(); - if (!current.url && last) void native.navigate(last).catch(() => undefined); - }) - .catch(() => undefined); - return () => { - live = false; - unsubscribe(); - }; - }, [native]); - - useEffect(() => { - if (!state.url) inputRef.current?.focus(); - }, [state.url]); - - const go = useCallback( - (input: string) => { - const url = normalizeBrowserUrl(input); - setInvalid(!url); - if (!url) return; - editing.current = false; - setDraft(url); - rememberUrl(url); - if (native) { - void native.navigate(url).catch(() => setInvalid(true)); - } else { - setState((current) => ({ ...current, url, loading: true, error: null })); - setFrameKey((key) => key + 1); - } - }, - [native], - ); - - const reload = useCallback(() => { - if (native) { - void (state.loading ? native.stop() : native.reload()).catch(() => undefined); - return; - } - if (!state.url) return; - setState((current) => ({ ...current, loading: true })); - setFrameKey((key) => key + 1); - }, [native, state.loading, state.url]); - - const onSubmit = (event: FormEvent) => { - event.preventDefault(); - go(draft); - inputRef.current?.blur(); - }; + const [mode, setMode] = useState("oo"); return (
-
- {native ? ( - <> - - - - ) : null} - - { - editing.current = true; - event.currentTarget.select(); - }} - onBlur={() => { - editing.current = false; - }} - onChange={(event) => { - setDraft(event.target.value); - setInvalid(false); - }} - onKeyDown={(event) => { - if (event.key !== "Escape") return; - setDraft(state.url); - setInvalid(false); - event.currentTarget.blur(); - }} - className="mx-1 h-control-sm flex-1 rounded-full bg-muted/60 px-3 text-ui-body-sm" - /> - {native ? ( +
+
- ) : null} - + +
-
-
-
- -
- {native ? ( - - ) : state.url ? ( -