diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 9165644..960972f 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -45,10 +45,12 @@ import { clearInactiveUserComputerControl, computerSupportsUpdate, computerUpdateView, + createBrowserProvider, createVoiceProvider, deploymentAutoReviewDefault, destroyBot, displayBotWorkspacePath, + engageBrowserCowork, enqueueTakeoverContinuation, expireComputerControl, filterTeamWorkspaceEntries, @@ -73,6 +75,7 @@ import { queueComputerUpdate, RECOMMENDED_MODEL, readProjectWorkspaceFile, + recordBrowserUserAction, releaseComputerExecutionLease, replaceComputer, resolveAutoReviewChecker, @@ -2682,6 +2685,77 @@ export function createRouter(deps: RouterDeps) { } return { ok: true as const }; }), + browserChromeState: authed.computer.browserChromeState.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + return readBrowserChrome(deps, bot, "state", context); + }), + browserNavigate: authed.computer.browserNavigate.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + const browser = createBrowserProvider(undefined, { sandbox: deps.sandbox }); + const result = await browser.navigate( + toComputerRef(bot.computer), + { url: input.url }, + computerContext(context.actor, bot.id, "browser.navigate"), + ); + if (result.fallback) { + throw new ORPCError("CONFLICT", { message: result.error ?? "Browser unavailable" }); + } + await noteBrowserNavigate(deps, context.actor, bot, input.url); + return readBrowserChrome(deps, bot, "state", context); + }), + browserBack: authed.computer.browserBack.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + return readBrowserChrome(deps, bot, "back", context); + }), + browserForward: authed.computer.browserForward.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + return readBrowserChrome(deps, bot, "forward", context); + }), + browserReload: authed.computer.browserReload.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer?.providerRef || bot.computer.state !== "running") { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + return readBrowserChrome(deps, bot, "reload", context); + }), + browserCoworkEngage: authed.computer.browserCoworkEngage.handler( + async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + if (!bot.computer) throw new IsolationError(); + const engaged = await engageBrowserCowork( + deps, + context.actor, + bot, + computerContext(context.actor, bot.id, "browser.cowork"), + ); + return engaged; + }, + ), + browserUserAction: authed.computer.browserUserAction.handler(async ({ context, input }) => { + const bot = await repos.getBot(context.actor, input.botId); + const threadId = bot.thread?.id; + if (!threadId) throw new IsolationError(); + await recordBrowserUserAction(deps, context.actor, { + botId: bot.id, + threadId, + kind: input.kind, + detail: input.detail, + passwordField: input.passwordField, + }); + return { ok: true as const }; + }), }, memory: { list: authed.memory.list.handler(async ({ context, input }) => { @@ -5842,6 +5916,55 @@ async function persistModelCredential( return modelCredentialDto(cred, input.plaintext); } +async function readBrowserChrome( + deps: RouterDeps, + bot: { + id: string; + computer: { homeKey: string; kind: string; providerRef: string | null } | null; + }, + command: "state" | "back" | "forward" | "reload", + context: { actor: Actor }, +) { + if (!bot.computer?.providerRef) { + throw new ORPCError("BAD_REQUEST", { message: "computer must be running" }); + } + const browser = createBrowserProvider(undefined, { sandbox: deps.sandbox }); + const chrome = browser as InstanceType; + const result = await chrome.chrome( + toComputerRef(bot.computer), + command, + computerContext(context.actor, bot.id, `browser.${command}`), + ); + if (!result.ok) { + throw new ORPCError("CONFLICT", { + message: "error" in result && result.error ? result.error : "Browser unavailable", + }); + } + return { + url: result.url, + title: result.title, + loading: result.loading, + canGoBack: result.canGoBack, + canGoForward: result.canGoForward, + }; +} + +async function noteBrowserNavigate( + deps: RouterDeps, + actor: Actor, + bot: { id: string; thread: { id: string } | null }, + url: string, +) { + const threadId = bot.thread?.id; + if (!threadId) return; + await recordBrowserUserAction(deps, actor, { + botId: bot.id, + threadId, + kind: "navigate", + detail: url, + }); +} + function throwIfAborted(signal?: AbortSignal) { if (signal?.aborted) throw signal.reason ?? new Error("Request cancelled"); } diff --git a/apps/web/e2e/in-app-browser.spec.ts b/apps/web/e2e/in-app-browser.spec.ts index 1862edb..e8f4c98 100644 --- a/apps/web/e2e/in-app-browser.spec.ts +++ b/apps/web/e2e/in-app-browser.spec.ts @@ -17,6 +17,7 @@ test("the in-app browser previews a page and follows it back to the browser", as await page.getByRole("button", { name: "Browser" }).click(); const browser = page.getByTestId("in-app-browser"); + await browser.getByTestId("in-app-browser-mode-local").click(); const address = browser.getByRole("textbox", { name: "Address" }); await expect(address).toBeFocused(); @@ -42,6 +43,7 @@ test("the in-app browser previews a page and follows it back to the browser", as await expect(browser).toHaveCount(0); // The last address comes back when the browser reopens. await page.getByRole("button", { name: "Browser" }).click(); + await page.getByTestId("in-app-browser-mode-local").click(); await expect(page.getByTestId("in-app-browser-frame")).toHaveAttribute( "src", "https://preview.example/app", @@ -113,6 +115,7 @@ test("on desktop the page is a native view glued to the panel", async ({ page }) await page.getByRole("button", { name: "Browser" }).click(); const browser = page.getByTestId("in-app-browser"); + await browser.getByTestId("in-app-browser-mode-local").click(); await browser.getByRole("textbox", { name: "Address" }).fill("localhost:5173"); await browser.getByRole("textbox", { name: "Address" }).press("Enter"); await expect.poll(calls).toContainEqual(["navigate", "http://localhost:5173/"]); diff --git a/apps/web/e2e/oo-computer-browser.spec.ts b/apps/web/e2e/oo-computer-browser.spec.ts new file mode 100644 index 0000000..949b7c4 --- /dev/null +++ b/apps/web/e2e/oo-computer-browser.spec.ts @@ -0,0 +1,88 @@ +import { expect, test } from "@playwright/test"; +import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers"; + +const SCREEN_URL = "https://screen.example/embed.html"; + +test("Oo browser shows live screen, cursor, cowork click, and user context", async ({ + page, +}, testInfo) => { + await signup(page, `oo-browser-${Date.now()}@rakazo.test`, "password12", "Browser Cowork"); + await completeOnboarding(page); + const botId = activeBotId(page); + await rpc(page, "computer/boot", { botId }); + + await page.route("https://screen.example/**", (route) => + route.fulfill({ + contentType: "text/html", + body: `Chrome`, + }), + ); + await page.route("**/rpc/computer/screenUrl", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { url: SCREEN_URL } }), + }), + ); + + let coworkInput = 0; + await page.route("**/rpc/computer/browserCoworkEngage", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ + json: { + leaseId: "lease-test", + expiresAt: new Date(Date.now() + 60_000).toISOString(), + interrupted: true, + }, + }), + }), + ); + await page.route("**/rpc/computer/input", (route) => { + coworkInput += 1; + return route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { ok: true } }), + }); + }); + await page.route("**/rpc/computer/browserChromeState", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ + json: { + url: "https://example.com/", + title: "Example", + loading: false, + canGoBack: false, + canGoForward: false, + }, + }), + }), + ); + + let userActionDetail = ""; + await page.route("**/rpc/computer/browserUserAction", (route) => { + const body = route.request().postDataJSON() as { json?: { detail?: string } }; + userActionDetail = body.json?.detail ?? ""; + return route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { ok: true } }), + }); + }); + + await page.getByRole("button", { name: "Browser", exact: true }).click(); + const browser = page.getByTestId("in-app-browser"); + await expect(browser.getByTestId("oo-computer-browser-screen")).toHaveAttribute( + "src", + SCREEN_URL, + ); + + await rpc(page, "threads/send", { botId, text: "use your screen and click center" }); + const cursor = browser.getByTestId("agent-cursor"); + await expect(cursor).toHaveCSS("opacity", "1", { timeout: 15_000 }); + + await browser.getByTestId("browser-cowork-overlay").click({ position: { x: 120, y: 80 } }); + await expect.poll(() => coworkInput).toBeGreaterThan(0); + await expect.poll(() => userActionDetail).toContain("("); + + await captureScreenshot(page, testInfo, "oo-computer-browser"); +}); diff --git a/apps/web/src/components/browser/BrowserCoworkOverlay.tsx b/apps/web/src/components/browser/BrowserCoworkOverlay.tsx new file mode 100644 index 0000000..b86a6a7 --- /dev/null +++ b/apps/web/src/components/browser/BrowserCoworkOverlay.tsx @@ -0,0 +1,181 @@ +import { useLingui } from "@lingui/react/macro"; +import type { ComputerStatus } from "@rakazo/contracts"; +import { DEFAULT_COMPUTER_SCREEN, mapTeachPointer } from "@rakazo/core"; +import { useEffect, useRef } from "react"; +import { rpc } from "../../lib/rpc"; + +/** Forwards pointer, scroll, and key input to the Oo's live computer after cowork engage. */ +export function BrowserCoworkOverlay({ + botId, + computer, + enabled, + onEngaged, + onUserAction, +}: { + botId: string; + computer: ComputerStatus | null; + enabled: boolean; + onEngaged: () => void; + onUserAction: (detail: { kind: "click" | "scroll" | "type"; detail: string }) => void; +}) { + const { t } = useLingui(); + const rootRef = useRef(null); + const engagedRef = useRef(false); + const inputChainRef = useRef(Promise.resolve()); + const width = computer?.screenWidth ?? DEFAULT_COMPUTER_SCREEN.width; + const height = computer?.screenHeight ?? DEFAULT_COMPUTER_SCREEN.height; + + useEffect(() => { + engagedRef.current = false; + }, [botId]); + + useEffect(() => { + if (!enabled) return; + const overlay = rootRef.current; + if (!overlay) return; + + function enqueueInput(task: () => Promise) { + inputChainRef.current = inputChainRef.current.then(task).catch(() => undefined); + } + + async function ensureEngaged() { + if (engagedRef.current) return; + await rpc.computer.browserCoworkEngage({ botId }); + engagedRef.current = true; + onEngaged(); + } + + function pointerAt(event: PointerEvent) { + const box = overlay!.getBoundingClientRect(); + return mapTeachPointer(event.clientX, event.clientY, box, { + width, + height, + }); + } + + async function sendPointer( + type: "move" | "down" | "up" | "click", + x: number, + y: number, + button: "left" | "right" = "left", + ) { + await rpc.computer.input({ + botId, + kind: "pointer", + payload: { x, y, button, type }, + }); + } + + async function sendKey(key: string) { + await rpc.computer.input({ botId, kind: "key", payload: { key } }); + } + + async function sendScroll(direction: "up" | "down", amount: number) { + await rpc.computer.input({ + botId, + kind: "scroll", + payload: { direction, amount }, + }); + } + + function buttonFor(event: PointerEvent): "left" | "right" { + return event.button === 2 ? "right" : "left"; + } + + function onPointerDown(event: PointerEvent) { + event.preventDefault(); + overlay!.setPointerCapture(event.pointerId); + const { x, y } = pointerAt(event); + enqueueInput(async () => { + await ensureEngaged(); + await sendPointer("down", x, y, buttonFor(event)); + onUserAction({ kind: "click", detail: `(${x}, ${y})` }); + }); + } + + let pendingMove: { x: number; y: number; button: "left" | "right" } | null = null; + let moveInFlight = false; + + function pumpMove() { + if (moveInFlight || !pendingMove) return; + const move = pendingMove; + pendingMove = null; + moveInFlight = true; + enqueueInput(async () => { + try { + await ensureEngaged(); + await sendPointer("move", move.x, move.y, move.button); + } finally { + moveInFlight = false; + pumpMove(); + } + }); + } + + function onPointerMove(event: PointerEvent) { + if (!event.buttons) return; + event.preventDefault(); + pendingMove = { ...pointerAt(event), button: buttonFor(event) }; + pumpMove(); + } + + function onPointerUp(event: PointerEvent) { + event.preventDefault(); + const { x, y } = pointerAt(event); + enqueueInput(async () => { + await ensureEngaged(); + await sendPointer("up", x, y, buttonFor(event)); + await sendPointer("click", x, y, buttonFor(event)); + }); + } + + function onWheel(event: WheelEvent) { + event.preventDefault(); + const direction = event.deltaY < 0 ? "up" : "down"; + const amount = Math.min(12, Math.max(1, Math.round(Math.abs(event.deltaY) / 40))); + enqueueInput(async () => { + await ensureEngaged(); + await sendScroll(direction, amount); + onUserAction({ kind: "scroll", detail: direction }); + }); + } + + function onKeyDown(event: KeyboardEvent) { + if (event.metaKey || event.ctrlKey || event.altKey) return; + event.preventDefault(); + enqueueInput(async () => { + await ensureEngaged(); + await sendKey(event.key.length === 1 ? event.key : event.code); + onUserAction({ + kind: "type", + detail: event.key.length === 1 ? event.key : event.code, + }); + }); + } + + overlay.addEventListener("pointerdown", onPointerDown); + overlay.addEventListener("pointermove", onPointerMove); + overlay.addEventListener("pointerup", onPointerUp); + overlay.addEventListener("wheel", onWheel, { passive: false }); + overlay.addEventListener("keydown", onKeyDown); + overlay.tabIndex = 0; + return () => { + overlay.removeEventListener("pointerdown", onPointerDown); + overlay.removeEventListener("pointermove", onPointerMove); + overlay.removeEventListener("pointerup", onPointerUp); + overlay.removeEventListener("wheel", onWheel); + overlay.removeEventListener("keydown", onKeyDown); + }; + }, [botId, enabled, height, onEngaged, onUserAction, width]); + + if (!enabled) return null; + return ( +
+ ); +} diff --git a/apps/web/src/components/browser/InAppBrowser.tsx b/apps/web/src/components/browser/InAppBrowser.tsx index 22ac61a..f50b2bd 100644 --- a/apps/web/src/components/browser/InAppBrowser.tsx +++ b/apps/web/src/components/browser/InAppBrowser.tsx @@ -1,214 +1,63 @@ -import { useLingui } from "@lingui/react/macro"; -import type { - RakazoDesktopBrowserPreview, - RakazoDesktopBrowserPreviewState, -} from "@rakazo/contracts"; -import { Button, cn, Input } from "@rakazo/ui-web"; -import { ArrowLeft, ArrowRight, Code, ExternalLink, RotateCw, X } from "lucide-react"; -import { type FormEvent, useCallback, useEffect, useRef, useState } from "react"; -import { browserFrameSandbox, normalizeBrowserUrl } from "../../lib/browser-url"; -import { desktopBridge } from "../../lib/desktop"; +import { Trans, useLingui } from "@lingui/react/macro"; +import type { Bot, ComputerStatus } from "@rakazo/contracts"; +import { Button, cn } from "@rakazo/ui-web"; +import { X } from "lucide-react"; +import { useState } from "react"; +import { LocalBrowserPreview } from "./LocalBrowserPreview"; +import { OoComputerBrowser } from "./OoComputerBrowser"; -const LAST_URL_KEY = "rk:in-app-browser:url"; +export type InAppBrowserMode = "oo" | "local"; -function readLastUrl(): string { - try { - return window.localStorage.getItem(LAST_URL_KEY) ?? ""; - } catch { - return ""; - } -} - -function rememberUrl(url: string) { - try { - window.localStorage.setItem(LAST_URL_KEY, url); - } catch { - /* A per-viewer convenience only. */ - } -} - -const EMPTY_STATE: RakazoDesktopBrowserPreviewState = { - url: "", - title: "", - loading: false, - canGoBack: false, - canGoForward: false, - error: null, -}; - -/** DOM layers the native desktop view would otherwise paint over. */ -const FLOATING_UI = '[role="dialog"],[role="alertdialog"],[role="menu"],[role="listbox"]'; - -/** - * Address bar plus page. On desktop the page is a native Chromium view laid over the reserved - * area, so dev servers keep hot reload, DevTools and full speed; in a browser it is an iframe. - */ -export function InAppBrowser({ onClose }: { onClose: () => void }) { +export function InAppBrowser({ + onClose, + active, + computer, + embeddedScreenUrl, + computerScreenError, + sandboxProvider, + booting, + onBoot, + onRefreshScreen, +}: { + onClose: () => void; + active: Bot | null; + computer: ComputerStatus | null; + embeddedScreenUrl: string | null; + computerScreenError: React.ReactNode; + sandboxProvider: string | undefined; + booting: boolean; + onBoot: () => Promise; + onRefreshScreen: () => Promise; +}) { const { t } = useLingui(); - const native = desktopBridge()?.browserPreview; - const [state, setState] = useState(() => ({ - ...EMPTY_STATE, - url: native ? "" : readLastUrl(), - })); - const [draft, setDraft] = useState(state.url); - const [invalid, setInvalid] = useState(false); - const [frameKey, setFrameKey] = useState(0); - const editing = useRef(false); - const inputRef = useRef(null); - - useEffect(() => { - if (!native) return; - let live = true; - const apply = (next: RakazoDesktopBrowserPreviewState) => { - if (!live) return; - setState(next); - if (!editing.current) setDraft(next.url); - }; - const unsubscribe = native.onState(apply); - void native - .state() - .then((current) => { - apply(current); - const last = readLastUrl(); - if (!current.url && last) void native.navigate(last).catch(() => undefined); - }) - .catch(() => undefined); - return () => { - live = false; - unsubscribe(); - }; - }, [native]); - - useEffect(() => { - if (!state.url) inputRef.current?.focus(); - }, [state.url]); - - const go = useCallback( - (input: string) => { - const url = normalizeBrowserUrl(input); - setInvalid(!url); - if (!url) return; - editing.current = false; - setDraft(url); - rememberUrl(url); - if (native) { - void native.navigate(url).catch(() => setInvalid(true)); - } else { - setState((current) => ({ ...current, url, loading: true, error: null })); - setFrameKey((key) => key + 1); - } - }, - [native], - ); - - const reload = useCallback(() => { - if (native) { - void (state.loading ? native.stop() : native.reload()).catch(() => undefined); - return; - } - if (!state.url) return; - setState((current) => ({ ...current, loading: true })); - setFrameKey((key) => key + 1); - }, [native, state.loading, state.url]); - - const onSubmit = (event: FormEvent) => { - event.preventDefault(); - go(draft); - inputRef.current?.blur(); - }; + const [mode, setMode] = useState("oo"); return (
-
- {native ? ( - <> - - - - ) : null} - - { - editing.current = true; - event.currentTarget.select(); - }} - onBlur={() => { - editing.current = false; - }} - onChange={(event) => { - setDraft(event.target.value); - setInvalid(false); - }} - onKeyDown={(event) => { - if (event.key !== "Escape") return; - setDraft(state.url); - setInvalid(false); - event.currentTarget.blur(); - }} - className="mx-1 h-control-sm flex-1 rounded-full bg-muted/60 px-3 text-ui-body-sm" - /> - {native ? ( +
+
- ) : null} - + +
-
-
-
- -
- {native ? ( - - ) : state.url ? ( -