cynarAI 49431def83
feat: show the bot's live cursor and make the in-app browser work (#113)
Bots now announce every click and typing step before it runs, and the
web app draws a cursor in the bot's colour that glides to that point
over the live computer screen, ripples on clicks and shows typing.

- contracts: ComputerPointerPayload on the existing thread.computer event
- executor: computer_act and browser_act announce pointer steps
  (best effort); the page-browser helper reports element screen boxes,
  which are stripped before results reach the model
- web: agentPointerFeed + AgentCursor over the side-panel preview and
  the full-window screen, mapped through the noVNC letterbox
- in-app browser: native WebContentsView on desktop (own session, http(s)
  only, clamped bounds, back/forward, reload, DevTools, load errors) and
  a sandboxed iframe on the web; replaces the dead panel, the fake
  50%/40% cursors and the unimplemented devServer bridge


Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi

Co-authored-by: Claude <noreply@anthropic.com>
2026-10-03 19:08:48 +02:00

318 lines
9.9 KiB
TypeScript

import { useLingui } from "@lingui/react/macro";
import type {
RakazoDesktopBrowserPreview,
RakazoDesktopBrowserPreviewState,
} from "@rakazo/contracts";
import { Button, cn, Input } from "@rakazo/ui-web";
import { ArrowLeft, ArrowRight, Code, ExternalLink, RotateCw, X } from "lucide-react";
import { type FormEvent, useCallback, useEffect, useRef, useState } from "react";
import { browserFrameSandbox, normalizeBrowserUrl } from "../../lib/browser-url";
import { desktopBridge } from "../../lib/desktop";
const LAST_URL_KEY = "rk:in-app-browser:url";
function readLastUrl(): string {
try {
return window.localStorage.getItem(LAST_URL_KEY) ?? "";
} catch {
return "";
}
}
function rememberUrl(url: string) {
try {
window.localStorage.setItem(LAST_URL_KEY, url);
} catch {
/* A per-viewer convenience only. */
}
}
const EMPTY_STATE: RakazoDesktopBrowserPreviewState = {
url: "",
title: "",
loading: false,
canGoBack: false,
canGoForward: false,
error: null,
};
/** DOM layers the native desktop view would otherwise paint over. */
const FLOATING_UI = '[role="dialog"],[role="alertdialog"],[role="menu"],[role="listbox"]';
/**
* Address bar plus page. On desktop the page is a native Chromium view laid over the reserved
* area, so dev servers keep hot reload, DevTools and full speed; in a browser it is an iframe.
*/
export function InAppBrowser({ onClose }: { onClose: () => void }) {
const { t } = useLingui();
const native = desktopBridge()?.browserPreview;
const [state, setState] = useState<RakazoDesktopBrowserPreviewState>(() => ({
...EMPTY_STATE,
url: native ? "" : readLastUrl(),
}));
const [draft, setDraft] = useState(state.url);
const [invalid, setInvalid] = useState(false);
const [frameKey, setFrameKey] = useState(0);
const editing = useRef(false);
const inputRef = useRef<HTMLInputElement>(null);
useEffect(() => {
if (!native) return;
let live = true;
const apply = (next: RakazoDesktopBrowserPreviewState) => {
if (!live) return;
setState(next);
if (!editing.current) setDraft(next.url);
};
const unsubscribe = native.onState(apply);
void native
.state()
.then((current) => {
apply(current);
const last = readLastUrl();
if (!current.url && last) void native.navigate(last).catch(() => undefined);
})
.catch(() => undefined);
return () => {
live = false;
unsubscribe();
};
}, [native]);
useEffect(() => {
if (!state.url) inputRef.current?.focus();
}, [state.url]);
const go = useCallback(
(input: string) => {
const url = normalizeBrowserUrl(input);
setInvalid(!url);
if (!url) return;
editing.current = false;
setDraft(url);
rememberUrl(url);
if (native) {
void native.navigate(url).catch(() => setInvalid(true));
} else {
setState((current) => ({ ...current, url, loading: true, error: null }));
setFrameKey((key) => key + 1);
}
},
[native],
);
const reload = useCallback(() => {
if (native) {
void (state.loading ? native.stop() : native.reload()).catch(() => undefined);
return;
}
if (!state.url) return;
setState((current) => ({ ...current, loading: true }));
setFrameKey((key) => key + 1);
}, [native, state.loading, state.url]);
const onSubmit = (event: FormEvent) => {
event.preventDefault();
go(draft);
inputRef.current?.blur();
};
return (
<div className="flex h-full min-h-0 flex-col" data-testid="in-app-browser">
<form onSubmit={onSubmit} className="relative flex items-center gap-0.5 border-b px-2 py-1.5">
{native ? (
<>
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Back`}
disabled={!state.canGoBack}
onClick={() => void native.back().catch(() => undefined)}
>
<ArrowLeft strokeWidth={1.7} className="size-icon-md" />
</Button>
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Forward`}
disabled={!state.canGoForward}
onClick={() => void native.forward().catch(() => undefined)}
>
<ArrowRight strokeWidth={1.7} className="size-icon-md" />
</Button>
</>
) : null}
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={state.loading ? t`Stop` : t`Reload`}
disabled={!state.url}
onClick={reload}
>
{state.loading && native ? (
<X strokeWidth={1.8} className="size-icon-md" />
) : (
<RotateCw strokeWidth={1.7} className="size-icon-md" />
)}
</Button>
<Input
ref={inputRef}
value={draft}
aria-label={t`Address`}
aria-invalid={invalid || undefined}
inputMode="url"
autoComplete="off"
autoCapitalize="off"
spellCheck={false}
placeholder="localhost:5173"
onFocus={(event) => {
editing.current = true;
event.currentTarget.select();
}}
onBlur={() => {
editing.current = false;
}}
onChange={(event) => {
setDraft(event.target.value);
setInvalid(false);
}}
onKeyDown={(event) => {
if (event.key !== "Escape") return;
setDraft(state.url);
setInvalid(false);
event.currentTarget.blur();
}}
className="mx-1 h-control-sm flex-1 rounded-full bg-muted/60 px-3 text-ui-body-sm"
/>
{native ? (
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Developer tools`}
disabled={!state.url}
onClick={() => void native.openDevTools().catch(() => undefined)}
>
<Code strokeWidth={1.7} className="size-icon-md" />
</Button>
) : null}
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Open in browser`}
disabled={!state.url}
onClick={() => window.open(state.url, "_blank", "noopener,noreferrer")}
>
<ExternalLink strokeWidth={1.7} className="size-icon-md" />
</Button>
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={t`Close panel`}
onClick={onClose}
>
<X strokeWidth={1.8} className="size-icon-md" />
</Button>
<div
aria-hidden
className={cn(
"absolute inset-x-0 -bottom-px h-0.5 overflow-hidden transition-opacity duration-300",
state.loading ? "opacity-100" : "opacity-0",
)}
>
<div className="rk-browser-progress h-full w-1/3 rounded-full bg-primary/70" />
</div>
</form>
<div className="relative min-h-0 flex-1 bg-background">
{native ? (
<NativeViewport browser={native} visible={Boolean(state.url) && !state.error} />
) : state.url ? (
<iframe
key={frameKey}
title={state.title || t`Page`}
src={state.url}
sandbox={browserFrameSandbox(state.url, window.location.origin)}
allow="clipboard-read; clipboard-write; fullscreen"
onLoad={() => setState((current) => ({ ...current, loading: false }))}
className="h-full w-full border-0 bg-background"
data-testid="in-app-browser-frame"
/>
) : null}
{state.error ? (
<div
role="alert"
className="absolute inset-0 grid place-items-center px-6 text-center text-ui-body-sm text-muted-foreground"
>
{state.error}
</div>
) : null}
</div>
</div>
);
}
/**
* Reserves the page area and keeps the native view glued to it while the panel resizes,
* hiding it whenever app UI (dialogs, menus) floats where the view would paint over it.
*/
function NativeViewport({
browser,
visible,
}: {
browser: RakazoDesktopBrowserPreview;
visible: boolean;
}) {
const ref = useRef<HTMLDivElement>(null);
useEffect(() => {
const element = ref.current;
if (!element) return;
let last = "";
let frame = 0;
const sync = () => {
frame = 0;
const rect = element.getBoundingClientRect();
const covered = Array.from(document.querySelectorAll(FLOATING_UI)).some(
(node) => !element.contains(node),
);
const show = visible && rect.width >= 1 && rect.height >= 1 && !covered;
const bounds = {
x: Math.round(rect.left),
y: Math.round(rect.top),
width: Math.round(rect.width),
height: Math.round(rect.height),
};
const key = show ? JSON.stringify(bounds) : "hidden";
if (key === last) return;
last = key;
void (show ? browser.show(bounds) : browser.hide()).catch(() => undefined);
};
const schedule = () => {
if (!frame) frame = requestAnimationFrame(sync);
};
const resize = new ResizeObserver(schedule);
resize.observe(element);
const floating = new MutationObserver(schedule);
floating.observe(document.body, { childList: true, subtree: true });
window.addEventListener("resize", schedule);
// Sibling layout (sidebars, transitions) can move the area without resizing it.
const settle = setInterval(schedule, 250);
schedule();
return () => {
cancelAnimationFrame(frame);
resize.disconnect();
floating.disconnect();
window.removeEventListener("resize", schedule);
clearInterval(settle);
void browser.hide().catch(() => undefined);
};
}, [browser, visible]);
return <div ref={ref} className="absolute inset-0" data-testid="in-app-browser-viewport" />;
}