Bots now announce every click and typing step before it runs, and the web app draws a cursor in the bot's colour that glides to that point over the live computer screen, ripples on clicks and shows typing. - contracts: ComputerPointerPayload on the existing thread.computer event - executor: computer_act and browser_act announce pointer steps (best effort); the page-browser helper reports element screen boxes, which are stripped before results reach the model - web: agentPointerFeed + AgentCursor over the side-panel preview and the full-window screen, mapped through the noVNC letterbox - in-app browser: native WebContentsView on desktop (own session, http(s) only, clamped bounds, back/forward, reload, DevTools, load errors) and a sandboxed iframe on the web; replaces the dead panel, the fake 50%/40% cursors and the unimplemented devServer bridge Claude-Session: https://claude.ai/code/session_01WiGooXtsKrZQmdszepRrNi Co-authored-by: Claude <noreply@anthropic.com>
318 lines
9.9 KiB
TypeScript
318 lines
9.9 KiB
TypeScript
import { useLingui } from "@lingui/react/macro";
|
|
import type {
|
|
RakazoDesktopBrowserPreview,
|
|
RakazoDesktopBrowserPreviewState,
|
|
} from "@rakazo/contracts";
|
|
import { Button, cn, Input } from "@rakazo/ui-web";
|
|
import { ArrowLeft, ArrowRight, Code, ExternalLink, RotateCw, X } from "lucide-react";
|
|
import { type FormEvent, useCallback, useEffect, useRef, useState } from "react";
|
|
import { browserFrameSandbox, normalizeBrowserUrl } from "../../lib/browser-url";
|
|
import { desktopBridge } from "../../lib/desktop";
|
|
|
|
const LAST_URL_KEY = "rk:in-app-browser:url";
|
|
|
|
function readLastUrl(): string {
|
|
try {
|
|
return window.localStorage.getItem(LAST_URL_KEY) ?? "";
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
function rememberUrl(url: string) {
|
|
try {
|
|
window.localStorage.setItem(LAST_URL_KEY, url);
|
|
} catch {
|
|
/* A per-viewer convenience only. */
|
|
}
|
|
}
|
|
|
|
const EMPTY_STATE: RakazoDesktopBrowserPreviewState = {
|
|
url: "",
|
|
title: "",
|
|
loading: false,
|
|
canGoBack: false,
|
|
canGoForward: false,
|
|
error: null,
|
|
};
|
|
|
|
/** DOM layers the native desktop view would otherwise paint over. */
|
|
const FLOATING_UI = '[role="dialog"],[role="alertdialog"],[role="menu"],[role="listbox"]';
|
|
|
|
/**
|
|
* Address bar plus page. On desktop the page is a native Chromium view laid over the reserved
|
|
* area, so dev servers keep hot reload, DevTools and full speed; in a browser it is an iframe.
|
|
*/
|
|
export function InAppBrowser({ onClose }: { onClose: () => void }) {
|
|
const { t } = useLingui();
|
|
const native = desktopBridge()?.browserPreview;
|
|
const [state, setState] = useState<RakazoDesktopBrowserPreviewState>(() => ({
|
|
...EMPTY_STATE,
|
|
url: native ? "" : readLastUrl(),
|
|
}));
|
|
const [draft, setDraft] = useState(state.url);
|
|
const [invalid, setInvalid] = useState(false);
|
|
const [frameKey, setFrameKey] = useState(0);
|
|
const editing = useRef(false);
|
|
const inputRef = useRef<HTMLInputElement>(null);
|
|
|
|
useEffect(() => {
|
|
if (!native) return;
|
|
let live = true;
|
|
const apply = (next: RakazoDesktopBrowserPreviewState) => {
|
|
if (!live) return;
|
|
setState(next);
|
|
if (!editing.current) setDraft(next.url);
|
|
};
|
|
const unsubscribe = native.onState(apply);
|
|
void native
|
|
.state()
|
|
.then((current) => {
|
|
apply(current);
|
|
const last = readLastUrl();
|
|
if (!current.url && last) void native.navigate(last).catch(() => undefined);
|
|
})
|
|
.catch(() => undefined);
|
|
return () => {
|
|
live = false;
|
|
unsubscribe();
|
|
};
|
|
}, [native]);
|
|
|
|
useEffect(() => {
|
|
if (!state.url) inputRef.current?.focus();
|
|
}, [state.url]);
|
|
|
|
const go = useCallback(
|
|
(input: string) => {
|
|
const url = normalizeBrowserUrl(input);
|
|
setInvalid(!url);
|
|
if (!url) return;
|
|
editing.current = false;
|
|
setDraft(url);
|
|
rememberUrl(url);
|
|
if (native) {
|
|
void native.navigate(url).catch(() => setInvalid(true));
|
|
} else {
|
|
setState((current) => ({ ...current, url, loading: true, error: null }));
|
|
setFrameKey((key) => key + 1);
|
|
}
|
|
},
|
|
[native],
|
|
);
|
|
|
|
const reload = useCallback(() => {
|
|
if (native) {
|
|
void (state.loading ? native.stop() : native.reload()).catch(() => undefined);
|
|
return;
|
|
}
|
|
if (!state.url) return;
|
|
setState((current) => ({ ...current, loading: true }));
|
|
setFrameKey((key) => key + 1);
|
|
}, [native, state.loading, state.url]);
|
|
|
|
const onSubmit = (event: FormEvent) => {
|
|
event.preventDefault();
|
|
go(draft);
|
|
inputRef.current?.blur();
|
|
};
|
|
|
|
return (
|
|
<div className="flex h-full min-h-0 flex-col" data-testid="in-app-browser">
|
|
<form onSubmit={onSubmit} className="relative flex items-center gap-0.5 border-b px-2 py-1.5">
|
|
{native ? (
|
|
<>
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={t`Back`}
|
|
disabled={!state.canGoBack}
|
|
onClick={() => void native.back().catch(() => undefined)}
|
|
>
|
|
<ArrowLeft strokeWidth={1.7} className="size-icon-md" />
|
|
</Button>
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={t`Forward`}
|
|
disabled={!state.canGoForward}
|
|
onClick={() => void native.forward().catch(() => undefined)}
|
|
>
|
|
<ArrowRight strokeWidth={1.7} className="size-icon-md" />
|
|
</Button>
|
|
</>
|
|
) : null}
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={state.loading ? t`Stop` : t`Reload`}
|
|
disabled={!state.url}
|
|
onClick={reload}
|
|
>
|
|
{state.loading && native ? (
|
|
<X strokeWidth={1.8} className="size-icon-md" />
|
|
) : (
|
|
<RotateCw strokeWidth={1.7} className="size-icon-md" />
|
|
)}
|
|
</Button>
|
|
<Input
|
|
ref={inputRef}
|
|
value={draft}
|
|
aria-label={t`Address`}
|
|
aria-invalid={invalid || undefined}
|
|
inputMode="url"
|
|
autoComplete="off"
|
|
autoCapitalize="off"
|
|
spellCheck={false}
|
|
placeholder="localhost:5173"
|
|
onFocus={(event) => {
|
|
editing.current = true;
|
|
event.currentTarget.select();
|
|
}}
|
|
onBlur={() => {
|
|
editing.current = false;
|
|
}}
|
|
onChange={(event) => {
|
|
setDraft(event.target.value);
|
|
setInvalid(false);
|
|
}}
|
|
onKeyDown={(event) => {
|
|
if (event.key !== "Escape") return;
|
|
setDraft(state.url);
|
|
setInvalid(false);
|
|
event.currentTarget.blur();
|
|
}}
|
|
className="mx-1 h-control-sm flex-1 rounded-full bg-muted/60 px-3 text-ui-body-sm"
|
|
/>
|
|
{native ? (
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={t`Developer tools`}
|
|
disabled={!state.url}
|
|
onClick={() => void native.openDevTools().catch(() => undefined)}
|
|
>
|
|
<Code strokeWidth={1.7} className="size-icon-md" />
|
|
</Button>
|
|
) : null}
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={t`Open in browser`}
|
|
disabled={!state.url}
|
|
onClick={() => window.open(state.url, "_blank", "noopener,noreferrer")}
|
|
>
|
|
<ExternalLink strokeWidth={1.7} className="size-icon-md" />
|
|
</Button>
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={t`Close panel`}
|
|
onClick={onClose}
|
|
>
|
|
<X strokeWidth={1.8} className="size-icon-md" />
|
|
</Button>
|
|
<div
|
|
aria-hidden
|
|
className={cn(
|
|
"absolute inset-x-0 -bottom-px h-0.5 overflow-hidden transition-opacity duration-300",
|
|
state.loading ? "opacity-100" : "opacity-0",
|
|
)}
|
|
>
|
|
<div className="rk-browser-progress h-full w-1/3 rounded-full bg-primary/70" />
|
|
</div>
|
|
</form>
|
|
<div className="relative min-h-0 flex-1 bg-background">
|
|
{native ? (
|
|
<NativeViewport browser={native} visible={Boolean(state.url) && !state.error} />
|
|
) : state.url ? (
|
|
<iframe
|
|
key={frameKey}
|
|
title={state.title || t`Page`}
|
|
src={state.url}
|
|
sandbox={browserFrameSandbox(state.url, window.location.origin)}
|
|
allow="clipboard-read; clipboard-write; fullscreen"
|
|
onLoad={() => setState((current) => ({ ...current, loading: false }))}
|
|
className="h-full w-full border-0 bg-background"
|
|
data-testid="in-app-browser-frame"
|
|
/>
|
|
) : null}
|
|
{state.error ? (
|
|
<div
|
|
role="alert"
|
|
className="absolute inset-0 grid place-items-center px-6 text-center text-ui-body-sm text-muted-foreground"
|
|
>
|
|
{state.error}
|
|
</div>
|
|
) : null}
|
|
</div>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Reserves the page area and keeps the native view glued to it while the panel resizes,
|
|
* hiding it whenever app UI (dialogs, menus) floats where the view would paint over it.
|
|
*/
|
|
function NativeViewport({
|
|
browser,
|
|
visible,
|
|
}: {
|
|
browser: RakazoDesktopBrowserPreview;
|
|
visible: boolean;
|
|
}) {
|
|
const ref = useRef<HTMLDivElement>(null);
|
|
|
|
useEffect(() => {
|
|
const element = ref.current;
|
|
if (!element) return;
|
|
let last = "";
|
|
let frame = 0;
|
|
const sync = () => {
|
|
frame = 0;
|
|
const rect = element.getBoundingClientRect();
|
|
const covered = Array.from(document.querySelectorAll(FLOATING_UI)).some(
|
|
(node) => !element.contains(node),
|
|
);
|
|
const show = visible && rect.width >= 1 && rect.height >= 1 && !covered;
|
|
const bounds = {
|
|
x: Math.round(rect.left),
|
|
y: Math.round(rect.top),
|
|
width: Math.round(rect.width),
|
|
height: Math.round(rect.height),
|
|
};
|
|
const key = show ? JSON.stringify(bounds) : "hidden";
|
|
if (key === last) return;
|
|
last = key;
|
|
void (show ? browser.show(bounds) : browser.hide()).catch(() => undefined);
|
|
};
|
|
const schedule = () => {
|
|
if (!frame) frame = requestAnimationFrame(sync);
|
|
};
|
|
const resize = new ResizeObserver(schedule);
|
|
resize.observe(element);
|
|
const floating = new MutationObserver(schedule);
|
|
floating.observe(document.body, { childList: true, subtree: true });
|
|
window.addEventListener("resize", schedule);
|
|
// Sibling layout (sidebars, transitions) can move the area without resizing it.
|
|
const settle = setInterval(schedule, 250);
|
|
schedule();
|
|
return () => {
|
|
cancelAnimationFrame(frame);
|
|
resize.disconnect();
|
|
floating.disconnect();
|
|
window.removeEventListener("resize", schedule);
|
|
clearInterval(settle);
|
|
void browser.hide().catch(() => undefined);
|
|
};
|
|
}, [browser, visible]);
|
|
|
|
return <div ref={ref} className="absolute inset-0" data-testid="in-app-browser-viewport" />;
|
|
}
|