* media: check ffmpeg filter support before subtitle operations * sandbox: handle synchronous spawn errors safely in desktop sandbox * Voice-first streaming, non-blocking follow-ups, quiet routines and chat UX Why: questions blocked the chat, follow-up messages had no acknowledgement, message actions sat beside the bubble, and voice turn-taking added latency. What: waiting runs accept steering messages and the bot marks each claimed follow-up as seen; unclaimed follow-ups show as queued. Reaction, copy and reply controls sit under each message. Streaming voice (VAD endpointing, barge-in that stops the run, clause-based TTS), an optimistic input queue, procedural earcons and delta-based quiet routines were added. Onboarding can be skipped straight to the main conversation. Dual-surface layout and project pickers are included. Status colors use semantic tokens. Tested: unit suites for core, adapters, api and web pass, typecheck and Biome are clean. Browser end-to-end tests were not run locally. * Update translation catalogs Re-extracted catalogs for the new and moved strings; the catalog sync test passes. * Add project plan and test readiness notes Notes from the milestone work: feature inventory, test infrastructure and test readiness. No code changes. * fix(ci): align scale guards, space deletion, computer toggle, and mobile expo deps * fix(ci): restore space deletion guard, browser toggle, and run-steps contracts * fix(ci): restore model resolution errors, group stage, and onboarding flow * fix(ci): restore in-app-browser close button, disable stop button while sending, and sync lingui catalogs * fix(ci): address web e2e failures for bot-crud, choice cards, message quote, and voice onboarding * test(web): wait for avatar and rig visibility in oo-avatar motion spec * test(web): wait for persisted message IDs in message-hover-actions pagination spec * Onboarding: always drop the local voice when the voice step is skipped Skipping waited only for the latest voice load and disconnected only when the client saw the connect succeed. An earlier or client-failed connect could still persist the local credential, so a user who skipped voice was left with one. Wait for every load and always disconnect. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs * Shell: hide the optimistic copy once the transcript shows the send A message sent while the bot works streamed into the transcript before its send request resolved, so the same text showed twice until the optimistic bubble was removed. Drop an optimistic message as soon as a persisted user message with the same text appears. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs --------- Co-authored-by: Claude <noreply@anthropic.com>
260 lines
8.5 KiB
TypeScript
260 lines
8.5 KiB
TypeScript
import { Trans, useLingui } from "@lingui/react/macro";
|
|
import type { Bot, ComputerStatus } from "@rakazo/contracts";
|
|
import { Button, cn, Input } from "@rakazo/ui-web";
|
|
import { ArrowLeft, ArrowRight, ExternalLink, Loader2, RotateCw } from "lucide-react";
|
|
import { type FormEvent, useCallback, useEffect, useRef, useState } from "react";
|
|
import { normalizeBrowserUrl } from "../../lib/browser-url";
|
|
import { rpc } from "../../lib/rpc";
|
|
import { screenIframeSandbox } from "../../pages/shell/computer-screen";
|
|
import { ComputersUnavailableHint, computersAreUnavailable } from "../ComputersUnavailableHint";
|
|
import { AgentCursor } from "../computer/AgentCursor";
|
|
import { BrowserCoworkOverlay } from "./BrowserCoworkOverlay";
|
|
|
|
type ChromeState = {
|
|
url: string;
|
|
title: string;
|
|
loading: boolean;
|
|
canGoBack: boolean;
|
|
canGoForward: boolean;
|
|
};
|
|
|
|
const EMPTY_CHROME: ChromeState = {
|
|
url: "",
|
|
title: "",
|
|
loading: false,
|
|
canGoBack: false,
|
|
canGoForward: false,
|
|
};
|
|
|
|
export function OoComputerBrowser({
|
|
active,
|
|
computer,
|
|
embeddedScreenUrl,
|
|
computerScreenError,
|
|
sandboxProvider,
|
|
booting,
|
|
onBoot,
|
|
onRefreshScreen,
|
|
}: {
|
|
active: Bot;
|
|
computer: ComputerStatus | null;
|
|
embeddedScreenUrl: string | null;
|
|
computerScreenError: React.ReactNode;
|
|
sandboxProvider: string | undefined;
|
|
booting: boolean;
|
|
onBoot: () => Promise<void>;
|
|
onRefreshScreen: () => Promise<unknown>;
|
|
}) {
|
|
const { t } = useLingui();
|
|
const [chrome, setChrome] = useState<ChromeState>(EMPTY_CHROME);
|
|
const [draft, setDraft] = useState("");
|
|
const [invalid, setInvalid] = useState(false);
|
|
const [busy, setBusy] = useState(false);
|
|
const editing = useRef(false);
|
|
const inputRef = useRef<HTMLInputElement>(null);
|
|
const engagedScreen = useRef(false);
|
|
|
|
const refreshChrome = useCallback(async () => {
|
|
if (computer?.state !== "running") return;
|
|
try {
|
|
const next = await rpc.computer.browserChromeState({ botId: active.id });
|
|
setChrome(next);
|
|
if (!editing.current) setDraft(next.url);
|
|
} catch {
|
|
/* Address bar may lag the live tab while the computer boots. */
|
|
}
|
|
}, [active.id, computer?.state]);
|
|
|
|
useEffect(() => {
|
|
if (computer?.state !== "running") return;
|
|
void refreshChrome();
|
|
const timer = window.setInterval(() => void refreshChrome(), 4000);
|
|
return () => window.clearInterval(timer);
|
|
}, [computer?.state, refreshChrome]);
|
|
|
|
useEffect(() => {
|
|
if (computer?.state === "running" && (embeddedScreenUrl || computer?.kind === "desktop"))
|
|
return;
|
|
void onBoot().catch(() => undefined);
|
|
}, [active.id, computer?.kind, computer?.state, embeddedScreenUrl, onBoot]);
|
|
|
|
const runNav = useCallback(
|
|
async (task: () => Promise<ChromeState>) => {
|
|
setBusy(true);
|
|
try {
|
|
const next = await task();
|
|
setChrome(next);
|
|
if (!editing.current) setDraft(next.url);
|
|
await onRefreshScreen();
|
|
} finally {
|
|
setBusy(false);
|
|
}
|
|
},
|
|
[onRefreshScreen],
|
|
);
|
|
|
|
const onSubmit = (event: FormEvent) => {
|
|
event.preventDefault();
|
|
const url = normalizeBrowserUrl(draft);
|
|
setInvalid(!url);
|
|
if (!url) return;
|
|
editing.current = false;
|
|
void runNav(async () => {
|
|
const next = await rpc.computer.browserNavigate({ botId: active.id, url });
|
|
await rpc.computer
|
|
.browserUserAction({
|
|
botId: active.id,
|
|
kind: "navigate",
|
|
detail: url,
|
|
})
|
|
.catch(() => undefined);
|
|
return next;
|
|
});
|
|
inputRef.current?.blur();
|
|
};
|
|
|
|
const reportUserAction = useCallback(
|
|
(detail: { kind: "click" | "scroll" | "type"; detail: string }) => {
|
|
void rpc.computer
|
|
.browserUserAction({
|
|
botId: active.id,
|
|
kind: detail.kind,
|
|
detail: detail.detail,
|
|
passwordField: detail.kind === "type" && detail.detail.length > 1 ? false : undefined,
|
|
})
|
|
.catch(() => undefined);
|
|
},
|
|
[active.id],
|
|
);
|
|
|
|
const showScreen = computer?.state === "running" && embeddedScreenUrl && !computerScreenError;
|
|
const unavailable = computersAreUnavailable(sandboxProvider);
|
|
|
|
return (
|
|
<>
|
|
<form onSubmit={onSubmit} className="relative flex items-center gap-0.5 border-b px-2 py-1.5">
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={t`Back`}
|
|
disabled={!chrome.canGoBack || busy}
|
|
onClick={() => void runNav(() => rpc.computer.browserBack({ botId: active.id }))}
|
|
>
|
|
<ArrowLeft strokeWidth={1.7} className="size-icon-md" />
|
|
</Button>
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={t`Forward`}
|
|
disabled={!chrome.canGoForward || busy}
|
|
onClick={() => void runNav(() => rpc.computer.browserForward({ botId: active.id }))}
|
|
>
|
|
<ArrowRight strokeWidth={1.7} className="size-icon-md" />
|
|
</Button>
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={t`Reload`}
|
|
disabled={busy || computer?.state !== "running"}
|
|
onClick={() => void runNav(() => rpc.computer.browserReload({ botId: active.id }))}
|
|
>
|
|
{busy || chrome.loading ? (
|
|
<Loader2 strokeWidth={1.7} className="size-icon-md animate-spin" />
|
|
) : (
|
|
<RotateCw strokeWidth={1.7} className="size-icon-md" />
|
|
)}
|
|
</Button>
|
|
<Input
|
|
ref={inputRef}
|
|
value={draft}
|
|
aria-label={t`Address`}
|
|
aria-invalid={invalid || undefined}
|
|
inputMode="url"
|
|
autoComplete="off"
|
|
autoCapitalize="off"
|
|
spellCheck={false}
|
|
placeholder="https://"
|
|
onFocus={(event) => {
|
|
editing.current = true;
|
|
event.currentTarget.select();
|
|
}}
|
|
onBlur={() => {
|
|
editing.current = false;
|
|
}}
|
|
onChange={(event) => {
|
|
setDraft(event.target.value);
|
|
setInvalid(false);
|
|
}}
|
|
className="mx-1 h-control-sm min-w-0 flex-1 rounded-full bg-muted/60 px-3 text-ui-body-sm"
|
|
/>
|
|
<Button
|
|
type="button"
|
|
variant="ghost"
|
|
size="icon-sm"
|
|
aria-label={t`Open in browser`}
|
|
disabled={!chrome.url}
|
|
onClick={() => window.open(chrome.url, "_blank", "noopener,noreferrer")}
|
|
>
|
|
<ExternalLink strokeWidth={1.7} className="size-icon-md" />
|
|
</Button>
|
|
<div
|
|
aria-hidden
|
|
className={cn(
|
|
"absolute inset-x-0 -bottom-px h-0.5 overflow-hidden transition-opacity duration-300",
|
|
chrome.loading || booting ? "opacity-100" : "opacity-0",
|
|
)}
|
|
>
|
|
<div className="rk-browser-progress h-full w-1/3 rounded-full bg-primary/70" />
|
|
</div>
|
|
</form>
|
|
<div
|
|
className="relative min-h-0 flex-1 bg-background"
|
|
data-testid="oo-computer-browser-viewport"
|
|
>
|
|
{unavailable ? (
|
|
<div className="grid h-full place-items-center px-6 text-center text-ui-body-sm text-muted-foreground">
|
|
<ComputersUnavailableHint />
|
|
</div>
|
|
) : showScreen ? (
|
|
<>
|
|
<iframe
|
|
title={t`${active.name} browser`}
|
|
src={embeddedScreenUrl}
|
|
sandbox={screenIframeSandbox(embeddedScreenUrl)}
|
|
className="h-full w-full border-0 bg-black"
|
|
allow="clipboard-read; clipboard-write; fullscreen"
|
|
data-testid="oo-computer-browser-screen"
|
|
/>
|
|
<AgentCursor botId={active.id} color={active.color} name={active.name} />
|
|
<BrowserCoworkOverlay
|
|
botId={active.id}
|
|
computer={computer}
|
|
enabled
|
|
onEngaged={() => {
|
|
engagedScreen.current = true;
|
|
void onRefreshScreen();
|
|
}}
|
|
onUserAction={reportUserAction}
|
|
/>
|
|
</>
|
|
) : (
|
|
<div className="grid h-full place-items-center px-6 text-center text-ui-body-sm text-muted-foreground">
|
|
{computerScreenError ??
|
|
(booting || computer?.state === "booting" ? (
|
|
<span className="inline-flex items-center gap-2">
|
|
<Loader2 className="size-icon-md animate-spin" aria-hidden />
|
|
<Trans>Starting computer</Trans>
|
|
</span>
|
|
) : (
|
|
<Trans>{active.name}'s browser appears here when the computer is running.</Trans>
|
|
))}
|
|
</div>
|
|
)}
|
|
</div>
|
|
</>
|
|
);
|
|
}
|