From e28c3acf027c04030f467f114ff515638411fdfc Mon Sep 17 00:00:00 2001 From: cynarAI <155274210+cynarAI@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:23:46 +0200 Subject: [PATCH 1/2] feat: let the pane beside a conversation follow the bot's work (#123) The live screen opens wide beside the chat while the bot works on it or asks for a takeover, a file the bot presents as its result opens next to the conversation, and a question that scrolled away gets a "Needs you" pill back to it. Shared core rules derive the focus from signals clients already have (pointer events, takeover requests, a new `present` flag on attach_file, pending asks), move the pane only on focus edges, never replace a pane the user opened, and learn per bot whether the screen opens on its own. Pane widths persist per device; mobile shows a pill that opens the computer. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7VSRHHzyt2KwoCkUkZ2Gc --- apps/mobile/app/thread.tsx | 61 ++++ apps/mobile/lib/api.ts | 3 + apps/web/e2e/adaptive-workspace.spec.ts | 104 ++++++ apps/web/e2e/artifact-preview.spec.ts | 2 + apps/web/e2e/golden.spec.ts | 5 +- apps/web/e2e/group-chats.spec.ts | 8 +- apps/web/src/components/ArtifactFileCard.tsx | 57 +--- apps/web/src/components/ArtifactPane.tsx | 165 ++++++++++ apps/web/src/components/AskCard.tsx | 1 + apps/web/src/hooks/useArtifactBytes.ts | 38 +++ apps/web/src/hooks/useSidebarResize.ts | 14 +- apps/web/src/hooks/useWideLayout.ts | 16 + apps/web/src/lib/agent-pointer.ts | 4 +- apps/web/src/lib/csv-preview.test.ts | 31 ++ apps/web/src/lib/csv-preview.ts | 43 +++ apps/web/src/lib/workspace-pane-pref.test.ts | 61 ++++ apps/web/src/lib/workspace-pane-pref.ts | 52 +++ apps/web/src/pages/Shell.tsx | 243 +++++++++++++- docs/ADAPTIVE-WORKSPACE.md | 32 ++ packages/adapters/src/builtin-tools.ts | 11 +- packages/adapters/src/executor.ts | 1 + packages/adapters/src/scripted-runtime.ts | 14 + .../adapters/src/thread-artifacts.test.ts | 33 ++ packages/adapters/src/thread-artifacts.ts | 17 +- packages/contracts/src/events.ts | 4 + packages/core/src/index.ts | 1 + packages/core/src/workspace-focus.test.ts | 305 ++++++++++++++++++ packages/core/src/workspace-focus.ts | 230 +++++++++++++ 28 files changed, 1485 insertions(+), 71 deletions(-) create mode 100644 apps/web/e2e/adaptive-workspace.spec.ts create mode 100644 apps/web/src/components/ArtifactPane.tsx create mode 100644 apps/web/src/hooks/useArtifactBytes.ts create mode 100644 apps/web/src/hooks/useWideLayout.ts create mode 100644 apps/web/src/lib/csv-preview.test.ts create mode 100644 apps/web/src/lib/csv-preview.ts create mode 100644 apps/web/src/lib/workspace-pane-pref.test.ts create mode 100644 apps/web/src/lib/workspace-pane-pref.ts create mode 100644 docs/ADAPTIVE-WORKSPACE.md create mode 100644 packages/core/src/workspace-focus.test.ts create mode 100644 packages/core/src/workspace-focus.ts diff --git a/apps/mobile/app/thread.tsx b/apps/mobile/app/thread.tsx index c2eb734..3c013ae 100644 --- a/apps/mobile/app/thread.tsx +++ b/apps/mobile/app/thread.tsx @@ -17,10 +17,12 @@ import { buildComposerMentionOptions, type ComposerMention, cloudAgentHttpsUrl, + deriveWorkspaceFocus, isApprovalAskBlock, isRunTerminalEvent, isSecretAskBlock, latestAnswerableAskMessageId, + liveScreenRunId, mentionChipKey, projectMessageReactions, resolveComposerSendPlan, @@ -288,6 +290,8 @@ function Thread() { ? { botId } : undefined; const [snap, setSnap] = useState(null); + /** Run that last pointed at the bot's screen, so the live screen is offered only during real screen work. */ + const [screenRunId, setScreenRunId] = useState(null); const activeThreadId = useRef(undefined); const [draft, setDraft] = useState(""); const [mentionQuery, setMentionQuery] = useState(null); @@ -860,6 +864,7 @@ function Thread() { } expandedHistoryThread.current = null; historyEpoch.current += 1; + setScreenRunId(null); const abort = new AbortController(); void (async () => { // Pending search jumps load the around-page separately; avoid replacing it with latest. @@ -885,6 +890,8 @@ function Thread() { (event) => { cursor = Math.max(cursor, event.seq ?? -1); retryMs = 250; + const screenRun = groupId ? null : liveScreenRunId(event); + if (screenRun) setScreenRunId(screenRun); if ( event.type === "thread.progress" || event.type === "agent.tool.called" || @@ -1271,6 +1278,23 @@ function Thread() { } const answerableAskMessageId = latestAnswerableAskMessageId(snap); + const screenFocus = useMemo(() => { + const computer = snap?.computer; + if (!snap || !computer || groupId) return null; + const focus = deriveWorkspaceFocus( + { + ...snap, + computer: { + kind: computer.kind ?? "", + state: computer.state, + screenAvailable: computer.screenAvailable, + takeoverRequested: computer.takeoverRequested ?? false, + }, + }, + { screenRunId }, + ); + return focus.pane === "screen" ? focus : null; + }, [groupId, screenRunId, snap]); const runError = snap?.run?.status === "failed" ? (snap.run.error ?? null) : null; const liveMessages = useMemo(() => [...visibleMessages].reverse(), [visibleMessages]); const messagesById = useMemo( @@ -1668,6 +1692,43 @@ function Thread() { ) : null} + {screenFocus && botId ? ( + + router.push({ + pathname: "/computer", + params: { botId, name: displayName ?? t("Bot") }, + }) + } + style={({ pressed }) => ({ + alignSelf: "center", + marginTop: 10, + flexDirection: "row", + alignItems: "center", + gap: 8, + borderRadius: 999, + borderWidth: 1, + borderColor: native.fillPressed, + backgroundColor: pressed ? native.fillPressed : native.fill, + paddingHorizontal: 14, + paddingVertical: 8, + })} + > + + + {screenFocus.takeover ? t("Needs takeover") : t("Open computer")} + + + ) : null} {replyTarget ? ( ; members?: MobileGroup["members"]; computer?: { + kind?: string; state: string; controlHolder: string; screenAvailable: boolean; + takeoverRequested?: boolean; mode: ComputerMode; busyBotName: string | null; }; @@ -896,6 +898,7 @@ type ThreadEvent = { type: string; seq?: number; runId?: string; + createdAt?: string; payload?: Record; }; diff --git a/apps/web/e2e/adaptive-workspace.spec.ts b/apps/web/e2e/adaptive-workspace.spec.ts new file mode 100644 index 0000000..b4b4416 --- /dev/null +++ b/apps/web/e2e/adaptive-workspace.spec.ts @@ -0,0 +1,104 @@ +import { expect, test } from "@playwright/test"; +import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers"; + +const SCREEN_URL = "https://screen.example/embed.html"; + +test("the bot's screen opens beside the chat while it works and steps back after", async ({ + page, +}, testInfo) => { + await signup(page, `workspace-screen-${Date.now()}@rakazo.test`, "password12", "Screen Pane"); + await completeOnboarding(page); + const botId = activeBotId(page); + await rpc(page, "computer/boot", { botId }); + await page.route("https://screen.example/**", (route) => + route.fulfill({ + contentType: "text/html", + body: `Desktop + `, + }), + ); + await page.route("**/rpc/computer/screenUrl", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { url: SCREEN_URL } }), + }), + ); + + const sidePanel = page.getByTestId("side-panel"); + await expect(sidePanel).toHaveAttribute("data-panel", "closed"); + + await rpc(page, "threads/send", { botId, text: "use your screen and type workspace-demo" }); + + await expect(sidePanel).toHaveAttribute("data-panel", "computer"); + await expect(sidePanel).toHaveAttribute("data-auto", ""); + await expect(page.getByTestId("computer-preview").locator("iframe")).toHaveAttribute( + "src", + SCREEN_URL, + ); + // The live screen gets more room than a settings panel. + expect((await sidePanel.boundingBox())!.width).toBeGreaterThan(400); + await captureScreenshot(page, testInfo, "workspace-screen-beside-chat"); + + // Once the work is done the conversation gets its full width back. + await expect(sidePanel).toHaveAttribute("data-panel", "closed", { timeout: 20_000 }); +}); + +test("a report the bot presents opens beside the conversation", async ({ page }, testInfo) => { + await signup(page, `workspace-report-${Date.now()}@rakazo.test`, "password12", "Report Pane"); + await completeOnboarding(page); + + const composer = page.getByPlaceholder(/Message/); + await composer.fill("present the report"); + await page.keyboard.press("Enter"); + + const pane = page.getByTestId("artifact-pane"); + await expect(pane).toBeVisible({ timeout: 30_000 }); + await expect(pane.getByRole("heading", { name: "report.md" })).toBeVisible(); + await expect(pane.getByRole("heading", { name: "Report", exact: true })).toBeVisible(); + await expect(pane.getByRole("cell", { name: "Research" })).toBeVisible(); + await expect(composer).toBeVisible(); + await captureScreenshot(page, testInfo, "workspace-report-beside-chat"); + + const downloadPromise = page.waitForEvent("download"); + await pane.getByRole("button", { name: "Download report.md" }).click(); + expect((await downloadPromise).suggestedFilename()).toBe("report.md"); + + await pane.getByRole("button", { name: "Close panel" }).click(); + await expect(pane).toHaveCount(0); + + // The file card in the chat opens the same pane again. + await page.getByRole("button", { name: "Preview report.md" }).click(); + await expect(pane.getByRole("heading", { name: "Report", exact: true })).toBeVisible(); +}); + +test("a question that scrolled away is one click back", async ({ page }, testInfo) => { + await page.setViewportSize({ width: 1280, height: 520 }); + await signup(page, `workspace-ask-${Date.now()}@rakazo.test`, "password12", "Ask Jump"); + await completeOnboarding(page); + + const composer = page.getByPlaceholder(/Message/); + // Earlier work gives the thread enough history to scroll back through. + for (const task of ["sort the inbox", "draft the agenda", "book the room"]) { + await composer.fill(task); + await page.keyboard.press("Enter"); + await expect(page.getByText(`done. i handled: ${task}`, { exact: true })).toBeVisible({ + timeout: 30_000, + }); + } + await composer.fill("ask me which city to use"); + await page.keyboard.press("Enter"); + const sendIt = page.getByRole("button", { name: "Send it" }); + await expect(sendIt).toBeVisible({ timeout: 30_000 }); + + const jump = page.getByTestId("needs-you-jump"); + await expect(jump).toHaveCount(0); + await page.getByTestId("transcript").hover(); + await page.mouse.wheel(0, -20_000); + await expect(jump).toBeVisible(); + await captureScreenshot(page, testInfo, "workspace-needs-you-jump"); + + await jump.click(); + await expect(sendIt).toBeInViewport(); + await expect(sendIt).toBeFocused(); + await expect(jump).toHaveCount(0); +}); diff --git a/apps/web/e2e/artifact-preview.spec.ts b/apps/web/e2e/artifact-preview.spec.ts index abeb027..8fa54ce 100644 --- a/apps/web/e2e/artifact-preview.spec.ts +++ b/apps/web/e2e/artifact-preview.spec.ts @@ -27,6 +27,8 @@ test("agent-attached files appear as downloadable cards", async ({ page }, testI test("agent-attached Markdown opens a rendered preview and can be downloaded", async ({ page, }, testInfo) => { + // Narrow windows preview in a dialog; wide ones open the file beside the chat. + await page.setViewportSize({ width: 900, height: 800 }); const stamp = Date.now(); await signup(page, `markdown-preview-${stamp}@rakazo.test`, "password12", "Markdown Preview"); await completeOnboarding(page); diff --git a/apps/web/e2e/golden.spec.ts b/apps/web/e2e/golden.spec.ts index 367349b..17ace91 100644 --- a/apps/web/e2e/golden.spec.ts +++ b/apps/web/e2e/golden.spec.ts @@ -69,9 +69,10 @@ test("takeover, routine, plugins, and export are reachable", async ({ page }, te }) .toBe("waiting_takeover"); await captureScreenshot(page, testInfo, "08-protected-input-request"); - await page.getByTitle("Agent computer").click(); + // A takeover request brings the bot's screen beside the chat on its own. const sidePanel = page.getByTestId("side-panel"); - await expect(sidePanel).toHaveCSS("width", "384px"); + await expect(sidePanel).toHaveAttribute("data-panel", "computer"); + await expect(sidePanel).toHaveAttribute("data-auto", ""); const [mainBox, panelBox] = await Promise.all([ page.locator("main").boundingBox(), sidePanel.boundingBox(), diff --git a/apps/web/e2e/group-chats.spec.ts b/apps/web/e2e/group-chats.spec.ts index e5d449c..762432a 100644 --- a/apps/web/e2e/group-chats.spec.ts +++ b/apps/web/e2e/group-chats.spec.ts @@ -303,9 +303,11 @@ test("create group from + and see two bots in one transcript", async ({ page }, const groupMarkdown = page.getByRole("button", { name: "Preview group-preview.md" }); await expect(groupMarkdown).toBeVisible({ timeout: 30_000 }); await groupMarkdown.click(); - const markdownDialog = page.getByRole("dialog", { name: "group-preview.md" }); - await expect(markdownDialog.getByRole("heading", { name: "Group artifact" })).toBeVisible(); - await markdownDialog.getByRole("button", { name: "Close preview" }).click(); + // Wide windows open the file beside the group conversation. + const markdownPane = page.getByTestId("artifact-pane"); + await expect(markdownPane.getByRole("heading", { name: "Group artifact" })).toBeVisible(); + await markdownPane.getByRole("button", { name: "Close panel" }).click(); + await expect(markdownPane).toHaveCount(0); await page.setViewportSize({ width: 390, height: 844 }); await expect(page.getByRole("button", { name: "Open navigation" })).toBeVisible(); diff --git a/apps/web/src/components/ArtifactFileCard.tsx b/apps/web/src/components/ArtifactFileCard.tsx index b36eb65..7122a97 100644 --- a/apps/web/src/components/ArtifactFileCard.tsx +++ b/apps/web/src/components/ArtifactFileCard.tsx @@ -4,13 +4,9 @@ import { Trans, useLingui } from "@lingui/react/macro"; import { ChatMarkdown } from "@rakazo/chat-ui/web"; import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web"; import { Download, FileText, X } from "lucide-react"; -import { type RefObject, useEffect, useRef, useState } from "react"; -import { - type ArtifactTarget, - downloadArtifact, - downloadArtifactBytes, - fetchArtifactBytes, -} from "../lib/artifact-open"; +import { type RefObject, useMemo, useRef, useState } from "react"; +import { useArtifactBytes } from "../hooks/useArtifactBytes"; +import { type ArtifactTarget, downloadArtifact, downloadArtifactBytes } from "../lib/artifact-open"; type ArtifactFileCardProps = { target: ArtifactTarget; @@ -18,6 +14,8 @@ type ArtifactFileCardProps = { name: string; mimeType: string; size: number; + /** Shows the preview beside the conversation instead of in a dialog. */ + onOpenBeside?: () => void; }; export function ArtifactFileCard(props: ArtifactFileCardProps) { @@ -63,7 +61,7 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) { ref={previewButton} type="button" aria-label={t`Preview ${props.name}`} - onClick={() => setPreviewOpen(true)} + onClick={() => (props.onOpenBeside ? props.onOpenBeside() : setPreviewOpen(true))} className="flex min-w-0 flex-1 items-center gap-3 px-4 py-3 text-left hover:bg-accent" > @@ -111,39 +109,16 @@ function MarkdownPreview({ }: ArtifactFileCardProps & { closeButtonRef: RefObject }) { const { t } = useLingui(); const [downloadError, setDownloadError] = useState(null); - const [state, setState] = useState< - | { status: "loading" } - | { status: "ready"; bytes: Uint8Array; markdown: string } - | { status: "error"; message: string } - >({ status: "loading" }); - const targetBotId = "botId" in target ? target.botId : undefined; - const targetGroupId = "groupId" in target ? target.groupId : undefined; - - useEffect(() => { - let cancelled = false; - const artifactTarget: ArtifactTarget = - targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! }; - void fetchArtifactBytes(artifactTarget, artifactId) - .then((bytes) => { - if (cancelled) return; - try { - const markdown = new TextDecoder("utf-8", { fatal: true }).decode(bytes); - setState({ status: "ready", bytes, markdown }); - } catch { - setState({ status: "error", message: t`This file is not valid UTF-8 Markdown.` }); - } - }) - .catch((error) => { - if (cancelled) return; - setState({ - status: "error", - message: error instanceof Error ? error.message : t`Could not load this file.`, - }); - }); - return () => { - cancelled = true; - }; - }, [artifactId, targetBotId, targetGroupId, t]); + const loaded = useArtifactBytes(target, artifactId); + const state = useMemo(() => { + if (loaded.status !== "ready") return loaded; + try { + const markdown = new TextDecoder("utf-8", { fatal: true }).decode(loaded.bytes); + return { status: "ready" as const, bytes: loaded.bytes, markdown }; + } catch { + return { status: "error" as const, message: t`This file is not valid UTF-8 Markdown.` }; + } + }, [loaded, t]); return ( <> diff --git a/apps/web/src/components/ArtifactPane.tsx b/apps/web/src/components/ArtifactPane.tsx new file mode 100644 index 0000000..90f628f --- /dev/null +++ b/apps/web/src/components/ArtifactPane.tsx @@ -0,0 +1,165 @@ +import { Trans, useLingui } from "@lingui/react/macro"; +import { ChatMarkdown } from "@rakazo/chat-ui/web"; +import { isAttachmentImageMimeType } from "@rakazo/contracts"; +import type { WorkspaceArtifact } from "@rakazo/core"; +import { Button } from "@rakazo/ui-web"; +import { Download, X } from "lucide-react"; +import { useEffect, useMemo, useState } from "react"; +import { useArtifactBytes } from "../hooks/useArtifactBytes"; +import { type ArtifactTarget, downloadArtifactBytes } from "../lib/artifact-open"; +import { parseCsvPreview } from "../lib/csv-preview"; + +/** A file the bot made, open beside the conversation so the user can read it while they talk. */ +export function ArtifactPane({ + target, + artifact, + onClose, +}: { + target: ArtifactTarget; + artifact: WorkspaceArtifact; + onClose: () => void; +}) { + const { t } = useLingui(); + const loaded = useArtifactBytes(target, artifact.artifactId); + const { name, mimeType } = artifact; + + return ( +
+
+

+ {name} +

+ + +
+
+ {loaded.status === "loading" ? ( +
+ Loading preview… +
+ ) : loaded.status === "error" ? ( +
+ {loaded.message} +
+ ) : isAttachmentImageMimeType(mimeType) ? ( + + ) : ( + + )} +
+
+ ); +} + +function PaneImage({ + bytes, + mimeType, + name, +}: { + bytes: Uint8Array; + mimeType: string; + name: string; +}) { + const [src, setSrc] = useState(null); + useEffect(() => { + const url = URL.createObjectURL(new Blob([new Uint8Array(bytes)], { type: mimeType })); + setSrc(url); + return () => URL.revokeObjectURL(url); + }, [bytes, mimeType]); + return src ? ( + {name} + ) : null; +} + +function PaneText({ bytes, mimeType }: { bytes: Uint8Array; mimeType: string }) { + const { t } = useLingui(); + const text = useMemo(() => new TextDecoder("utf-8").decode(bytes), [bytes]); + if (mimeType === "text/markdown") { + return ( +
+ {text} +
+ ); + } + if (mimeType === "text/csv") { + const { rows, truncated } = parseCsvPreview(text); + const [head, ...body] = rows; + const shownRows = body.length; + return ( +
+ + {head ? ( + + + {head.map((cell, index) => ( + + ))} + + + ) : null} + + {body.map((row, rowIndex) => ( + + {row.map((cell, index) => ( + + ))} + + ))} + +
+ {cell} +
+ {cell} +
+ {truncated ? ( +

+ {t`First ${shownRows} rows`} +

+ ) : null} +
+ ); + } + return ( + 
+      {mimeType === "application/json" ? prettyJson(text) : text}
+
+ ); +} + +function prettyJson(text: string): string { + try { + return JSON.stringify(JSON.parse(text), null, 2); + } catch { + return text; + } +} diff --git a/apps/web/src/components/AskCard.tsx b/apps/web/src/components/AskCard.tsx index 3b4f59d..c6ad939 100644 --- a/apps/web/src/components/AskCard.tsx +++ b/apps/web/src/components/AskCard.tsx @@ -87,6 +87,7 @@ export function AskCard({ return (
diff --git a/apps/web/src/hooks/useArtifactBytes.ts b/apps/web/src/hooks/useArtifactBytes.ts new file mode 100644 index 0000000..e0b61dd --- /dev/null +++ b/apps/web/src/hooks/useArtifactBytes.ts @@ -0,0 +1,38 @@ +import { useLingui } from "@lingui/react/macro"; +import { useEffect, useState } from "react"; +import { type ArtifactTarget, fetchArtifactBytes } from "../lib/artifact-open"; + +export type ArtifactBytesState = + | { status: "loading" } + | { status: "ready"; bytes: Uint8Array } + | { status: "error"; message: string }; + +export function useArtifactBytes(target: ArtifactTarget, artifactId: string): ArtifactBytesState { + const { t } = useLingui(); + const [state, setState] = useState({ status: "loading" }); + const targetBotId = "botId" in target ? target.botId : undefined; + const targetGroupId = "groupId" in target ? target.groupId : undefined; + + useEffect(() => { + let cancelled = false; + setState({ status: "loading" }); + const artifactTarget: ArtifactTarget = + targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! }; + void fetchArtifactBytes(artifactTarget, artifactId) + .then((bytes) => { + if (!cancelled) setState({ status: "ready", bytes }); + }) + .catch((error) => { + if (cancelled) return; + setState({ + status: "error", + message: error instanceof Error ? error.message : t`Could not load this file.`, + }); + }); + return () => { + cancelled = true; + }; + }, [artifactId, targetBotId, targetGroupId, t]); + + return state; +} diff --git a/apps/web/src/hooks/useSidebarResize.ts b/apps/web/src/hooks/useSidebarResize.ts index 9932858..0835844 100644 --- a/apps/web/src/hooks/useSidebarResize.ts +++ b/apps/web/src/hooks/useSidebarResize.ts @@ -1,13 +1,20 @@ -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { readPaneWidth, writePaneWidth } from "../lib/workspace-pane-pref"; export function useSidebarResize( initialWidth: number, side: "left" | "right", minWidth: number, maxWidth: number, + /** Remembers the width across reloads on this device. */ + storageKey?: string, ) { - const [width, setWidth] = useState(initialWidth); + const [width, setWidth] = useState(() => + storageKey ? readPaneWidth(storageKey, initialWidth, minWidth, maxWidth) : initialWidth, + ); const [isResizing, setIsResizing] = useState(false); + const widthRef = useRef(width); + widthRef.current = width; useEffect(() => { if (!isResizing) return; @@ -26,6 +33,7 @@ export function useSidebarResize( const handlePointerUp = () => { setIsResizing(false); document.body.style.cursor = "default"; + if (storageKey) writePaneWidth(storageKey, widthRef.current); }; window.addEventListener("pointermove", handlePointerMove); @@ -38,7 +46,7 @@ export function useSidebarResize( window.removeEventListener("pointerup", handlePointerUp); document.body.style.cursor = "default"; }; - }, [isResizing, side, minWidth, maxWidth]); + }, [isResizing, side, minWidth, maxWidth, storageKey]); const startResizing = useCallback((e: React.PointerEvent) => { e.preventDefault(); diff --git a/apps/web/src/hooks/useWideLayout.ts b/apps/web/src/hooks/useWideLayout.ts new file mode 100644 index 0000000..115a33b --- /dev/null +++ b/apps/web/src/hooks/useWideLayout.ts @@ -0,0 +1,16 @@ +import { useEffect, useState } from "react"; + +/** Wide enough to keep the conversation readable beside a working pane. */ +const WIDE_LAYOUT_QUERY = "(min-width: 1024px)"; + +export function useWideLayout(): boolean { + const [wide, setWide] = useState(() => window.matchMedia(WIDE_LAYOUT_QUERY).matches); + useEffect(() => { + const media = window.matchMedia(WIDE_LAYOUT_QUERY); + const sync = () => setWide(media.matches); + sync(); + media.addEventListener("change", sync); + return () => media.removeEventListener("change", sync); + }, []); + return wide; +} diff --git a/apps/web/src/lib/agent-pointer.ts b/apps/web/src/lib/agent-pointer.ts index fb8619a..38090f1 100644 --- a/apps/web/src/lib/agent-pointer.ts +++ b/apps/web/src/lib/agent-pointer.ts @@ -3,12 +3,12 @@ import { ComputerPointerPayload, type ComputerPointerStep, } from "@rakazo/contracts"; +import { POINTER_EVENT_MAX_AGE_MS } from "@rakazo/core"; export type PointerPoint = { x: number; y: number }; export type PointerScreen = ComputerPointerPayload["screen"]; -/** Older announcements describe actions that already finished; replaying them would mislead. */ -export const POINTER_EVENT_MAX_AGE_MS = 30_000; +export { POINTER_EVENT_MAX_AGE_MS }; export const POINTER_IDLE_MS = 3_500; type Listener = (payload: ComputerPointerPayload) => void; diff --git a/apps/web/src/lib/csv-preview.test.ts b/apps/web/src/lib/csv-preview.test.ts new file mode 100644 index 0000000..3bd9ed4 --- /dev/null +++ b/apps/web/src/lib/csv-preview.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { parseCsvPreview } from "./csv-preview"; + +describe("parseCsvPreview", () => { + it("reads quoted fields, doubled quotes and CRLF rows", () => { + expect(parseCsvPreview('name,note\r\n"Smith, Ann","said ""hi"""\r\nBo,\n')).toEqual({ + rows: [ + ["name", "note"], + ["Smith, Ann", 'said "hi"'], + ["Bo", ""], + ], + truncated: false, + }); + }); + + it("keeps a final row without a trailing newline and skips blank lines", () => { + expect(parseCsvPreview("a,b\n\n1,2").rows).toEqual([ + ["a", "b"], + ["1", "2"], + ]); + }); + + it("keeps newlines inside quoted fields", () => { + expect(parseCsvPreview('a\n"line 1\nline 2"').rows).toEqual([["a"], ["line 1\nline 2"]]); + }); + + it("stops after the row limit", () => { + const result = parseCsvPreview("1\n2\n3\n4\n", 2); + expect(result).toEqual({ rows: [["1"], ["2"]], truncated: true }); + }); +}); diff --git a/apps/web/src/lib/csv-preview.ts b/apps/web/src/lib/csv-preview.ts new file mode 100644 index 0000000..3fac6a2 --- /dev/null +++ b/apps/web/src/lib/csv-preview.ts @@ -0,0 +1,43 @@ +/** Parses enough of RFC 4180 to show a CSV as a table: quoted fields, doubled quotes, CRLF. */ +export function parseCsvPreview( + text: string, + maxRows = 500, +): { rows: string[][]; truncated: boolean } { + const rows: string[][] = []; + let row: string[] = []; + let field = ""; + let quoted = false; + let index = 0; + const endRow = () => { + row.push(field); + field = ""; + if (row.length > 1 || row[0] !== "") rows.push(row); + row = []; + }; + while (index < text.length) { + const char = text[index]; + if (quoted) { + if (char === '"' && text[index + 1] === '"') { + field += '"'; + index += 2; + continue; + } + if (char === '"') quoted = false; + else field += char; + index += 1; + continue; + } + if (char === '"' && field === "") quoted = true; + else if (char === ",") { + row.push(field); + field = ""; + } else if (char === "\n" || char === "\r") { + endRow(); + if (char === "\r" && text[index + 1] === "\n") index += 1; + if (rows.length > maxRows) break; + } else field += char; + index += 1; + } + if (rows.length <= maxRows && (field !== "" || row.length > 0)) endRow(); + return { rows: rows.slice(0, maxRows), truncated: rows.length > maxRows }; +} diff --git a/apps/web/src/lib/workspace-pane-pref.test.ts b/apps/web/src/lib/workspace-pane-pref.test.ts new file mode 100644 index 0000000..e7265f6 --- /dev/null +++ b/apps/web/src/lib/workspace-pane-pref.test.ts @@ -0,0 +1,61 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + readPaneWidth, + readScreenPaneHabit, + writePaneWidth, + writeScreenPaneHabit, +} from "./workspace-pane-pref"; + +function stubStorage() { + const store = new Map(); + vi.stubGlobal("window", { + localStorage: { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => { + store.set(key, value); + }, + removeItem: (key: string) => { + store.delete(key); + }, + }, + }); + return store; +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("workspace pane preferences", () => { + it("keeps the learned screen habit per user and bot", () => { + const store = stubStorage(); + expect(readScreenPaneHabit("user-1", "bot-1")).toBe(0); + writeScreenPaneHabit("user-1", "bot-1", 2); + expect(readScreenPaneHabit("user-1", "bot-1")).toBe(2); + expect(readScreenPaneHabit("user-1", "bot-2")).toBe(0); + writeScreenPaneHabit("user-1", "bot-1", 0); + expect(store.size).toBe(0); + writeScreenPaneHabit(null, "bot-1", 3); + expect(store.size).toBe(0); + }); + + it("restores a stored pane width within its bounds", () => { + stubStorage(); + expect(readPaneWidth("width", 500, 320, 900)).toBe(500); + writePaneWidth("width", 640.4); + expect(readPaneWidth("width", 500, 320, 900)).toBe(640); + writePaneWidth("width", 2_000); + expect(readPaneWidth("width", 500, 320, 900)).toBe(900); + }); + + it("falls back when storage is unavailable", () => { + vi.stubGlobal("window", { + get localStorage(): Storage { + throw new Error("blocked"); + }, + }); + expect(readScreenPaneHabit("user-1", "bot-1")).toBe(0); + expect(readPaneWidth("width", 500, 320, 900)).toBe(500); + expect(() => writePaneWidth("width", 600)).not.toThrow(); + }); +}); diff --git a/apps/web/src/lib/workspace-pane-pref.ts b/apps/web/src/lib/workspace-pane-pref.ts new file mode 100644 index 0000000..279cb08 --- /dev/null +++ b/apps/web/src/lib/workspace-pane-pref.ts @@ -0,0 +1,52 @@ +import { parseScreenPaneHabit, type ScreenPaneHabit } from "@rakazo/core"; + +function screenHabitKey(userId: string | null | undefined, botId: string | null | undefined) { + if (!userId || !botId) return null; + return `rakazo:screen-pane:${userId}:${botId}`; +} + +export function readScreenPaneHabit( + userId: string | null | undefined, + botId: string | null | undefined, +): ScreenPaneHabit { + const key = screenHabitKey(userId, botId); + if (!key) return 0; + try { + return parseScreenPaneHabit(window.localStorage.getItem(key)); + } catch { + return 0; + } +} + +export function writeScreenPaneHabit( + userId: string | null | undefined, + botId: string | null | undefined, + habit: ScreenPaneHabit, +): void { + const key = screenHabitKey(userId, botId); + if (!key) return; + try { + if (habit === 0) window.localStorage.removeItem(key); + else window.localStorage.setItem(key, String(habit)); + } catch { + // Private mode or a full quota only loses the learned default. + } +} + +export function readPaneWidth(storageKey: string, fallback: number, min: number, max: number) { + try { + const stored = Number(window.localStorage.getItem(storageKey)); + if (Number.isFinite(stored) && stored > 0) return Math.max(min, Math.min(max, stored)); + } catch { + // Fall through to the default width. + } + return fallback; +} + +export function writePaneWidth(storageKey: string, width: number): void { + try { + window.localStorage.setItem(storageKey, String(Math.round(width))); + } catch { + // The width still applies for this session. + } +} diff --git a/apps/web/src/pages/Shell.tsx b/apps/web/src/pages/Shell.tsx index 8afe54f..5437789 100644 --- a/apps/web/src/pages/Shell.tsx +++ b/apps/web/src/pages/Shell.tsx @@ -49,11 +49,13 @@ import { normalizeCreateBotProfile, } from "@rakazo/contracts"; import { + applyWorkspaceFocus, attachmentsForThread, buildComposerMentionOptions, type ComposerMention, clampMentionHighlightIndex, cronFromPreset, + deriveWorkspaceFocus, groupBotsForSidebar, inferAttachmentMimeType, isActive, @@ -61,21 +63,27 @@ import { isRunTerminalEvent, isToolActivityBlock, latestAnswerableAskMessageId, + liveScreenRunId, matchVoiceByName, mentionChipKey, + type PaneState, parseVoiceChatCommand, projectMessageReactions, + recordScreenPaneChoice, reorderBotTo, resolveComposerSendPlan, resolveMentionPickerKey, runThreadSubscription, SLASH_ACTIONS, type SlashActionId, + screenPaneMode, searchHitThreadTarget, serializeComposerPrompt, speechFromBlocks, truncateSlashDescription, userVisibleMessages, + type WorkspaceArtifact, + type WorkspaceFocus, } from "@rakazo/core"; import { AvatarStyleProvider, @@ -156,6 +164,7 @@ import { import { createPortal } from "react-dom"; import { useNavigate, useParams, useSearchParams } from "react-router-dom"; import { ArtifactFileCard } from "../components/ArtifactFileCard"; +import { ArtifactPane } from "../components/ArtifactPane"; import { AskCard } from "../components/AskCard"; import { ActiveBotGlyph, CollaborationMarker } from "../components/ai/CollaborationMarker"; import { DelegatedTaskCard, SentMessageCard } from "../components/ai/DelegatedTaskCard"; @@ -180,6 +189,7 @@ import { TeachComputerOverlayControl } from "../components/teach/TeachComputerOv import { TeachRecordingChrome, TeachStopButton } from "../components/teach/TeachRecordingChrome"; import { VoiceAuroraRim } from "../components/voice/VoiceAurora"; import { useSidebarResize } from "../hooks/useSidebarResize"; +import { useWideLayout } from "../hooks/useWideLayout"; import { readActivityMode, writeActivityMode } from "../lib/activity-mode"; import { needsYouChipLabel, rosterActivitySubtitle, rosterStatusDot } from "../lib/agent-activity"; import { agentPointerFeed } from "../lib/agent-pointer"; @@ -249,6 +259,7 @@ import { transcriptMovedDown, } from "../lib/transcript-scroll"; import { speaker } from "../lib/tts"; +import { readScreenPaneHabit, writeScreenPaneHabit } from "../lib/workspace-pane-pref"; import { ActivityList } from "./ActivityList"; import type { ContextMenuPosition } from "./BotContextMenu"; import { CreateGroupForm, GroupSettings, memberName } from "./GroupPanel"; @@ -327,6 +338,7 @@ type Panel = | "create-group" | "group-settings" | "browser" + | "artifact" | null; type PendingAttachment = { @@ -351,6 +363,14 @@ const THREAD_SNAPSHOT_TIMEOUT_MS = 2_000; const VOICE_STATUS_REFRESH_TIMEOUT_MS = 10_000; const MOBILE_SIDEBAR_SWIPE_EDGE_PX = 32; const MOBILE_SIDEBAR_SWIPE_DISTANCE_PX = 56; +/** The screen lingers after the work so its final state and the bot's reply land first. */ +const AUTO_SCREEN_CLOSE_DELAY_MS = 4_000; +/** How long one pointer announcement counts as the bot working on its screen. */ +const SCREEN_ACTIVE_MS = 5_000; + +function defaultWorkspacePaneWidth(): number { + return Math.round(Math.max(420, Math.min(760, window.innerWidth * 0.42))); +} function threadSnapshotSignal(parent: AbortSignal): AbortSignal { return AbortSignal.any([parent, AbortSignal.timeout(THREAD_SNAPSHOT_TIMEOUT_MS)]); @@ -438,12 +458,24 @@ export function ShellPage() { width: leftSidebarWidth, isResizing: isResizingLeft, startResizing: startResizingLeft, - } = useSidebarResize(316, "left", 240, 450); + } = useSidebarResize(316, "left", 240, 450, "rakazo:left-sidebar-width"); const { width: rightSidebarWidth, isResizing: isResizingRight, startResizing: startResizingRight, - } = useSidebarResize(384, "right", 280, 500); + } = useSidebarResize(384, "right", 280, 500, "rakazo:side-panel-width"); + // The screen and presented files get a wider pane than settings: they are what the user watches. + const { + width: workspacePaneWidth, + isResizing: isResizingWorkspacePane, + startResizing: startResizingWorkspacePane, + } = useSidebarResize( + defaultWorkspacePaneWidth(), + "right", + 360, + 960, + "rakazo:workspace-pane-width", + ); const [heuteActionCount, setHeuteActionCount] = useState(0); const [archivedOpen, setArchivedOpen] = useState(false); @@ -458,6 +490,15 @@ export function ShellPage() { }, [userId]); const [panel, setPanel] = useState(null); + /** The pane the conversation's focus opened; cleared as soon as the user changes the pane. */ + const [autoPanel, setAutoPanel] = useState(null); + const [artifactPane, setArtifactPane] = useState(null); + /** Run that last pointed at the bot's screen; the live screen only follows real screen work. */ + const [screenRunId, setScreenRunId] = useState(null); + const [screenActive, setScreenActive] = useState(false); + const screenActiveTimer = useRef(undefined); + const [screenHabit, setScreenHabit] = useState(0); + const wideLayout = useWideLayout(); const [showCreateBotModal, setShowCreateBotModal] = useState(false); @@ -1458,6 +1499,16 @@ export function ShellPage() { void refreshComputerScreen(active.id).catch(() => undefined); } else if (event.type === "thread.computer") { agentPointerFeed.publish(event.botId, event.payload, event.createdAt); + const screenRun = liveScreenRunId(event); + if (screenRun) { + setScreenRunId(screenRun); + setScreenActive(true); + window.clearTimeout(screenActiveTimer.current); + screenActiveTimer.current = window.setTimeout( + () => setScreenActive(false), + SCREEN_ACTIVE_MS, + ); + } } }, }); @@ -2102,6 +2153,83 @@ export function ShellPage() { () => (inGroup ? { groupId: groupId ?? "" } : { botId: active?.id ?? "" }), [active?.id, groupId, inGroup], ); + const workspaceFocus = useMemo( + () => + deriveWorkspaceFocus( + activeSnapshot ? { ...activeSnapshot, computer: inGroup ? null : computer } : null, + { screenRunId, screenActive, screenMode: screenPaneMode(screenHabit) }, + ), + [activeSnapshot, computer, inGroup, screenActive, screenHabit, screenRunId], + ); + const workspaceFocusRef = useRef<{ threadId: string; focus: WorkspaceFocus } | null>(null); + const autoPanelRef = useRef(autoPanel); + autoPanelRef.current = autoPanel; + const autoCloseTimer = useRef(undefined); + useEffect(() => () => window.clearTimeout(autoCloseTimer.current), []); + // The pane follows the bot's work on focus edges only, so it never jumps while nothing changed. + useEffect(() => { + const threadId = activeSnapshot?.threadId; + const sameThread = Boolean(threadId) && workspaceFocusRef.current?.threadId === threadId; + // New work or another conversation cancels a pending close; an unrelated edge does not. + if (!sameThread || workspaceFocus.pane !== null) window.clearTimeout(autoCloseTimer.current); + if (!threadId) { + workspaceFocusRef.current = null; + return; + } + const previous = sameThread ? (workspaceFocusRef.current?.focus ?? null) : null; + workspaceFocusRef.current = { threadId, focus: workspaceFocus }; + // Narrow windows keep the conversation full width; a pane would cover it. + if (!wideLayout) return; + const paneState: PaneState = { + open: + panel === "computer" + ? "screen" + : panel === "artifact" + ? "artifact" + : panel + ? "other" + : null, + auto: panel !== null && panel === autoPanel, + }; + const next = applyWorkspaceFocus(previous, workspaceFocus, paneState); + if (!next || next.open === "other") return; + if (next.open === null) { + const closing = autoPanelRef.current; + autoCloseTimer.current = window.setTimeout(() => { + if (closing === null || autoPanelRef.current !== closing) return; + setPanel(null); + setAutoPanel(null); + }, AUTO_SCREEN_CLOSE_DELAY_MS); + return; + } + if (next.open === "artifact") setArtifactPane(workspaceFocus.artifact); + const nextPanel: Panel = next.open === "screen" ? "computer" : next.open; + setPanel(nextPanel); + setAutoPanel(next.auto ? nextPanel : null); + }, [workspaceFocus.key, activeSnapshot?.threadId, wideLayout]); + const learnScreenPane = useCallback( + (choice: "opened" | "dismissed") => { + const botId = activeBotId.current; + if (!botId) return; + const next = recordScreenPaneChoice(screenHabit, choice); + setScreenHabit(next); + writeScreenPaneHabit(userId, botId, next); + }, + [screenHabit, userId], + ); + useEffect(() => { + if (autoPanel === null || panel === autoPanel) return; + // Closing the live screen while the bot still works on it teaches this bot's default. + const focus = workspaceFocusRef.current?.focus; + if (autoPanel === "computer" && panel === null && focus?.pane === "screen" && !focus.takeover) { + learnScreenPane("dismissed"); + } + setAutoPanel(null); + }, [panel]); + const openArtifactBeside = useCallback((artifact: WorkspaceArtifact) => { + setArtifactPane(artifact); + setPanel("artifact"); + }, []); const transcriptMembers = activeSnapshot?.members ?? activeGroup?.members; // Bots of every space, so a group message from a bot that is not a member still has a speaker. const knownBots = useMemo(() => { @@ -3110,8 +3238,22 @@ export function ShellPage() { setComputerOpen(false); setComputerError(null); setComputerErrorFromScreen(false); + setScreenRunId(null); + setScreenActive(false); + window.clearTimeout(screenActiveTimer.current); }, [active?.id]); + useEffect(() => { + setScreenHabit(readScreenPaneHabit(userId, active?.id)); + }, [active?.id, userId]); + + // A presented file belongs to its conversation, and a new conversation starts without auto panes. + useEffect(() => { + setPanel((current) => (current === "artifact" ? null : current)); + setArtifactPane(null); + setAutoPanel(null); + }, [active?.id, groupId]); + useEffect(() => { if (focusPromptBotIdRef.current && focusPromptBotIdRef.current !== active?.id) { cancelFocusPrompt(); @@ -3228,6 +3370,9 @@ export function ShellPage() { } const embeddedScreenUrl = embeddableScreenUrl(screenUrl); + // Only what the work opened gets the wide pane; settings, routines and a hand-opened screen keep + // one width so switching between them never jumps. + const workspacePaneOpen = panel === "artifact" || (panel === "computer" && autoPanel === panel); const hasControl = userHoldsComputerControl(computer, active?.id); const hideScreenLoadError = computerErrorFromScreen && Boolean(embeddedScreenUrl); const computerScreenError = @@ -4410,6 +4555,7 @@ export function ShellPage() { title={t`Agent computer`} onClick={() => { const next = panel === "computer" ? null : "computer"; + if (next === "computer" && transcriptRunning) learnScreenPane("opened"); setPanel(next); if (next === "computer" && active) { // Refresh run/computer so Take control isn't stuck on a stale busyBotName. @@ -4500,6 +4646,7 @@ export function ShellPage() { }} scrollRef={messageScroll} artifactTarget={transcriptArtifactTarget} + onOpenArtifact={wideLayout ? openArtifactBeside : undefined} messages={transcriptMessages} olderCursor={activeSnapshot?.olderCursor ?? null} loadingOlder={loadingOlder} @@ -4605,6 +4752,7 @@ export function ShellPage() {