From 979034a8a64b6bb88aa4ca95f9d7d6a3a2d8bb79 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:50:02 +0000 Subject: [PATCH 1/6] feat(web): play video and audio inline and stream uploads and downloads Media sent in a conversation had nowhere to go: the composer capped every file at 10 MiB and pushed it through a base64 RPC, video and audio blocks did not render, and downloads decoded whole files in the page. - Video and audio blocks play inline through MediaPlayer, which loads nothing until the player is near the viewport, sizes the frame from the block's dimensions (vertical clips stay within the column) and uses the poster when one exists. Files the browser cannot play fall back to their name with a download action. - Images load straight from the artifact route; the lightbox gains a download action. - File cards, the Markdown preview and downloads read from the artifact route instead of decoding base64. - The composer accepts each kind up to its cap and names the computed limit when it skips a file; video, audio and anything above the inline cap upload as a raw stream, small files keep the inline RPC. - MediaPlayer and artifactUrl are what the media card builds on. Tests: MediaPlayer laziness, sizing and fallback, the upload helper, and an E2E that uploads a tiny MP4 fixture (VP9, since Playwright's Chromium has no H.264 decoder), plays it from the artifact route and captures media-video-block, plus the image lightbox download. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- apps/web/e2e/fixtures/media-clip.mp4 | Bin 0 -> 3117 bytes apps/web/e2e/media-pipeline.spec.ts | 79 ++++++++++ apps/web/src/components/ArtifactFileCard.tsx | 80 ++-------- .../src/components/media/MediaPlayer.test.tsx | 84 ++++++++++ apps/web/src/components/media/MediaPlayer.tsx | 121 ++++++++++++++ apps/web/src/lib/artifact-open.ts | 43 ++--- apps/web/src/lib/artifact-upload.test.ts | 83 ++++++++++ apps/web/src/lib/artifact-upload.ts | 68 ++++++++ apps/web/src/lib/artifact-url.ts | 32 +--- apps/web/src/lib/format-bytes.ts | 12 ++ apps/web/src/pages/Shell.tsx | 57 +++---- apps/web/src/pages/shell/message-cards.tsx | 148 +++++++----------- 12 files changed, 562 insertions(+), 245 deletions(-) create mode 100644 apps/web/e2e/fixtures/media-clip.mp4 create mode 100644 apps/web/e2e/media-pipeline.spec.ts create mode 100644 apps/web/src/components/media/MediaPlayer.test.tsx create mode 100644 apps/web/src/components/media/MediaPlayer.tsx create mode 100644 apps/web/src/lib/artifact-upload.test.ts create mode 100644 apps/web/src/lib/artifact-upload.ts create mode 100644 apps/web/src/lib/format-bytes.ts diff --git a/apps/web/e2e/fixtures/media-clip.mp4 b/apps/web/e2e/fixtures/media-clip.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61abe46b742c15b4dd4ec3b36b137c7967f96e41 GIT binary patch literal 3117 zcmZuz3p|ti8-JPPQbKZwj*$>TQY`l=%bjvLQQ2&hWwWuFTdrZ8Sc*hhbEz~%)JWQ2 zp$?&hWJm6|9EnKfx}A5bbNcr=&*y#K@AG@^@AJH$?F9fp*`E?Y!jXvt01yTQ2(lr8 zWV9OqAi@M9kqQ6+o-Mf@k6(R1gM_75E5I2)H1BP>`b%){_3% zwmRmkO&Jq{75E59VBZiLf`@*Ug^b7)KRhVkBvZ)W7b5VAPVTmoL?1I;A0;G zMF9&)Xp4Sb%`O$18(6^jFnCI?#Vj z)&gyj-)sT^^`cimj0Kqvatz3Sf(+(G6G0RZm=~f4qIoEWf&y_b0lS6*o9d5C0~Z~F zB9R1h{fESGU*8Nd{}3#=QStQz42qIqcLok@{vf$e&qQ5W;WGev9NV?Xel)k*#w$xN zSn+abty-`SyBF2p_&VG6ZPz@hxbaYpluES%?a0#<513jySIzWnosBt9&a^zw^x|W! z79#T+XBI`1R`eSG?IBA1!tK6*$h3r>1N~wrqQ*{#4MsLyaNf8l(Yhi2c{AMjeCH$@ zTimUsF-2}MQ<2dw-+DMFijk3WfQ@jp@2!nn4rYAOt-5pl<=j$InO$zCr(F57&fi= z#f9g}Fen8?tNC*c{fQnBv#aJW<8aNF(nqFaVH*X1tX3D)@RoF0VtY?L{vpikjF5N>q7uQQH3 z^Lb}T={*=Qu~Xc;bz^dntqSV5+%d?wj z?^Kzi2eaU#_H$D=i#*I5yBNW*d2g)zJ!7uQ9oNVYiaf8q)mC=P(;F*_h1N4;;_>%= zIgWLvn!wp3BiM+)OgL;2Tj$Dz7slBAdbHWvcd{ai2Gm0_$+PUd+D`BDg}e@~>7f1i zBa5t$keM$RjGCd=udpl1cbOYer4XS#bVrgs&gJC6nc5R`fnPY%jt*7klW!KWGwM0? z5oLRW=>?XP?H^+k^h(+_3ws7D2pPQobO@IVyB2wobpYy09e6h30>64_IFD`kUSji> z*x$VtuE)A&nl5hXR_Y=2tUh=&l)V%83TvF0%2Ru=nkF~HGlcCIrQ29cb?}qZmDquN z4Jo-5`<8=QvFfep{(BIiX>kLyz1gYRL%eOBzSeru)hktTx!jKnP#KQ1Lx1W(fEu3t zeAIm|PEq;%B&6ZD#37yzxg)Mr%sW*^TX{I|I4D(N9?Ta#S&Sgi&Pr4PhaGpQz zh1TSFc}Dj&Y|xYsPy6IY^`7qzi9c2Y2w!S9i#Lr`d?&fD#PUWaF4t-1O2FMn~%y+A;y*#NOYT-j&bx@pKtopo2P zq`Lzxj!yRRDbb^GdyT`5LfvbF<_gYvN{k}9nG);?9J<0N9&7TfGjfNo(g=4n=uv6} zVkzep5!qBGs#R)z=1p+GoWAd>ZD`7ihce_M3zJ`qgC7RFd}()g9iFPa)@u1e{oPq= zoa0gW-4aMV+?ES-E6wxmc2H>=em^GiAb#sJxOPLg^pSShAAMP>@Kwjts>fVR7|{iy zg<^KrhW(*w$1{CB_t2Bk#})3x-*E^~SuXM$nbVEX&!;7)x6^BzOywV>meypO2(#N< zlL>X^K(kIQ#cYasnOlYYuQVqs zE9%`cJN!+yjahf0>6Y}3YHB^_R7CE|GSNreUP%1{b(2rEOblXbO!vP#-%M|R zUS5fxcXLU)EIEEm{BnHg-O26cuSHi^l%Y17vW!cCG_Pa?b91=PCrA=!I0sL+=;)_8 zhH7&qCu?_0?=*Ri-|9VVMMakEA*&Bi{F9PRBW%X*?M5WjRxYvB=N@W4Dn7H`vhH0A zvgI+53|L)Ijynnfn_}`P!L1%sNL$bK^Z5Yay;td6CA>_rMbSOX_uUTCKH6;Qkfl^B z>-ka6z;$Pcuu_Jl|AjhtlebjGI3`3dqpKk2oQ=rXEz@VQuaw$KOGF`icq7gpJ*eEc z6%RE9hB&TZ=43|S$SGgQe#f3JYxH8sge2a<3WvY zwl+FBgu(HpgvR1%{=8%64RvH~(ZoS71qRlVJx!r8RowjfSPk!{0ZqG>kQaB|ch{sT zJD6_{<~Fa7tIS%klPDyYd|-b#cVACO__nRp~o%9nw}@3vz+Z% zG~Zwv-V*5d?=T=p!2nJgI}wk5(KT`Qv^XywxX!b?hgyTn_!GGVS1|CfC4Q2tNZ7=8 z_pXQuF*A&QYn*m;L;<2tE_L1&S)_EdOi+_hJ`PrYSY)ZLiTGJpsgfE?T}{F0rF^o+2?>GPvnL4bye* z>AugbuB9aYQ=Z-VZE#wBZldDQQ{4|u+Ce=7b?4`) fLFK^;r>9NC+;71~6wX{VI>-D&eYuQ5P<8zq^f}lX literal 0 HcmV?d00001 diff --git a/apps/web/e2e/media-pipeline.spec.ts b/apps/web/e2e/media-pipeline.spec.ts new file mode 100644 index 0000000..dc712cc --- /dev/null +++ b/apps/web/e2e/media-pipeline.spec.ts @@ -0,0 +1,79 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "@playwright/test"; +import { captureScreenshot, completeOnboarding, signup } from "./helpers"; + +test.describe.configure({ mode: "serial" }); + +// VP9 in MP4: Playwright's Chromium has no H.264 decoder, real browsers play both. +const clip = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "media-clip.mp4"); +const pixel = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", +); + +test("an uploaded video streams from the artifact route and plays inline", async ({ + page, +}, testInfo) => { + const stamp = Date.now(); + await signup(page, `media-video-${stamp}@rakazo.test`, "password12", "Media Video"); + await completeOnboarding(page); + + const upload = page.waitForResponse( + (response) => + response.request().method() === "POST" && /\/api\/artifacts\?/.test(response.url()), + ); + await page.locator('input[type="file"]').setInputFiles(clip); + await expect(page.getByRole("button", { name: "Remove media-clip.mp4" })).toBeVisible(); + await page.getByRole("combobox", { name: /Message/ }).fill("here is the clip"); + await page.keyboard.press("Enter"); + expect((await upload).status()).toBe(201); + + const video = page.getByTestId("media-video").locator("video"); + await expect(video).toBeVisible({ timeout: 30_000 }); + await expect(video).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/); + await expect + .poll(() => video.evaluate((element: HTMLVideoElement) => element.readyState), { + timeout: 15_000, + }) + .toBeGreaterThanOrEqual(1); + expect(await video.evaluate((element: HTMLVideoElement) => element.duration)).toBeCloseTo(1, 0); + + await video.evaluate(async (element: HTMLVideoElement) => { + element.muted = true; + await element.play(); + }); + await expect + .poll(() => video.evaluate((element: HTMLVideoElement) => element.currentTime)) + .toBeGreaterThan(0); + await video.evaluate((element: HTMLVideoElement) => { + element.pause(); + element.currentTime = 0.5; + element.scrollIntoView({ block: "center" }); + }); + await captureScreenshot(page, testInfo, "media-video-block"); +}); + +test("the image lightbox downloads the original from the artifact route", async ({ page }) => { + const stamp = Date.now(); + await signup(page, `media-image-${stamp}@rakazo.test`, "password12", "Media Image"); + await completeOnboarding(page); + + await page + .locator('input[type="file"]') + .setInputFiles({ name: "pixel.png", mimeType: "image/png", buffer: pixel }); + await page.getByRole("combobox", { name: /Message/ }).fill("a pixel"); + await page.keyboard.press("Enter"); + + const thumbnail = page.getByRole("img", { name: "pixel.png" }); + await expect(thumbnail).toBeVisible({ timeout: 30_000 }); + await expect(thumbnail).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/); + await thumbnail.click(); + + const dialog = page.getByRole("dialog", { name: "pixel.png" }); + const download = dialog.getByRole("link", { name: "Download pixel.png" }); + await expect(download).toHaveAttribute("href", /\/api\/artifacts\/[^/?]+\?download=1$/); + const saved = page.waitForEvent("download"); + await download.click(); + expect((await saved).suggestedFilename()).toBe("pixel.png"); +}); diff --git a/apps/web/src/components/ArtifactFileCard.tsx b/apps/web/src/components/ArtifactFileCard.tsx index b36eb65..41c11c1 100644 --- a/apps/web/src/components/ArtifactFileCard.tsx +++ b/apps/web/src/components/ArtifactFileCard.tsx @@ -1,19 +1,12 @@ -import { i18n } from "@lingui/core"; -import { t } from "@lingui/core/macro"; import { Trans, useLingui } from "@lingui/react/macro"; import { ChatMarkdown } from "@rakazo/chat-ui/web"; import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web"; import { Download, FileText, X } from "lucide-react"; import { type RefObject, useEffect, useRef, useState } from "react"; -import { - type ArtifactTarget, - downloadArtifact, - downloadArtifactBytes, - fetchArtifactBytes, -} from "../lib/artifact-open"; +import { downloadArtifact, fetchArtifactBytes } from "../lib/artifact-open"; +import { formatBytes } from "../lib/format-bytes"; type ArtifactFileCardProps = { - target: ArtifactTarget; artifactId: string; name: string; mimeType: string; @@ -26,23 +19,14 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) { const previewButton = useRef(null); const closeButton = useRef(null); const [previewOpen, setPreviewOpen] = useState(false); - const [downloadError, setDownloadError] = useState(null); - - async function startDownload() { - setDownloadError(null); - try { - await downloadArtifact(props.target, props.artifactId, props.name, props.mimeType); - } catch { - setDownloadError(t`Could not download ${props.name}. Try again.`); - } - } + const startDownload = () => downloadArtifact(props.artifactId, props.name); if (!markdown) { return (
- {downloadError ? : null} ); } @@ -80,13 +63,12 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) { type="button" aria-label={t`Download ${props.name}`} title={t`Download ${props.name}`} - onClick={() => void startDownload()} + onClick={startDownload} className="grid w-14 shrink-0 place-items-center border-l border-border text-muted-foreground hover:bg-accent hover:text-foreground" > - {downloadError ? : null} }) { const { t } = useLingui(); - const [downloadError, setDownloadError] = useState(null); const [state, setState] = useState< | { status: "loading" } - | { status: "ready"; bytes: Uint8Array; markdown: string } + | { status: "ready"; markdown: string } | { status: "error"; message: string } >({ status: "loading" }); - const targetBotId = "botId" in target ? target.botId : undefined; - const targetGroupId = "groupId" in target ? target.groupId : undefined; useEffect(() => { let cancelled = false; - const artifactTarget: ArtifactTarget = - targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! }; - void fetchArtifactBytes(artifactTarget, artifactId) + void fetchArtifactBytes(artifactId) .then((bytes) => { if (cancelled) return; try { const markdown = new TextDecoder("utf-8", { fatal: true }).decode(bytes); - setState({ status: "ready", bytes, markdown }); + setState({ status: "ready", markdown }); } catch { setState({ status: "error", message: t`This file is not valid UTF-8 Markdown.` }); } @@ -143,7 +118,7 @@ function MarkdownPreview({ return () => { cancelled = true; }; - }, [artifactId, targetBotId, targetGroupId, t]); + }, [artifactId, t]); return ( <> @@ -157,17 +132,7 @@ function MarkdownPreview({ className="rounded-full text-muted-foreground" aria-label={t`Download ${name}`} title={t`Download ${name}`} - onClick={() => - void (async () => { - setDownloadError(null); - try { - if (state.status === "ready") downloadArtifactBytes(name, mimeType, state.bytes); - else await downloadArtifact(target, artifactId, name, mimeType); - } catch { - setDownloadError(t`Could not download ${name}. Try again.`); - } - })() - } + onClick={() => downloadArtifact(artifactId, name)} > @@ -181,11 +146,6 @@ function MarkdownPreview({ - {downloadError ? ( -
- -
- ) : null}
{state.status === "loading" ? ( @@ -204,23 +164,3 @@ function MarkdownPreview({ ); } - -function DownloadError({ message }: { message: string }) { - return ( -
- {message} -
- ); -} - -function formatBytes(size: number) { - const locale = i18n.locale || "en"; - if (size < 1024) return t`${size} B`; - const format = (value: number) => - new Intl.NumberFormat(locale, { - minimumFractionDigits: 1, - maximumFractionDigits: 1, - }).format(value); - if (size < 1024 * 1024) return t`${format(size / 1024)} KB`; - return t`${format(size / (1024 * 1024))} MB`; -} diff --git a/apps/web/src/components/media/MediaPlayer.test.tsx b/apps/web/src/components/media/MediaPlayer.test.tsx new file mode 100644 index 0000000..0be2403 --- /dev/null +++ b/apps/web/src/components/media/MediaPlayer.test.tsx @@ -0,0 +1,84 @@ +// @vitest-environment jsdom + +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@lingui/react/macro", () => { + const t = (parts: TemplateStringsArray, ...values: unknown[]) => + parts.reduce( + (text, part, index) => text + part + (index < values.length ? values[index] : ""), + "", + ); + return { useLingui: () => ({ t }) }; +}); + +import { MediaPlayer } from "./MediaPlayer"; + +let container: HTMLDivElement; +let root: Root; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +describe("MediaPlayer", () => { + it("loads nothing before it is near the viewport and fits vertical video to the column", () => { + const html = renderToStaticMarkup( + , + ); + expect(html).toContain('aria-label="clip.mp4"'); + expect(html).not.toContain("src="); + expect(html).toContain("width:270px"); + expect(html).toContain("aspect-ratio:270 / 480"); + }); + + it("streams video with its poster from the artifact route once visible", async () => { + vi.stubGlobal( + "IntersectionObserver", + class { + constructor(private readonly callback: IntersectionObserverCallback) {} + observe() { + this.callback([{ isIntersecting: true } as IntersectionObserverEntry], this as never); + } + disconnect() {} + }, + ); + await act(async () => + root.render( + , + ), + ); + const video = container.querySelector("video")!; + expect(video.getAttribute("src")).toBe("/api/artifacts/art-1"); + expect(video.getAttribute("poster")).toBe("/api/artifacts/art-2"); + expect(video.getAttribute("preload")).toBe("metadata"); + expect(video.hasAttribute("controls")).toBe(true); + }); + + it("falls back to a download link when the browser cannot play the file", async () => { + await act(async () => + root.render(), + ); + const audio = container.querySelector("audio")!; + expect(audio.getAttribute("src")).toBe("/api/artifacts/art-3"); + await act(async () => { + audio.dispatchEvent(new Event("error")); + }); + expect(container.querySelector("audio")).toBeNull(); + const link = container.querySelector("a")!; + expect(link.getAttribute("href")).toBe("/api/artifacts/art-3?download=1"); + expect(link.getAttribute("download")).toBe("memo.ogg"); + expect(link.getAttribute("aria-label")).toBe("Download memo.ogg"); + }); +}); diff --git a/apps/web/src/components/media/MediaPlayer.tsx b/apps/web/src/components/media/MediaPlayer.tsx new file mode 100644 index 0000000..d793ba4 --- /dev/null +++ b/apps/web/src/components/media/MediaPlayer.tsx @@ -0,0 +1,121 @@ +import { useLingui } from "@lingui/react/macro"; +import { buttonVariants } from "@rakazo/ui-web"; +import { Download } from "lucide-react"; +import { type RefObject, useEffect, useRef, useState } from "react"; +import { artifactUrl } from "../../lib/artifact-url"; + +export type MediaPlayerProps = { + artifactId: string; + kind: "video" | "audio"; + name: string; + posterArtifactId?: string; + width?: number; + height?: number; +}; + +const MAX_VIDEO_WIDTH = 420; +const MAX_VIDEO_HEIGHT = 480; + +/** Plays a video or audio artifact in place; loads nothing until it is near the viewport. */ +export function MediaPlayer({ + artifactId, + kind, + name, + posterArtifactId, + width, + height, +}: MediaPlayerProps) { + const container = useRef(null); + const near = useNearViewport(container); + const [failed, setFailed] = useState(false); + const src = near ? artifactUrl(artifactId) : undefined; + + if (failed) return ; + + if (kind === "audio") { + return ( +
+ {/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */} +
+ ); + } + + const box = videoBox(width, height); + return ( +
+ {/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */} +
+ ); +} + +/** Fits the frame into the chat column, defaulting to 16:9 until the size is known. */ +function videoBox(width?: number, height?: number) { + const w = width && width > 0 ? width : 16; + const h = height && height > 0 ? height : 9; + const scale = Math.min(MAX_VIDEO_WIDTH / w, MAX_VIDEO_HEIGHT / h); + return { width: Math.round(w * scale), height: Math.round(h * scale) }; +} + +function MediaFallback({ artifactId, name }: { artifactId: string; name: string }) { + const { t } = useLingui(); + return ( +
+ {name} + + + +
+ ); +} + +function useNearViewport(ref: RefObject) { + const [near, setNear] = useState(false); + useEffect(() => { + const element = ref.current; + if (!element || typeof IntersectionObserver === "undefined") { + setNear(true); + return; + } + const observer = new IntersectionObserver( + (entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + setNear(true); + observer.disconnect(); + } + }, + { rootMargin: "320px" }, + ); + observer.observe(element); + return () => observer.disconnect(); + }, [ref]); + return near; +} diff --git a/apps/web/src/lib/artifact-open.ts b/apps/web/src/lib/artifact-open.ts index a1ed8eb..e832642 100644 --- a/apps/web/src/lib/artifact-open.ts +++ b/apps/web/src/lib/artifact-open.ts @@ -1,40 +1,29 @@ -import { rpc } from "./rpc"; +import { t } from "@lingui/core/macro"; +import { artifactUrl } from "./artifact-url"; export type ArtifactTarget = { botId: string } | { groupId: string }; -export function decodeArtifactBase64(contentBase64: string): Uint8Array { - const binary = atob(contentBase64); - const bytes = new Uint8Array(binary.length); - for (let index = 0; index < binary.length; index += 1) { - bytes[index] = binary.charCodeAt(index); - } - return bytes; -} - -export async function fetchArtifactBytes( - target: ArtifactTarget, - artifactId: string, -): Promise { - const artifact = await rpc.artifacts.get({ ...target, artifactId }); - return decodeArtifactBase64(artifact.contentBase64); +export async function fetchArtifactBytes(artifactId: string): Promise { + const response = await fetch(artifactUrl(artifactId), { credentials: "include" }); + if (!response.ok) throw new Error(t`Could not load this file.`); + return new Uint8Array(await response.arrayBuffer()); } export function downloadArtifactBytes(name: string, mimeType: string, bytes: Uint8Array): void { const blob = new Blob([new Uint8Array(bytes)], { type: mimeType }); const url = URL.createObjectURL(blob); + saveUrl(url, name); + window.setTimeout(() => URL.revokeObjectURL(url), 60_000); +} + +/** Lets the browser stream the download from the artifact route instead of buffering it. */ +export function downloadArtifact(artifactId: string, name: string): void { + saveUrl(artifactUrl(artifactId, { download: true }), name); +} + +function saveUrl(url: string, name: string) { const anchor = document.createElement("a"); anchor.href = url; anchor.download = name; anchor.click(); - window.setTimeout(() => URL.revokeObjectURL(url), 60_000); -} - -export async function downloadArtifact( - target: ArtifactTarget, - artifactId: string, - name: string, - mimeType: string, -): Promise { - const bytes = await fetchArtifactBytes(target, artifactId); - downloadArtifactBytes(name, mimeType, bytes); } diff --git a/apps/web/src/lib/artifact-upload.test.ts b/apps/web/src/lib/artifact-upload.test.ts new file mode 100644 index 0000000..5c0440b --- /dev/null +++ b/apps/web/src/lib/artifact-upload.test.ts @@ -0,0 +1,83 @@ +import { ATTACHMENT_MAX_BYTES } from "@rakazo/contracts"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@lingui/core/macro", () => ({ + t: (parts: TemplateStringsArray, ...values: unknown[]) => + parts.reduce( + (text, part, index) => text + part + (index < values.length ? values[index] : ""), + "", + ), +})); +const create = vi.hoisted(() => vi.fn()); +vi.mock("./rpc", () => ({ + rpc: { artifacts: { create } }, + withSpaceHeaders: (init: HeadersInit) => new Headers(init), +})); + +import { + attachmentMaxBytes, + uploadArtifactFile, + uploadAttachment, + usesStreamingUpload, +} from "./artifact-upload"; + +const artifact = { + id: "art-1", + botId: "bot-1", + groupId: null, + runId: null, + name: "clip.mp4", + mimeType: "video/mp4", + size: 4, + createdAt: "2026-10-03T12:00:00.000Z", +}; + +afterEach(() => { + vi.unstubAllGlobals(); + create.mockReset(); +}); + +describe("artifact uploads", () => { + it("streams media and large files and keeps small files on the inline RPC", () => { + expect(usesStreamingUpload(10, "video/mp4")).toBe(true); + expect(usesStreamingUpload(10, "audio/mpeg")).toBe(true); + expect(usesStreamingUpload(10, "image/png")).toBe(false); + expect(usesStreamingUpload(ATTACHMENT_MAX_BYTES + 1, "image/png")).toBe(true); + expect(attachmentMaxBytes("video/mp4")).toBe(500 * 1024 * 1024); + expect(attachmentMaxBytes("text/plain")).toBe(25 * 1024 * 1024); + }); + + it("posts the raw file with its target, name and type", async () => { + const fetch = vi.fn(async () => Response.json(artifact, { status: 201 })); + vi.stubGlobal("fetch", fetch); + const file = new File(["data"], "clip.mp4", { type: "video/mp4" }); + + await expect(uploadAttachment({ groupId: "group-1" }, file, "video/mp4")).resolves.toEqual( + artifact, + ); + const [url, init] = fetch.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe("/api/artifacts?groupId=group-1&name=clip.mp4"); + expect(init.method).toBe("POST"); + expect(init.credentials).toBe("include"); + expect(new Headers(init.headers).get("content-type")).toBe("video/mp4"); + expect(init.body).toBe(file); + expect(create).not.toHaveBeenCalled(); + }); + + it("surfaces the server's reason, or a generic one", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ error: "File is too large." }, { status: 413 })), + ); + await expect( + uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"), + ).rejects.toThrow("File is too large."); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("bad gateway", { status: 502 })), + ); + await expect( + uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"), + ).rejects.toThrow("Could not upload big.mp4. Try again."); + }); +}); diff --git a/apps/web/src/lib/artifact-upload.ts b/apps/web/src/lib/artifact-upload.ts new file mode 100644 index 0000000..12d11e4 --- /dev/null +++ b/apps/web/src/lib/artifact-upload.ts @@ -0,0 +1,68 @@ +import { t } from "@lingui/core/macro"; +import { + ARTIFACT_MAX_BYTES_BY_KIND, + ArtifactSchema, + ATTACHMENT_MAX_BYTES, + attachmentKindOf, +} from "@rakazo/contracts"; +import type { ArtifactTarget } from "./artifact-open"; +import { rpc, withSpaceHeaders } from "./rpc"; + +type UploadedArtifact = ReturnType; + +/** Largest file of this type the composer accepts. */ +export function attachmentMaxBytes(mimeType: string): number { + return ARTIFACT_MAX_BYTES_BY_KIND[attachmentKindOf(mimeType)]; +} + +/** Media and anything above the inline RPC cap upload as a raw stream instead of base64. */ +export function usesStreamingUpload(size: number, mimeType: string): boolean { + const kind = attachmentKindOf(mimeType); + return size > ATTACHMENT_MAX_BYTES || kind === "video" || kind === "audio"; +} + +export async function uploadAttachment( + target: ArtifactTarget, + file: File, + mimeType: string, +): Promise { + if (usesStreamingUpload(file.size, mimeType)) { + return uploadArtifactFile(target, file, file.name, mimeType); + } + const contentBase64 = await readFileAsBase64(file); + return rpc.artifacts.create({ ...target, name: file.name, mimeType, contentBase64 }); +} + +/** Uploads a file as a raw streamed body to the artifact route. */ +export async function uploadArtifactFile( + target: ArtifactTarget, + file: Blob, + name: string, + mimeType: string, +): Promise { + const query = new URLSearchParams({ ...target, name }); + const response = await fetch(`/api/artifacts?${query}`, { + method: "POST", + headers: withSpaceHeaders({ "content-type": mimeType }), + credentials: "include", + body: file, + }); + const body: unknown = await response.json().catch(() => null); + const artifact = ArtifactSchema.safeParse(body); + if (response.ok && artifact.success) return artifact.data; + const error = (body as { error?: unknown } | null)?.error; + throw new Error(typeof error === "string" ? error : t`Could not upload ${name}. Try again.`); +} + +function readFileAsBase64(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const result = typeof reader.result === "string" ? reader.result : ""; + const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result; + resolve(base64); + }; + reader.onerror = () => reject(reader.error ?? new Error("Failed to read file")); + reader.readAsDataURL(file); + }); +} diff --git a/apps/web/src/lib/artifact-url.ts b/apps/web/src/lib/artifact-url.ts index a87f4d4..5dff453 100644 --- a/apps/web/src/lib/artifact-url.ts +++ b/apps/web/src/lib/artifact-url.ts @@ -1,38 +1,8 @@ -import { t } from "@lingui/core/macro"; -import { ArtifactSchema } from "@rakazo/contracts"; -import type { ArtifactTarget } from "./artifact-open"; -import { withSpaceHeaders } from "./rpc"; - -const ARTIFACT_PATH = "/api/artifacts"; - /** * Same-origin URL that streams an artifact with Range support, for `
+ ); + } + if (block.kind === "video" || block.kind === "audio") { + return ( +
+
); @@ -7856,7 +7873,6 @@ const MessageView = memo(function MessageView({ className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`} > { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const result = typeof reader.result === "string" ? reader.result : ""; - const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result; - resolve(base64); - }; - reader.onerror = () => reject(reader.error ?? new Error("Failed to read file")); - reader.readAsDataURL(file); - }); -} diff --git a/apps/web/src/pages/shell/message-cards.tsx b/apps/web/src/pages/shell/message-cards.tsx index 14131d6..294bbd4 100644 --- a/apps/web/src/pages/shell/message-cards.tsx +++ b/apps/web/src/pages/shell/message-cards.tsx @@ -1,11 +1,18 @@ import { Trans, useLingui } from "@lingui/react/macro"; import type { MessageBlock } from "@rakazo/contracts"; import { abortableDelay } from "@rakazo/core"; -import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web"; -import { X } from "lucide-react"; +import { + Button, + buttonVariants, + Dialog, + DialogClose, + DialogContent, + DialogTitle, +} from "@rakazo/ui-web"; +import { Download, X } from "lucide-react"; import { useEffect, useRef, useState } from "react"; import { BuiCard, SuccessPop } from "../../components/ai/primitives"; -import { type ArtifactTarget, decodeArtifactBase64 } from "../../lib/artifact-open"; +import { artifactUrl } from "../../lib/artifact-url"; import { chartViewport } from "../../lib/chart-viewport"; import { connectMcpOauth } from "../../lib/mcp-connect"; import { rpc } from "../../lib/rpc"; @@ -461,104 +468,65 @@ export function ChartBlockView({ ); } -export function ArtifactImage({ - target, - artifactId, - name, -}: { - target: ArtifactTarget; - artifactId: string; - name: string; -}) { +export function ArtifactImage({ artifactId, name }: { artifactId: string; name: string }) { const { t } = useLingui(); - const [src, setSrc] = useState(null); const [open, setOpen] = useState(false); - const [visible, setVisible] = useState(false); - const container = useRef(null); - const targetBotId = "botId" in target ? target.botId : undefined; - const targetGroupId = "groupId" in target ? target.groupId : undefined; + const [failed, setFailed] = useState(false); + const src = artifactUrl(artifactId); - useEffect(() => { - const element = container.current; - if (!element || typeof IntersectionObserver === "undefined") { - setVisible(true); - return; - } - const observer = new IntersectionObserver( - (entries) => { - if (entries.some((entry) => entry.isIntersecting)) { - setVisible(true); - observer.disconnect(); - } - }, - { rootMargin: "320px" }, + if (failed) { + return ( +
+ {name} +
); - observer.observe(element); - return () => observer.disconnect(); - }, []); - - useEffect(() => { - if (!visible) return; - let cancelled = false; - let objectUrl: string | null = null; - setSrc(null); - void rpc.artifacts - .get( - targetBotId - ? { botId: targetBotId, artifactId } - : { groupId: targetGroupId ?? "", artifactId }, - ) - .then((artifact) => { - const bytes = decodeArtifactBase64(artifact.contentBase64); - objectUrl = URL.createObjectURL( - new Blob([new Uint8Array(bytes)], { type: artifact.mimeType }), - ); - if (cancelled) URL.revokeObjectURL(objectUrl); - else setSrc(objectUrl); - }) - .catch(() => undefined); - return () => { - cancelled = true; - if (objectUrl) URL.revokeObjectURL(objectUrl); - }; - }, [artifactId, targetBotId, targetGroupId, visible]); - + } return ( -
- {src ? ( - + + - {name} - - ) : ( -
- {name} -
- )} - {src ? ( - - - {name} + {name} +
+ + + } + render={
 - ) : null} +
+ {name} + + ); } From 4d75144cb82ada3cc47f99f4477a2eb8bae19e86 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:53:48 +0000 Subject: [PATCH 2/6] feat(media): give agents attached media and larger bot files Agents only received the documents of a turn on their computer and could attach workspace files up to 10 MiB, so they could neither cut a clip a user sent nor post the video they rendered. - Current-turn images, video and audio are copied into the bot's workspace (Team Computers under bots//) like documents. Remote computers move files as base64 JSON, so anything above their 12 MiB transfer cap stays in storage and the agent is told it by artifact id instead of failing the turn; local computers copy up to the kind cap. - attach_file accepts video and audio by extension and attaches up to the kind cap when the store streams (still bounded by the transfer cap of a remote computer), stored through the shared artifact helper with a real SHA-256. - The render_plot fallback that attaches a chart PNG now records the group, so the chart opens in group conversations. - Agent history names image artifacts by id, like video and audio, so a later turn can refer to them. - artifacts.get refuses files above 25 MiB: base64 reads hold the whole file in memory, and larger media is served by the artifact route. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- apps/api/src/artifacts.ts | 9 ++ apps/api/src/router.ts | 26 ++-- packages/adapters/src/builtin-tools.ts | 2 +- packages/adapters/src/computer-support.ts | 12 ++ packages/adapters/src/executor.ts | 27 ++-- .../adapters/src/history-compaction.test.ts | 2 +- packages/adapters/src/reply-context.test.ts | 4 +- .../adapters/src/thread-artifacts.test.ts | 90 +++++++++-- packages/adapters/src/thread-artifacts.ts | 140 +++++++++--------- packages/core/src/attachments.test.ts | 2 +- packages/core/src/attachments.ts | 3 +- packages/db/src/events.test.ts | 2 +- packages/testkit/src/artifact-routes.test.ts | 13 ++ 13 files changed, 226 insertions(+), 106 deletions(-) diff --git a/apps/api/src/artifacts.ts b/apps/api/src/artifacts.ts index b0804a0..9dec9c4 100644 --- a/apps/api/src/artifacts.ts +++ b/apps/api/src/artifacts.ts @@ -11,6 +11,12 @@ import { } from "@rakazo/core"; import { IsolationError, type PrismaClient } from "@rakazo/db"; +/** + * Base64 reads load the whole file into memory; larger media is served by + * GET /api/artifacts/:id instead. + */ +export const ARTIFACT_INLINE_READ_MAX_BYTES = 25 * 1024 * 1024; + function adapterContext(actor: Actor, botId: string, operationId: string) { return { operationId, @@ -131,6 +137,9 @@ async function readArtifact( }, contextBotId: string, ) { + if (row.size > ARTIFACT_INLINE_READ_MAX_BYTES) { + throw new AttachmentValidationError("This file is too large to load inline."); + } const bytes = await artifacts.get( row.storageKey, adapterContext(actor, contextBotId, `artifact-get:${row.id}`), diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 2baf21e..cc14276 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -4640,24 +4640,26 @@ export function createRouter(deps: RouterDeps) { } }), get: authed.artifacts.get.handler(async ({ context, input }) => { - if (input.groupId) { - const group = await groupRepos.getGroupTarget(context.actor, input.groupId); - const contextBotId = group.members[0]?.bot.id; - if (!contextBotId) throw new IsolationError(); - return getSpaceArtifact(deps, context.actor, { - artifactId: input.artifactId, - groupId: input.groupId, - contextBotId, - }); - } - await repos.getBot(context.actor, input.botId!); try { + if (input.groupId) { + const group = await groupRepos.getGroupTarget(context.actor, input.groupId); + const contextBotId = group.members[0]?.bot.id; + if (!contextBotId) throw new IsolationError(); + return await getSpaceArtifact(deps, context.actor, { + artifactId: input.artifactId, + groupId: input.groupId, + contextBotId, + }); + } + await repos.getBot(context.actor, input.botId!); return await getOwnedArtifact(deps, context.actor, { botId: input.botId!, artifactId: input.artifactId, }); } catch (error) { - if (error instanceof IsolationError) throw error; + if (error instanceof AttachmentValidationError) { + throw new ORPCError("BAD_REQUEST", { message: error.message }); + } throw error; } }), diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index 0ff52fa..043aa9d 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -144,7 +144,7 @@ export const builtinAgentTools: ConnectorTool[] = [ { name: "attach_file", description: - "Attach a workspace file from this bot's home to the chat thread as an image or common file. The file stays in place; users can open it from the message.", + "Attach a workspace file from this bot's home to the chat thread as an image, video, audio or common file. The file stays in place; users can open or play it from the message.", inputSchema: { type: "object", properties: { path: { type: "string" } }, diff --git a/packages/adapters/src/computer-support.ts b/packages/adapters/src/computer-support.ts index 62deb83..a6e6519 100644 --- a/packages/adapters/src/computer-support.ts +++ b/packages/adapters/src/computer-support.ts @@ -17,6 +17,18 @@ export function toComputerRef(computer: { }; } +/** + * Remote computers move files as base64 JSON (the Docker supervisor accepts 16 MiB requests), so + * files copied to or read from them stay below this size. Local computers have no transfer cap. + */ +export const REMOTE_COMPUTER_FILE_TRANSFER_MAX_BYTES = 12 * 1024 * 1024; + +export function computerFileTransferMaxBytes(computer: Pick): number { + return computer.kind === "desktop" || computer.kind === "fake" + ? Number.POSITIVE_INFINITY + : REMOTE_COMPUTER_FILE_TRANSFER_MAX_BYTES; +} + const EMPTY_PNG = Uint8Array.from( Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", diff --git a/packages/adapters/src/executor.ts b/packages/adapters/src/executor.ts index 5cb9181..8b8f8a6 100644 --- a/packages/adapters/src/executor.ts +++ b/packages/adapters/src/executor.ts @@ -197,6 +197,7 @@ import { withComputerScreenAvailability } from "./computer-screens.js"; import { assertTeamShellCommandAllowed, assertTeamWorkspacePathAllowed, + computerFileTransferMaxBytes, displayBotWorkspacePath, filterTeamWorkspaceEntries, resolveBotWorkspaceCwd, @@ -368,8 +369,10 @@ import { } from "./takeover-resume.js"; import { getActiveTeachingSession, parsePlaybook } from "./teaching-session.js"; import { + attachableFileMaxBytes, attachWorkspaceFileToThread, currentTurnFilesInstruction, + formatMiB, materializeCurrentTurnFiles, } from "./thread-artifacts.js"; import { advanceToolCallLoopGuard } from "./tool-loop.js"; @@ -2366,6 +2369,7 @@ export function createRunExecutor(deps: ExecutorDeps) { spaceId: run.spaceId, userId: run.userId, botId: bot.id, + groupId: thread.groupId ?? undefined, runId: run.id, filePath: outPath, bytes: png, @@ -2391,18 +2395,23 @@ export function createRunExecutor(deps: ExecutorDeps) { return finish({ error: "artifact storage unavailable", path: filePath }); } const storedPath = resolveStoredWorkspacePath(filePath); - let bytes: Uint8Array; - try { - bytes = await deps.sandbox.readFile(computer, storedPath, context, { - maxBytes: ATTACHMENT_MAX_BYTES, - }); - } catch { - return finish({ error: "file not found or unreadable", path: filePath }); - } const mimeType = inferAttachmentMimeType(filePath); if (!mimeType) { return finish({ error: "unsupported attachment type", path: filePath }); } + const maxBytes = Math.min( + attachableFileMaxBytes(deps.artifacts, mimeType), + computerFileTransferMaxBytes(computer), + ); + let bytes: Uint8Array; + try { + bytes = await deps.sandbox.readFile(computer, storedPath, context, { maxBytes }); + } catch { + return finish({ + error: `file not found, unreadable or larger than ${formatMiB(maxBytes)}`, + path: filePath, + }); + } try { const attached = await attachWorkspaceFileToThread( { prisma: deps.prisma, artifacts: deps.artifacts }, @@ -3886,7 +3895,7 @@ export function createRunExecutor(deps: ExecutorDeps) { item.blocks, context.signal, ); - workspaceCheckpoint.markFiles(files); + workspaceCheckpoint.markFiles(files.filter((file) => file.path)); const filesInstruction = currentTurnFilesInstruction(files); return { id: item.id, diff --git a/packages/adapters/src/history-compaction.test.ts b/packages/adapters/src/history-compaction.test.ts index ef8ad64..b130201 100644 --- a/packages/adapters/src/history-compaction.test.ts +++ b/packages/adapters/src/history-compaction.test.ts @@ -498,7 +498,7 @@ describe("compactHistory", () => { const [request] = harness.runtime.run.mock.calls[0]!; expect(request.prompt).toContain("[file: plan.pdf (application/pdf, 123 bytes)]"); - expect(request.prompt).toContain("[image: diagram.png]"); + expect(request.prompt).toContain("[image: diagram.png (artifact artifact-2)]"); }); it("compacts locally without a semantic memory provider", async () => { diff --git a/packages/adapters/src/reply-context.test.ts b/packages/adapters/src/reply-context.test.ts index 136dea7..9b0db83 100644 --- a/packages/adapters/src/reply-context.test.ts +++ b/packages/adapters/src/reply-context.test.ts @@ -55,11 +55,11 @@ describe("reply context", () => { ...target, blocks: [ { kind: "text", text: "" }, - { kind: "image", name: "example.png" }, + { kind: "image", artifactId: "artifact-1", name: "example.png" }, ], }); const context = await loadReplyContext(prisma, "thread-1", "user-reply"); - expect(context).toContain("[image: example.png]"); + expect(context).toContain("[image: example.png (artifact artifact-1)]"); expect(context).toContain("\\u003c/reply_target\\u003e"); expect(context?.match(/<\/reply_target>/g)).toHaveLength(1); }); diff --git a/packages/adapters/src/thread-artifacts.test.ts b/packages/adapters/src/thread-artifacts.test.ts index 5d164bd..fb79536 100644 --- a/packages/adapters/src/thread-artifacts.test.ts +++ b/packages/adapters/src/thread-artifacts.test.ts @@ -8,6 +8,7 @@ import type { MessageBlock } from "@rakazo/contracts"; import type { PrismaClient } from "@rakazo/db"; import { describe, expect, it, vi } from "vitest"; import { + attachableFileMaxBytes, attachWorkspaceFileToThread, currentTurnFilesInstruction, materializeCurrentTurnFiles, @@ -25,7 +26,8 @@ describe("current-turn thread files", () => { artifact: { create: vi.fn().mockRejectedValue(failure) }, } as unknown as PrismaClient, artifacts: { - put: vi.fn().mockResolvedValue({ id: "stored-1", hash: "hash" }), + describe: () => ({ capabilities: { stream: true } }), + putStream: vi.fn().mockResolvedValue({ key: "stored-1", size: 3, sha256: "hash" }), remove, } as unknown as ArtifactStore, }, @@ -111,6 +113,7 @@ describe("current-turn thread files", () => { ); expect(files).toEqual([ { + artifactId: "artifact-1", name: "../quarterly report.pdf", mimeType: "application/pdf", size: 4, @@ -120,7 +123,73 @@ describe("current-turn thread files", () => { expect(currentTurnFilesInstruction(files)).toContain('"attachments/artifact-1.pdf"'); }); - it("does not load images as computer files", async () => { + it("copies images, video and audio too, and keeps files over a remote computer's cap in storage", async () => { + const mb = 1024 * 1024; + const findMany = vi.fn().mockResolvedValue([ + { id: "img-1", name: "photo.png", mimeType: "image/png", size: 3, storageKey: "k-img" }, + { id: "aud-1", name: "memo.mp3", mimeType: "audio/mpeg", size: 3, storageKey: "k-aud" }, + { id: "vid-1", name: "clip.mp4", mimeType: "video/mp4", size: 40 * mb, storageKey: "k-vid" }, + ]); + const get = vi.fn().mockResolvedValue(new Uint8Array([1, 2, 3])); + const writeFile = vi.fn().mockResolvedValue(undefined); + const context: AdapterContext & { botId: string } = { + operationId: "run-1", + traceId: "run-1", + spaceId: "workspace-1", + userId: "user-1", + botId: "bot-1", + signal: new AbortController().signal, + }; + const blocks: MessageBlock[] = [ + { kind: "image", artifactId: "img-1", name: "photo.png", mimeType: "image/png" }, + { kind: "audio", artifactId: "aud-1", name: "memo.mp3", mimeType: "audio/mpeg", size: 3 }, + { + kind: "video", + artifactId: "vid-1", + name: "clip.mp4", + mimeType: "video/mp4", + size: 40 * mb, + }, + { kind: "text", text: "see these" }, + ]; + const deps = { + prisma: { artifact: { findMany } } as unknown as PrismaClient, + artifacts: { get } as unknown as ArtifactStore, + sandbox: { writeFile } as unknown as SandboxProvider, + }; + + const remote = await materializeCurrentTurnFiles(deps, blocks, { + context, + computer: { id: "computer-1", botId: "bot-1", kind: "docker", providerRef: "docker-1" }, + computerMode: "dedicated", + }); + expect(remote.map((file) => [file.artifactId, file.path])).toEqual([ + ["img-1", "attachments/img-1.png"], + ["aud-1", "attachments/aud-1.mp3"], + ["vid-1", null], + ]); + expect(get).not.toHaveBeenCalledWith("k-vid", context); + const instruction = currentTurnFilesInstruction(remote); + expect(instruction).toContain('"attachments/img-1.png"'); + expect(instruction).toContain("too large to copy to your computer"); + expect(instruction).toContain('"clip.mp4" (video/mp4, 41943040 bytes): artifact vid-1'); + + writeFile.mockClear(); + const local = await materializeCurrentTurnFiles(deps, blocks, { + context, + computer: { id: "computer-2", botId: "bot-1", kind: "desktop", providerRef: "desktop-1" }, + computerMode: "dedicated", + }); + expect(local.map((file) => file.path)).toEqual([ + "attachments/img-1.png", + "attachments/aud-1.mp3", + "attachments/vid-1.mp4", + ]); + expect(writeFile).toHaveBeenCalledTimes(3); + expect(currentTurnFilesInstruction(local)).not.toContain("too large"); + }); + + it("skips loading when a turn has no attachments", async () => { const findMany = vi.fn(); const files = await materializeCurrentTurnFiles( { @@ -128,14 +197,7 @@ describe("current-turn thread files", () => { artifacts: {} as ArtifactStore, sandbox: {} as SandboxProvider, }, - [ - { - kind: "image", - artifactId: "image-1", - name: "photo.png", - mimeType: "image/png", - }, - ], + [{ kind: "text", text: "hello" }], { context: { operationId: "run-1", @@ -158,4 +220,12 @@ describe("current-turn thread files", () => { expect(files).toEqual([]); expect(findMany).not.toHaveBeenCalled(); }); + + it("caps attachments by kind when the store streams and at the inline cap otherwise", () => { + const streaming = { describe: () => ({ capabilities: { stream: true } }) } as ArtifactStore; + const buffered = { describe: () => ({ capabilities: { stream: false } }) } as ArtifactStore; + expect(attachableFileMaxBytes(streaming, "video/mp4")).toBe(500 * 1024 * 1024); + expect(attachableFileMaxBytes(streaming, "text/plain")).toBe(25 * 1024 * 1024); + expect(attachableFileMaxBytes(buffered, "video/mp4")).toBe(10 * 1024 * 1024); + }); }); diff --git a/packages/adapters/src/thread-artifacts.ts b/packages/adapters/src/thread-artifacts.ts index 72bc432..8d82aef 100644 --- a/packages/adapters/src/thread-artifacts.ts +++ b/packages/adapters/src/thread-artifacts.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import path from "node:path"; import type { AdapterContext, @@ -12,18 +11,39 @@ import { attachmentExtensionForMimeType, inferAttachmentMimeType, messageBlockForArtifact, - validateAttachmentMimeType, } from "@rakazo/core"; import type { PrismaClient } from "@rakazo/db"; -import { assertTeamWorkspacePathAllowed, resolveBotWorkspacePath } from "./computer-support.js"; +import { artifactMaxBytes, storeArtifactStream } from "./artifact-files.js"; +import { + assertTeamWorkspacePathAllowed, + computerFileTransferMaxBytes, + resolveBotWorkspacePath, +} from "./computer-support.js"; export type MaterializedThreadFile = { + artifactId: string; name: string; mimeType: string; size: number; - path: string; + /** Workspace path, or null when the file is too large to copy to this computer. */ + path: string | null; }; +type AttachmentBlock = Extract; + +const ATTACHMENT_BLOCK_KINDS: ReadonlySet = new Set(["image", "video", "audio", "file"]); + +/** Media up to its kind cap when the store streams; otherwise the inline attachment cap. */ +export function attachableFileMaxBytes(artifacts: ArtifactStore, mimeType: string): number { + return artifacts.describe().capabilities.stream + ? artifactMaxBytes(mimeType) + : Math.min(ATTACHMENT_MAX_BYTES, artifactMaxBytes(mimeType)); +} + +export function formatMiB(bytes: number): string { + return `${Math.floor(bytes / (1024 * 1024))} MiB`; +} + export async function attachWorkspaceFileToThread( deps: { prisma: PrismaClient; @@ -39,60 +59,38 @@ export async function attachWorkspaceFileToThread( bytes: Uint8Array; operationId: string; }, -): Promise<{ - artifactId: string; - block: Extract; -}> { +): Promise<{ artifactId: string; block: AttachmentBlock }> { const name = path.basename(input.filePath) || input.filePath; const mimeType = inferAttachmentMimeType(name); if (!mimeType) { throw new Error(`Unsupported attachment type for ${name}`); } - validateAttachmentMimeType(mimeType); - if (input.bytes.byteLength > ATTACHMENT_MAX_BYTES) { - throw new Error("file exceeds the 10 MiB attachment limit"); + const maxBytes = attachableFileMaxBytes(deps.artifacts, mimeType); + if (input.bytes.byteLength > maxBytes) { + throw new Error(`file exceeds the ${formatMiB(maxBytes)} attachment limit`); } - - const context = { - operationId: input.operationId, - traceId: input.operationId, + const bytes = input.bytes; + const row = await storeArtifactStream(deps, { spaceId: input.spaceId, userId: input.userId, botId: input.botId, - signal: new AbortController().signal, - }; - const stored = await deps.artifacts.put({ name, mimeType, bytes: input.bytes }, context); - const hash = createHash("sha256").update(input.bytes).digest("hex"); - const row = await deps.prisma.artifact - .create({ - data: { - spaceId: input.spaceId, - userId: input.userId, - botId: input.botId, - groupId: input.groupId, - runId: input.runId, - name, - mimeType, - size: input.bytes.byteLength, - hash, - storageKey: stored.id, - }, - }) - .catch(async (error) => { - await deps.artifacts.remove(stored.id, context).catch(() => undefined); - throw error; - }); - return { - artifactId: row.id, - block: messageBlockForArtifact({ - id: row.id, - name: row.name, - mimeType: row.mimeType, - size: row.size, - }), - }; + groupId: input.groupId ?? null, + runId: input.runId, + name, + mimeType, + stream: (async function* () { + yield bytes; + })(), + maxBytes, + }); + return { artifactId: row.id, block: messageBlockForArtifact(row) }; } +/** + * Copies this turn's attachments (files, images, video and audio) into the bot's workspace so its + * tools can open them. Files above the computer's transfer cap stay in storage and are listed + * with their artifact id instead. + */ export async function materializeCurrentTurnFiles( deps: { prisma: PrismaClient; @@ -108,28 +106,31 @@ export async function materializeCurrentTurnFiles( markWorkspaceDirty?: () => void; }, ): Promise { - const fileBlocks = blocks?.filter( - (block): block is Extract => block.kind === "file", + const attachmentBlocks = blocks?.filter((block): block is AttachmentBlock => + ATTACHMENT_BLOCK_KINDS.has(block.kind), ); - if (!fileBlocks?.length) return []; + if (!attachmentBlocks?.length) return []; const rows = await deps.prisma.artifact.findMany({ where: { - id: { in: fileBlocks.map((block) => block.artifactId) }, + id: { in: attachmentBlocks.map((block) => block.artifactId) }, spaceId: input.context.spaceId, userId: input.context.userId, }, }); const byId = new Map(rows.map((row) => [row.id, row])); + const transferMaxBytes = computerFileTransferMaxBytes(input.computer); const materialized: MaterializedThreadFile[] = []; - for (const block of fileBlocks) { + for (const block of attachmentBlocks) { const row = byId.get(block.artifactId); if (!row) throw new Error(`Attached file is unavailable: ${block.name}`); - const bytes = await deps.artifacts.get(row.storageKey, input.context); - if (bytes.byteLength > ATTACHMENT_MAX_BYTES) { - throw new Error(`Attached file exceeds the 10 MiB limit: ${row.name}`); + const entry = { artifactId: row.id, name: row.name, mimeType: row.mimeType, size: row.size }; + if (row.size > Math.min(transferMaxBytes, artifactMaxBytes(row.mimeType))) { + materialized.push({ ...entry, path: null }); + continue; } + const bytes = await deps.artifacts.get(row.storageKey, input.context); const relativePath = `attachments/${row.id}${attachmentExtensionForMimeType(row.mimeType)}`; const storedPath = resolveBotWorkspacePath( input.computerMode, @@ -146,24 +147,29 @@ export async function materializeCurrentTurnFiles( }, input.context, ); - materialized.push({ - name: row.name, - mimeType: row.mimeType, - size: row.size, - path: relativePath, - }); + materialized.push({ ...entry, path: relativePath }); } return materialized; } export function currentTurnFilesInstruction(files: readonly MaterializedThreadFile[]): string { - if (!files.length) return ""; + const copied = files.filter((file) => file.path !== null); + const kept = files.filter((file) => file.path === null); + const describe = (file: MaterializedThreadFile) => + `- ${JSON.stringify(file.name)} (${file.mimeType}, ${file.size} bytes)`; return [ - "Current-turn attached files are available on your computer. Inspect them before answering:", - ...files.map( - (file) => - `- ${JSON.stringify(file.name)} (${file.mimeType}, ${file.size} bytes): ${JSON.stringify(file.path)}`, - ), + ...(copied.length + ? [ + "Current-turn attached files are available on your computer. Inspect them before answering:", + ...copied.map((file) => `${describe(file)}: ${JSON.stringify(file.path)}`), + ] + : []), + ...(kept.length + ? [ + "These current-turn attachments are too large to copy to your computer. Refer to them by artifact id; do not guess their contents:", + ...kept.map((file) => `${describe(file)}: artifact ${file.artifactId}`), + ] + : []), ].join("\n"); } diff --git a/packages/core/src/attachments.test.ts b/packages/core/src/attachments.test.ts index 6db2e57..b39c7d4 100644 --- a/packages/core/src/attachments.test.ts +++ b/packages/core/src/attachments.test.ts @@ -44,7 +44,7 @@ describe("attachment helpers", () => { size: 99, }, ]), - ).toBe("hello\n[image: shot.png]\n[file: brief.pdf (application/pdf, 99 bytes)]"); + ).toBe("hello\n[image: shot.png (artifact a1)]\n[file: brief.pdf (application/pdf, 99 bytes)]"); }); it("infers attachment mime types from extensions", () => { diff --git a/packages/core/src/attachments.ts b/packages/core/src/attachments.ts index b4a6759..9953cb8 100644 --- a/packages/core/src/attachments.ts +++ b/packages/core/src/attachments.ts @@ -119,12 +119,11 @@ export function blocksToAgentHistoryText(blocks: MessageBlock[]): string { .map((block) => { if (block.kind === "text") return block.text; if (block.kind === "chart") return `[chart: ${block.name}]`; - if (block.kind === "image") return `[image: ${block.name}]`; if (block.kind === "file") { return `[file: ${block.name} (${block.mimeType}, ${block.size} bytes)]`; } // Media keeps the artifact id so a later turn can edit or export what it refers to. - if (block.kind === "video" || block.kind === "audio") { + if (block.kind === "image" || block.kind === "video" || block.kind === "audio") { return `[${block.kind}: ${block.name} (artifact ${block.artifactId})]`; } if (block.kind === "media") { diff --git a/packages/db/src/events.test.ts b/packages/db/src/events.test.ts index e8ca709..afc0e39 100644 --- a/packages/db/src/events.test.ts +++ b/packages/db/src/events.test.ts @@ -1739,7 +1739,7 @@ describe("claimSteering", () => { { id: "steer-1", messageId: "message-1", - text: "First\n[image: chart.png]", + text: "First\n[image: chart.png (artifact artifact-1)]", blocks: [ { kind: "text", text: "First" }, { diff --git a/packages/testkit/src/artifact-routes.test.ts b/packages/testkit/src/artifact-routes.test.ts index bc33ca4..3b6a3e6 100644 --- a/packages/testkit/src/artifact-routes.test.ts +++ b/packages/testkit/src/artifact-routes.test.ts @@ -283,6 +283,19 @@ describeRoutes("artifact HTTP routes", () => { const me = await rpc<{ spaceId: string }>(app, owner, "me"); const spaceDir = path.join(dataDir, "artifacts", me.spaceId); expect(readdirSafe(spaceDir)).toEqual([]); + + // Video far above the inline cap uploads, streams by range, and is refused as base64. + const large = Buffer.alloc(26 * 1024 * 1024, 1); + large.write("head", 0); + const uploaded = await upload(app, owner, { botId: bot.id }, "long.mp4", "video/mp4", large); + expect(uploaded.status).toBe(201); + const largeVideo = (await uploaded.json()) as { id: string }; + const head = await get(app, owner, largeVideo.id, { range: "bytes=0-3" }); + expect(head.status).toBe(206); + expect(await head.text()).toBe("head"); + await expect( + rpc(app, owner, "artifacts/get", { botId: bot.id, artifactId: largeVideo.id }), + ).rejects.toThrow(/400: .*too large to load inline/); }); }); From c008112ab4875029ae434d60cb4dcb51d95cb818 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:53:55 +0000 Subject: [PATCH 3/6] fix(mobile): show video, audio and media cards as plain rows Mobile has no media UI yet, so video, audio and media-card blocks rendered as empty bubbles or fell out of previews. - Thread text and reply previews describe video and audio files and the media card's title and status. - The bubble lists video, audio and a finished card's outputs as tappable rows that open the file like other attachments. - Opening video or audio downloads it from the artifact route to the cache instead of a base64 RPC, so long recordings open without the server holding them in memory. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- apps/mobile/app/thread.tsx | 15 +++++++---- apps/mobile/lib/api.test.ts | 46 ++++++++++++++++++++++++++++++++ apps/mobile/lib/api.ts | 22 +++++++++++++++ apps/mobile/lib/artifact-open.ts | 13 ++++++++- 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/apps/mobile/app/thread.tsx b/apps/mobile/app/thread.tsx index c2eb734..c8ae7d4 100644 --- a/apps/mobile/app/thread.tsx +++ b/apps/mobile/app/thread.tsx @@ -77,6 +77,7 @@ import { type MobileMessagePage, type MobileSnapshot, mergeMobileSnapshot, + messageAttachments, messagingProviderLabel, prependMobileMessagePage, rpc, @@ -2243,7 +2244,11 @@ function previewMessageText(message: MobileMessage): string { .join(" ") .trim(); if (text) return text; - if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) { + if ( + message.blocks.some((block) => + ["image", "file", "video", "audio", "media"].includes(block.kind), + ) + ) { return t("Attachment"); } return t("Message"); @@ -2676,9 +2681,7 @@ const MessageBubble = memo(function MessageBubble({ ); } - const attachments = message.blocks.filter( - (block) => block.kind === "image" || block.kind === "file", - ); + const attachments = messageAttachments(message.blocks); const caption = message.blocks .flatMap((block) => { if (block.kind === "channel_message" && block.text) { @@ -2686,6 +2689,7 @@ const MessageBubble = memo(function MessageBubble({ `${messagingProviderLabel(block.provider, block.transport)} ยท ${block.fromLabel}: ${block.text}`, ]; } + if (block.kind === "media") return [block.title]; return block.kind === "text" && block.text ? [block.text] : []; }) .join("\n"); @@ -2797,7 +2801,8 @@ const MessageBubble = memo(function MessageBubble({ fontSize: 15, }} > - ๐Ÿ“Ž {attachment.name ?? t("File")} + {attachment.kind === "video" ? "๐ŸŽฌ" : attachment.kind === "audio" ? "๐ŸŽต" : "๐Ÿ“Ž"}{" "} + {attachment.name ?? t("File")} {attachment.size ? ( { ).toBe("Slack ยท Alex: Hello from the group"); }); + it("falls back to plain text and file rows for video, audio and media cards", () => { + const blocks = [ + { + kind: "video" as const, + artifactId: "vid-1", + mimeType: "video/mp4", + name: "clip.mp4", + size: 10, + width: 1080, + height: 1920, + }, + { + kind: "audio" as const, + artifactId: "aud-1", + mimeType: "audio/mpeg", + name: "memo.mp3", + size: 5, + }, + { + kind: "media" as const, + jobId: "job-1", + operation: "export_video" as const, + status: "done" as const, + title: "Vertical cut", + outputs: [ + { + artifactId: "out-1", + kind: "video" as const, + mimeType: "video/mp4", + name: "cut.mp4", + size: 7, + }, + ], + }, + ]; + expect(blockText(mobileMessage("media-1", blocks))).toBe( + "[video: clip.mp4]\n[audio: memo.mp3]\nVertical cut: done", + ); + expect(messageAttachments(blocks).map((item) => [item.kind, item.artifactId])).toEqual([ + ["video", "vid-1"], + ["audio", "aud-1"], + ["video", "out-1"], + ]); + }); + it("deduplicates durable messages and replaces matching transient subagent state", () => { const initial = snapshot([ mobileMessage("message-1", [{ kind: "text", text: "old" }]), diff --git a/apps/mobile/lib/api.ts b/apps/mobile/lib/api.ts index f328abc..ac39cf1 100644 --- a/apps/mobile/lib/api.ts +++ b/apps/mobile/lib/api.ts @@ -860,6 +860,24 @@ export function copyableMobileMessageText(message: MobileMessage): string { .trim(); } +export type MobileAttachment = { + kind: "image" | "video" | "audio" | "file"; + artifactId: string; + name: string; + mimeType: string; + size?: number; +}; + +/** Files a message carries, including the outputs of a finished media card, as tappable rows. */ +export function messageAttachments(blocks: readonly MessageBlock[]): MobileAttachment[] { + return blocks.flatMap((block): MobileAttachment[] => { + if (block.kind === "image") return [{ ...block, kind: "image" }]; + if (block.kind === "video" || block.kind === "audio" || block.kind === "file") return [block]; + if (block.kind === "media") return block.outputs; + return []; + }); +} + export function blockText(message: MobileMessage) { return message.blocks .map((block) => { @@ -879,6 +897,10 @@ export function blockText(message: MobileMessage) { if (block.kind === "file") { return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`; } + if (block.kind === "video" || block.kind === "audio") { + return `[${block.kind}: ${block.name ?? "attachment"}]`; + } + if (block.kind === "media") return `${block.title}: ${block.status}`; if (block.kind === "steps") { return (block.steps ?? []) .map((step) => `${step.label}${step.count > 1 ? ` ร—${step.count}` : ""}`) diff --git a/apps/mobile/lib/artifact-open.ts b/apps/mobile/lib/artifact-open.ts index 17f0e00..ab18d9a 100644 --- a/apps/mobile/lib/artifact-open.ts +++ b/apps/mobile/lib/artifact-open.ts @@ -1,6 +1,7 @@ +import { attachmentKindOf } from "@rakazo/contracts"; import { File, Paths } from "expo-file-system"; import * as Sharing from "expo-sharing"; -import { rpc } from "./api"; +import { captureApiRequestContext, rpc } from "./api"; import { artifactCacheFileName } from "./artifact-file"; import { t } from "./i18n"; @@ -11,6 +12,16 @@ async function cacheMobileArtifact( artifactId: string, mimeType: string, ): Promise { + const kind = attachmentKindOf(mimeType); + if (kind === "video" || kind === "audio") { + // Media can be hundreds of megabytes: stream it to disk instead of a base64 RPC payload. + const { apiBase, headers } = await captureApiRequestContext(); + return File.downloadFileAsync( + `${apiBase}/api/artifacts/${encodeURIComponent(artifactId)}`, + new File(Paths.cache, artifactCacheFileName(artifactId, mimeType)), + { headers, idempotent: true }, + ); + } const artifact = await rpc<{ contentBase64: string }>("artifacts/get", { ...target, artifactId, From 147611a8a183224368ffcaa1fac745d14d8c2d0c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:56:47 +0000 Subject: [PATCH 4/6] fix(api): refuse artifact uploads from untrusted origins Uploads with a text/plain, text/csv or text/markdown body are CORS "simple" requests that a browser sends cross-site without a preflight. The session cookie's SameSite default already keeps it off such requests, but a deployment that relaxes it would accept uploads from any page. The route now applies the same origin rule as CORS before reading the session. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- apps/api/src/app.ts | 1 + apps/api/src/artifact-http.test.ts | 14 ++++++++++++++ apps/api/src/artifact-http.ts | 6 +++++- packages/testkit/src/artifact-routes.test.ts | 9 +++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index c05a71f..8893a50 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -589,6 +589,7 @@ export async function createApp( artifacts, sessionUserId: async (request) => (await auth.api.getSession({ headers: sessionHeaders(request) }))?.user?.id ?? null, + trustedOrigin: (origin) => isTrustedOrigin(origin, env), }); mountWebhookHttpRoutes(app, { prisma, secrets, events, jobs }); // Shared with stop so a shutdown during retry delays does not restart polling. diff --git a/apps/api/src/artifact-http.test.ts b/apps/api/src/artifact-http.test.ts index a1d6d44..0a2a6fc 100644 --- a/apps/api/src/artifact-http.test.ts +++ b/apps/api/src/artifact-http.test.ts @@ -77,6 +77,7 @@ describe("artifact HTTP routes", () => { prisma: prisma as unknown as PrismaClient, artifacts: artifacts as unknown as ArtifactStore, sessionUserId, + trustedOrigin: (origin) => origin === "http://127.0.0.1:5173", }); return { hono, prisma, artifacts }; } @@ -95,6 +96,19 @@ describe("artifact HTTP routes", () => { expect(artifacts.putStream).not.toHaveBeenCalled(); }); + it("refuses uploads from untrusted origins before reading the session", async () => { + const sessionUserId = vi.fn(async () => "user-1"); + const { hono, artifacts } = app(sessionUserId); + const response = await hono.request("/api/artifacts?botId=bot-1&name=a.txt", { + method: "POST", + headers: { "content-type": "text/plain", origin: "https://evil.example" }, + body: "data", + }); + expect(response.status).toBe(403); + expect(sessionUserId).not.toHaveBeenCalled(); + expect(artifacts.putStream).not.toHaveBeenCalled(); + }); + it("answers ids that are not artifact ids with 404 without a lookup", async () => { const { hono, prisma } = app(async () => "user-1"); const response = await hono.request(`/api/artifacts/${encodeURIComponent("../etc/passwd")}`); diff --git a/apps/api/src/artifact-http.ts b/apps/api/src/artifact-http.ts index b0c048c..d8b9df5 100644 --- a/apps/api/src/artifact-http.ts +++ b/apps/api/src/artifact-http.ts @@ -20,6 +20,8 @@ export type ArtifactHttpDeps = { artifacts: ArtifactStore; /** The signed-in user (cookie or bearer session), or null. */ sessionUserId: (request: Request) => Promise; + /** Same rule as CORS. Raw uploads can be CORS "simple" requests, so the origin is checked. */ + trustedOrigin: (origin: string) => boolean; }; /** Generated ids only; anything else is not an artifact. */ @@ -208,10 +210,12 @@ export function mountArtifactHttpRoutes(app: Hono, deps: ArtifactHttpDeps) { app.post(ARTIFACT_HTTP_PATH, async (c) => { const request = c.req.raw; - const reject = (status: 400 | 401 | 404 | 413 | 415, error: string) => { + const reject = (status: 400 | 401 | 403 | 404 | 413 | 415, error: string) => { cancelBody(request.body); return c.json({ error }, status); }; + const origin = c.req.header("origin"); + if (origin && !deps.trustedOrigin(origin)) return reject(403, "Forbidden"); const userId = await deps.sessionUserId(request); if (!userId) return reject(401, "Unauthorized"); const actor = await requireMembership( diff --git a/packages/testkit/src/artifact-routes.test.ts b/packages/testkit/src/artifact-routes.test.ts index 3b6a3e6..6e55fdc 100644 --- a/packages/testkit/src/artifact-routes.test.ts +++ b/packages/testkit/src/artifact-routes.test.ts @@ -148,6 +148,15 @@ describeRoutes("artifact HTTP routes", () => { (await upload(app, intruder, { groupId: group.id }, "x.mp4", "video/mp4", video)).status, ).toBe(404); + // A cross-site page cannot upload with the owner's cookie. + expect( + ( + await upload(app, owner, { botId: bot.id }, "x.txt", "text/plain", video, { + origin: "https://evil.example", + }) + ).status, + ).toBe(403); + // Signed out. expect((await app.request(`/api/artifacts/${botArtifact.id}`)).status).toBe(401); expect( From d67a6275a4f5bdf8e2a64f78b71f422baaabb84a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 14:18:06 +0000 Subject: [PATCH 5/6] test(web): cover group uploads in the main conversation The image E2E now runs in the main conversation, which is a group thread, so the browser test covers group uploads and reads from the artifact route as well as bot threads. Bot file attachments keep their run's operation id for tracing, and the non-streaming attach cap is the inline cap directly. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- apps/web/e2e/media-pipeline.spec.ts | 9 ++++++--- packages/adapters/src/artifact-files.ts | 4 +++- packages/adapters/src/thread-artifacts.ts | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/apps/web/e2e/media-pipeline.spec.ts b/apps/web/e2e/media-pipeline.spec.ts index dc712cc..eb63fe8 100644 --- a/apps/web/e2e/media-pipeline.spec.ts +++ b/apps/web/e2e/media-pipeline.spec.ts @@ -54,10 +54,13 @@ test("an uploaded video streams from the artifact route and plays inline", async await captureScreenshot(page, testInfo, "media-video-block"); }); -test("the image lightbox downloads the original from the artifact route", async ({ page }) => { +test("an image in the main conversation opens and downloads from the artifact route", async ({ + page, +}) => { const stamp = Date.now(); await signup(page, `media-image-${stamp}@rakazo.test`, "password12", "Media Image"); - await completeOnboarding(page); + // The main conversation is a group thread, so this covers group uploads and reads. + await completeOnboarding(page, undefined, { mainConversation: true }); await page .locator('input[type="file"]') @@ -65,7 +68,7 @@ test("the image lightbox downloads the original from the artifact route", async await page.getByRole("combobox", { name: /Message/ }).fill("a pixel"); await page.keyboard.press("Enter"); - const thumbnail = page.getByRole("img", { name: "pixel.png" }); + const thumbnail = page.getByTestId("transcript").getByRole("img", { name: "pixel.png" }); await expect(thumbnail).toBeVisible({ timeout: 30_000 }); await expect(thumbnail).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/); await thumbnail.click(); diff --git a/packages/adapters/src/artifact-files.ts b/packages/adapters/src/artifact-files.ts index ad88094..b228bf1 100644 --- a/packages/adapters/src/artifact-files.ts +++ b/packages/adapters/src/artifact-files.ts @@ -47,6 +47,7 @@ export async function storeArtifactStream( stream: ArtifactStreamPut["stream"]; maxBytes?: number; signal?: AbortSignal; + operationId?: string; }, ) { validateAttachmentMimeType(input.mimeType); @@ -54,7 +55,8 @@ export async function storeArtifactStream( input.maxBytes ?? Number.POSITIVE_INFINITY, artifactMaxBytes(input.mimeType), ); - const operationId = `artifact-store:${input.runId ?? input.botId ?? input.userId}`; + const operationId = + input.operationId ?? `artifact-store:${input.runId ?? input.botId ?? input.userId}`; const context: AdapterContext = { operationId, traceId: operationId, diff --git a/packages/adapters/src/thread-artifacts.ts b/packages/adapters/src/thread-artifacts.ts index 8d82aef..f6630bd 100644 --- a/packages/adapters/src/thread-artifacts.ts +++ b/packages/adapters/src/thread-artifacts.ts @@ -37,7 +37,7 @@ const ATTACHMENT_BLOCK_KINDS: ReadonlySet = new Set(["image", "video", " export function attachableFileMaxBytes(artifacts: ArtifactStore, mimeType: string): number { return artifacts.describe().capabilities.stream ? artifactMaxBytes(mimeType) - : Math.min(ATTACHMENT_MAX_BYTES, artifactMaxBytes(mimeType)); + : ATTACHMENT_MAX_BYTES; } export function formatMiB(bytes: number): string { @@ -82,6 +82,7 @@ export async function attachWorkspaceFileToThread( yield bytes; })(), maxBytes, + operationId: input.operationId, }); return { artifactId: row.id, block: messageBlockForArtifact(row) }; } From 17d285f87eb4097cc73375306f61a2d877c64139 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 14:26:11 +0000 Subject: [PATCH 6/6] fix(web): size uploaded videos from their own frame Uploaded videos carry no dimensions, so a vertical clip sat letterboxed in a 16:9 frame. The player now takes the frame size from the file's metadata when the block has none. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- .../web/src/components/media/MediaPlayer.test.tsx | 15 +++++++++++++++ apps/web/src/components/media/MediaPlayer.tsx | 10 +++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/media/MediaPlayer.test.tsx b/apps/web/src/components/media/MediaPlayer.test.tsx index 0be2403..c1d9f5a 100644 --- a/apps/web/src/components/media/MediaPlayer.test.tsx +++ b/apps/web/src/components/media/MediaPlayer.test.tsx @@ -66,6 +66,21 @@ describe("MediaPlayer", () => { expect(video.hasAttribute("controls")).toBe(true); }); + it("sizes the frame from the file when the block has no dimensions", async () => { + await act(async () => + root.render(), + ); + const frame = container.querySelector('[data-testid="media-video"]')!; + expect(frame.style.width).toBe("420px"); + const video = container.querySelector("video")!; + Object.defineProperty(video, "videoWidth", { value: 1080 }); + Object.defineProperty(video, "videoHeight", { value: 1920 }); + await act(async () => { + video.dispatchEvent(new Event("loadedmetadata")); + }); + expect(frame.style.width).toBe("270px"); + }); + it("falls back to a download link when the browser cannot play the file", async () => { await act(async () => root.render(), diff --git a/apps/web/src/components/media/MediaPlayer.tsx b/apps/web/src/components/media/MediaPlayer.tsx index d793ba4..0df489d 100644 --- a/apps/web/src/components/media/MediaPlayer.tsx +++ b/apps/web/src/components/media/MediaPlayer.tsx @@ -28,6 +28,7 @@ export function MediaPlayer({ const container = useRef(null); const near = useNearViewport(container); const [failed, setFailed] = useState(false); + const [frame, setFrame] = useState<{ width: number; height: number } | null>(null); const src = near ? artifactUrl(artifactId) : undefined; if (failed) return ; @@ -48,7 +49,8 @@ export function MediaPlayer({ ); } - const box = videoBox(width, height); + // Uploads carry no dimensions; size the frame from the file once its metadata loads. + const box = videoBox(width ?? frame?.width, height ?? frame?.height); return (
setFailed(true)} + onLoadedMetadata={(event) => { + const { videoWidth, videoHeight } = event.currentTarget; + if (videoWidth > 0 && videoHeight > 0) { + setFrame({ width: videoWidth, height: videoHeight }); + } + }} />
);