From 979034a8a64b6bb88aa4ca95f9d7d6a3a2d8bb79 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:50:02 +0000 Subject: [PATCH] feat(web): play video and audio inline and stream uploads and downloads Media sent in a conversation had nowhere to go: the composer capped every file at 10 MiB and pushed it through a base64 RPC, video and audio blocks did not render, and downloads decoded whole files in the page. - Video and audio blocks play inline through MediaPlayer, which loads nothing until the player is near the viewport, sizes the frame from the block's dimensions (vertical clips stay within the column) and uses the poster when one exists. Files the browser cannot play fall back to their name with a download action. - Images load straight from the artifact route; the lightbox gains a download action. - File cards, the Markdown preview and downloads read from the artifact route instead of decoding base64. - The composer accepts each kind up to its cap and names the computed limit when it skips a file; video, audio and anything above the inline cap upload as a raw stream, small files keep the inline RPC. - MediaPlayer and artifactUrl are what the media card builds on. Tests: MediaPlayer laziness, sizing and fallback, the upload helper, and an E2E that uploads a tiny MP4 fixture (VP9, since Playwright's Chromium has no H.264 decoder), plays it from the artifact route and captures media-video-block, plus the image lightbox download. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- apps/web/e2e/fixtures/media-clip.mp4 | Bin 0 -> 3117 bytes apps/web/e2e/media-pipeline.spec.ts | 79 ++++++++++ apps/web/src/components/ArtifactFileCard.tsx | 80 ++-------- .../src/components/media/MediaPlayer.test.tsx | 84 ++++++++++ apps/web/src/components/media/MediaPlayer.tsx | 121 ++++++++++++++ apps/web/src/lib/artifact-open.ts | 43 ++--- apps/web/src/lib/artifact-upload.test.ts | 83 ++++++++++ apps/web/src/lib/artifact-upload.ts | 68 ++++++++ apps/web/src/lib/artifact-url.ts | 32 +--- apps/web/src/lib/format-bytes.ts | 12 ++ apps/web/src/pages/Shell.tsx | 57 +++---- apps/web/src/pages/shell/message-cards.tsx | 148 +++++++----------- 12 files changed, 562 insertions(+), 245 deletions(-) create mode 100644 apps/web/e2e/fixtures/media-clip.mp4 create mode 100644 apps/web/e2e/media-pipeline.spec.ts create mode 100644 apps/web/src/components/media/MediaPlayer.test.tsx create mode 100644 apps/web/src/components/media/MediaPlayer.tsx create mode 100644 apps/web/src/lib/artifact-upload.test.ts create mode 100644 apps/web/src/lib/artifact-upload.ts create mode 100644 apps/web/src/lib/format-bytes.ts diff --git a/apps/web/e2e/fixtures/media-clip.mp4 b/apps/web/e2e/fixtures/media-clip.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61abe46b742c15b4dd4ec3b36b137c7967f96e41 GIT binary patch literal 3117 zcmZuz3p|ti8-JPPQbKZwj*$>TQY`l=%bjvLQQ2&hWwWuFTdrZ8Sc*hhbEz~%)JWQ2 zp$?&hWJm6|9EnKfx}A5bbNcr=&*y#K@AG@^@AJH$?F9fp*`E?Y!jXvt01yTQ2(lr8 zWV9OqAi@M9kqQ6+o-Mf@k6(R1gM_75E5I2)H1BP>`b%){_3% zwmRmkO&Jq{75E59VBZiLf`@*Ug^b7)KRhVkBvZ)W7b5VAPVTmoL?1I;A0;G zMF9&)Xp4Sb%`O$18(6^jFnCI?#Vj z)&gyj-)sT^^`cimj0Kqvatz3Sf(+(G6G0RZm=~f4qIoEWf&y_b0lS6*o9d5C0~Z~F zB9R1h{fESGU*8Nd{}3#=QStQz42qIqcLok@{vf$e&qQ5W;WGev9NV?Xel)k*#w$xN zSn+abty-`SyBF2p_&VG6ZPz@hxbaYpluES%?a0#<513jySIzWnosBt9&a^zw^x|W! z79#T+XBI`1R`eSG?IBA1!tK6*$h3r>1N~wrqQ*{#4MsLyaNf8l(Yhi2c{AMjeCH$@ zTimUsF-2}MQ<2dw-+DMFijk3WfQ@jp@2!nn4rYAOt-5pl<=j$InO$zCr(F57&fi= z#f9g}Fen8?tNC*c{fQnBv#aJW<8aNF(nqFaVH*X1tX3D)@RoF0VtY?L{vpikjF5N>q7uQQH3 z^Lb}T={*=Qu~Xc;bz^dntqSV5+%d?wj z?^Kzi2eaU#_H$D=i#*I5yBNW*d2g)zJ!7uQ9oNVYiaf8q)mC=P(;F*_h1N4;;_>%= zIgWLvn!wp3BiM+)OgL;2Tj$Dz7slBAdbHWvcd{ai2Gm0_$+PUd+D`BDg}e@~>7f1i zBa5t$keM$RjGCd=udpl1cbOYer4XS#bVrgs&gJC6nc5R`fnPY%jt*7klW!KWGwM0? z5oLRW=>?XP?H^+k^h(+_3ws7D2pPQobO@IVyB2wobpYy09e6h30>64_IFD`kUSji> z*x$VtuE)A&nl5hXR_Y=2tUh=&l)V%83TvF0%2Ru=nkF~HGlcCIrQ29cb?}qZmDquN z4Jo-5`<8=QvFfep{(BIiX>kLyz1gYRL%eOBzSeru)hktTx!jKnP#KQ1Lx1W(fEu3t zeAIm|PEq;%B&6ZD#37yzxg)Mr%sW*^TX{I|I4D(N9?Ta#S&Sgi&Pr4PhaGpQz zh1TSFc}Dj&Y|xYsPy6IY^`7qzi9c2Y2w!S9i#Lr`d?&fD#PUWaF4t-1O2FMn~%y+A;y*#NOYT-j&bx@pKtopo2P zq`Lzxj!yRRDbb^GdyT`5LfvbF<_gYvN{k}9nG);?9J<0N9&7TfGjfNo(g=4n=uv6} zVkzep5!qBGs#R)z=1p+GoWAd>ZD`7ihce_M3zJ`qgC7RFd}()g9iFPa)@u1e{oPq= zoa0gW-4aMV+?ES-E6wxmc2H>=em^GiAb#sJxOPLg^pSShAAMP>@Kwjts>fVR7|{iy zg<^KrhW(*w$1{CB_t2Bk#})3x-*E^~SuXM$nbVEX&!;7)x6^BzOywV>meypO2(#N< zlL>X^K(kIQ#cYasnOlYYuQVqs zE9%`cJN!+yjahf0>6Y}3YHB^_R7CE|GSNreUP%1{b(2rEOblXbO!vP#-%M|R zUS5fxcXLU)EIEEm{BnHg-O26cuSHi^l%Y17vW!cCG_Pa?b91=PCrA=!I0sL+=;)_8 zhH7&qCu?_0?=*Ri-|9VVMMakEA*&Bi{F9PRBW%X*?M5WjRxYvB=N@W4Dn7H`vhH0A zvgI+53|L)Ijynnfn_}`P!L1%sNL$bK^Z5Yay;td6CA>_rMbSOX_uUTCKH6;Qkfl^B z>-ka6z;$Pcuu_Jl|AjhtlebjGI3`3dqpKk2oQ=rXEz@VQuaw$KOGF`icq7gpJ*eEc z6%RE9hB&TZ=43|S$SGgQe#f3JYxH8sge2a<3WvY zwl+FBgu(HpgvR1%{=8%64RvH~(ZoS71qRlVJx!r8RowjfSPk!{0ZqG>kQaB|ch{sT zJD6_{<~Fa7tIS%klPDyYd|-b#cVACO__nRp~o%9nw}@3vz+Z% zG~Zwv-V*5d?=T=p!2nJgI}wk5(KT`Qv^XywxX!b?hgyTn_!GGVS1|CfC4Q2tNZ7=8 z_pXQuF*A&QYn*m;L;<2tE_L1&S)_EdOi+_hJ`PrYSY)ZLiTGJpsgfE?T}{F0rF^o+2?>GPvnL4bye* z>AugbuB9aYQ=Z-VZE#wBZldDQQ{4|u+Ce=7b?4`) fLFK^;r>9NC+;71~6wX{VI>-D&eYuQ5P<8zq^f}lX literal 0 HcmV?d00001 diff --git a/apps/web/e2e/media-pipeline.spec.ts b/apps/web/e2e/media-pipeline.spec.ts new file mode 100644 index 0000000..dc712cc --- /dev/null +++ b/apps/web/e2e/media-pipeline.spec.ts @@ -0,0 +1,79 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "@playwright/test"; +import { captureScreenshot, completeOnboarding, signup } from "./helpers"; + +test.describe.configure({ mode: "serial" }); + +// VP9 in MP4: Playwright's Chromium has no H.264 decoder, real browsers play both. +const clip = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "media-clip.mp4"); +const pixel = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", +); + +test("an uploaded video streams from the artifact route and plays inline", async ({ + page, +}, testInfo) => { + const stamp = Date.now(); + await signup(page, `media-video-${stamp}@rakazo.test`, "password12", "Media Video"); + await completeOnboarding(page); + + const upload = page.waitForResponse( + (response) => + response.request().method() === "POST" && /\/api\/artifacts\?/.test(response.url()), + ); + await page.locator('input[type="file"]').setInputFiles(clip); + await expect(page.getByRole("button", { name: "Remove media-clip.mp4" })).toBeVisible(); + await page.getByRole("combobox", { name: /Message/ }).fill("here is the clip"); + await page.keyboard.press("Enter"); + expect((await upload).status()).toBe(201); + + const video = page.getByTestId("media-video").locator("video"); + await expect(video).toBeVisible({ timeout: 30_000 }); + await expect(video).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/); + await expect + .poll(() => video.evaluate((element: HTMLVideoElement) => element.readyState), { + timeout: 15_000, + }) + .toBeGreaterThanOrEqual(1); + expect(await video.evaluate((element: HTMLVideoElement) => element.duration)).toBeCloseTo(1, 0); + + await video.evaluate(async (element: HTMLVideoElement) => { + element.muted = true; + await element.play(); + }); + await expect + .poll(() => video.evaluate((element: HTMLVideoElement) => element.currentTime)) + .toBeGreaterThan(0); + await video.evaluate((element: HTMLVideoElement) => { + element.pause(); + element.currentTime = 0.5; + element.scrollIntoView({ block: "center" }); + }); + await captureScreenshot(page, testInfo, "media-video-block"); +}); + +test("the image lightbox downloads the original from the artifact route", async ({ page }) => { + const stamp = Date.now(); + await signup(page, `media-image-${stamp}@rakazo.test`, "password12", "Media Image"); + await completeOnboarding(page); + + await page + .locator('input[type="file"]') + .setInputFiles({ name: "pixel.png", mimeType: "image/png", buffer: pixel }); + await page.getByRole("combobox", { name: /Message/ }).fill("a pixel"); + await page.keyboard.press("Enter"); + + const thumbnail = page.getByRole("img", { name: "pixel.png" }); + await expect(thumbnail).toBeVisible({ timeout: 30_000 }); + await expect(thumbnail).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/); + await thumbnail.click(); + + const dialog = page.getByRole("dialog", { name: "pixel.png" }); + const download = dialog.getByRole("link", { name: "Download pixel.png" }); + await expect(download).toHaveAttribute("href", /\/api\/artifacts\/[^/?]+\?download=1$/); + const saved = page.waitForEvent("download"); + await download.click(); + expect((await saved).suggestedFilename()).toBe("pixel.png"); +}); diff --git a/apps/web/src/components/ArtifactFileCard.tsx b/apps/web/src/components/ArtifactFileCard.tsx index b36eb65..41c11c1 100644 --- a/apps/web/src/components/ArtifactFileCard.tsx +++ b/apps/web/src/components/ArtifactFileCard.tsx @@ -1,19 +1,12 @@ -import { i18n } from "@lingui/core"; -import { t } from "@lingui/core/macro"; import { Trans, useLingui } from "@lingui/react/macro"; import { ChatMarkdown } from "@rakazo/chat-ui/web"; import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web"; import { Download, FileText, X } from "lucide-react"; import { type RefObject, useEffect, useRef, useState } from "react"; -import { - type ArtifactTarget, - downloadArtifact, - downloadArtifactBytes, - fetchArtifactBytes, -} from "../lib/artifact-open"; +import { downloadArtifact, fetchArtifactBytes } from "../lib/artifact-open"; +import { formatBytes } from "../lib/format-bytes"; type ArtifactFileCardProps = { - target: ArtifactTarget; artifactId: string; name: string; mimeType: string; @@ -26,23 +19,14 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) { const previewButton = useRef(null); const closeButton = useRef(null); const [previewOpen, setPreviewOpen] = useState(false); - const [downloadError, setDownloadError] = useState(null); - - async function startDownload() { - setDownloadError(null); - try { - await downloadArtifact(props.target, props.artifactId, props.name, props.mimeType); - } catch { - setDownloadError(t`Could not download ${props.name}. Try again.`); - } - } + const startDownload = () => downloadArtifact(props.artifactId, props.name); if (!markdown) { return (
- {downloadError ? : null} ); } @@ -80,13 +63,12 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) { type="button" aria-label={t`Download ${props.name}`} title={t`Download ${props.name}`} - onClick={() => void startDownload()} + onClick={startDownload} className="grid w-14 shrink-0 place-items-center border-l border-border text-muted-foreground hover:bg-accent hover:text-foreground" > - {downloadError ? : null} }) { const { t } = useLingui(); - const [downloadError, setDownloadError] = useState(null); const [state, setState] = useState< | { status: "loading" } - | { status: "ready"; bytes: Uint8Array; markdown: string } + | { status: "ready"; markdown: string } | { status: "error"; message: string } >({ status: "loading" }); - const targetBotId = "botId" in target ? target.botId : undefined; - const targetGroupId = "groupId" in target ? target.groupId : undefined; useEffect(() => { let cancelled = false; - const artifactTarget: ArtifactTarget = - targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! }; - void fetchArtifactBytes(artifactTarget, artifactId) + void fetchArtifactBytes(artifactId) .then((bytes) => { if (cancelled) return; try { const markdown = new TextDecoder("utf-8", { fatal: true }).decode(bytes); - setState({ status: "ready", bytes, markdown }); + setState({ status: "ready", markdown }); } catch { setState({ status: "error", message: t`This file is not valid UTF-8 Markdown.` }); } @@ -143,7 +118,7 @@ function MarkdownPreview({ return () => { cancelled = true; }; - }, [artifactId, targetBotId, targetGroupId, t]); + }, [artifactId, t]); return ( <> @@ -157,17 +132,7 @@ function MarkdownPreview({ className="rounded-full text-muted-foreground" aria-label={t`Download ${name}`} title={t`Download ${name}`} - onClick={() => - void (async () => { - setDownloadError(null); - try { - if (state.status === "ready") downloadArtifactBytes(name, mimeType, state.bytes); - else await downloadArtifact(target, artifactId, name, mimeType); - } catch { - setDownloadError(t`Could not download ${name}. Try again.`); - } - })() - } + onClick={() => downloadArtifact(artifactId, name)} > @@ -181,11 +146,6 @@ function MarkdownPreview({ - {downloadError ? ( -
- -
- ) : null}
{state.status === "loading" ? ( @@ -204,23 +164,3 @@ function MarkdownPreview({ ); } - -function DownloadError({ message }: { message: string }) { - return ( -
- {message} -
- ); -} - -function formatBytes(size: number) { - const locale = i18n.locale || "en"; - if (size < 1024) return t`${size} B`; - const format = (value: number) => - new Intl.NumberFormat(locale, { - minimumFractionDigits: 1, - maximumFractionDigits: 1, - }).format(value); - if (size < 1024 * 1024) return t`${format(size / 1024)} KB`; - return t`${format(size / (1024 * 1024))} MB`; -} diff --git a/apps/web/src/components/media/MediaPlayer.test.tsx b/apps/web/src/components/media/MediaPlayer.test.tsx new file mode 100644 index 0000000..0be2403 --- /dev/null +++ b/apps/web/src/components/media/MediaPlayer.test.tsx @@ -0,0 +1,84 @@ +// @vitest-environment jsdom + +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@lingui/react/macro", () => { + const t = (parts: TemplateStringsArray, ...values: unknown[]) => + parts.reduce( + (text, part, index) => text + part + (index < values.length ? values[index] : ""), + "", + ); + return { useLingui: () => ({ t }) }; +}); + +import { MediaPlayer } from "./MediaPlayer"; + +let container: HTMLDivElement; +let root: Root; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +describe("MediaPlayer", () => { + it("loads nothing before it is near the viewport and fits vertical video to the column", () => { + const html = renderToStaticMarkup( + , + ); + expect(html).toContain('aria-label="clip.mp4"'); + expect(html).not.toContain("src="); + expect(html).toContain("width:270px"); + expect(html).toContain("aspect-ratio:270 / 480"); + }); + + it("streams video with its poster from the artifact route once visible", async () => { + vi.stubGlobal( + "IntersectionObserver", + class { + constructor(private readonly callback: IntersectionObserverCallback) {} + observe() { + this.callback([{ isIntersecting: true } as IntersectionObserverEntry], this as never); + } + disconnect() {} + }, + ); + await act(async () => + root.render( + , + ), + ); + const video = container.querySelector("video")!; + expect(video.getAttribute("src")).toBe("/api/artifacts/art-1"); + expect(video.getAttribute("poster")).toBe("/api/artifacts/art-2"); + expect(video.getAttribute("preload")).toBe("metadata"); + expect(video.hasAttribute("controls")).toBe(true); + }); + + it("falls back to a download link when the browser cannot play the file", async () => { + await act(async () => + root.render(), + ); + const audio = container.querySelector("audio")!; + expect(audio.getAttribute("src")).toBe("/api/artifacts/art-3"); + await act(async () => { + audio.dispatchEvent(new Event("error")); + }); + expect(container.querySelector("audio")).toBeNull(); + const link = container.querySelector("a")!; + expect(link.getAttribute("href")).toBe("/api/artifacts/art-3?download=1"); + expect(link.getAttribute("download")).toBe("memo.ogg"); + expect(link.getAttribute("aria-label")).toBe("Download memo.ogg"); + }); +}); diff --git a/apps/web/src/components/media/MediaPlayer.tsx b/apps/web/src/components/media/MediaPlayer.tsx new file mode 100644 index 0000000..d793ba4 --- /dev/null +++ b/apps/web/src/components/media/MediaPlayer.tsx @@ -0,0 +1,121 @@ +import { useLingui } from "@lingui/react/macro"; +import { buttonVariants } from "@rakazo/ui-web"; +import { Download } from "lucide-react"; +import { type RefObject, useEffect, useRef, useState } from "react"; +import { artifactUrl } from "../../lib/artifact-url"; + +export type MediaPlayerProps = { + artifactId: string; + kind: "video" | "audio"; + name: string; + posterArtifactId?: string; + width?: number; + height?: number; +}; + +const MAX_VIDEO_WIDTH = 420; +const MAX_VIDEO_HEIGHT = 480; + +/** Plays a video or audio artifact in place; loads nothing until it is near the viewport. */ +export function MediaPlayer({ + artifactId, + kind, + name, + posterArtifactId, + width, + height, +}: MediaPlayerProps) { + const container = useRef(null); + const near = useNearViewport(container); + const [failed, setFailed] = useState(false); + const src = near ? artifactUrl(artifactId) : undefined; + + if (failed) return ; + + if (kind === "audio") { + return ( +
+ {/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */} +
+ ); + } + + const box = videoBox(width, height); + return ( +
+ {/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */} +
+ ); +} + +/** Fits the frame into the chat column, defaulting to 16:9 until the size is known. */ +function videoBox(width?: number, height?: number) { + const w = width && width > 0 ? width : 16; + const h = height && height > 0 ? height : 9; + const scale = Math.min(MAX_VIDEO_WIDTH / w, MAX_VIDEO_HEIGHT / h); + return { width: Math.round(w * scale), height: Math.round(h * scale) }; +} + +function MediaFallback({ artifactId, name }: { artifactId: string; name: string }) { + const { t } = useLingui(); + return ( +
+ {name} + + + +
+ ); +} + +function useNearViewport(ref: RefObject) { + const [near, setNear] = useState(false); + useEffect(() => { + const element = ref.current; + if (!element || typeof IntersectionObserver === "undefined") { + setNear(true); + return; + } + const observer = new IntersectionObserver( + (entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + setNear(true); + observer.disconnect(); + } + }, + { rootMargin: "320px" }, + ); + observer.observe(element); + return () => observer.disconnect(); + }, [ref]); + return near; +} diff --git a/apps/web/src/lib/artifact-open.ts b/apps/web/src/lib/artifact-open.ts index a1ed8eb..e832642 100644 --- a/apps/web/src/lib/artifact-open.ts +++ b/apps/web/src/lib/artifact-open.ts @@ -1,40 +1,29 @@ -import { rpc } from "./rpc"; +import { t } from "@lingui/core/macro"; +import { artifactUrl } from "./artifact-url"; export type ArtifactTarget = { botId: string } | { groupId: string }; -export function decodeArtifactBase64(contentBase64: string): Uint8Array { - const binary = atob(contentBase64); - const bytes = new Uint8Array(binary.length); - for (let index = 0; index < binary.length; index += 1) { - bytes[index] = binary.charCodeAt(index); - } - return bytes; -} - -export async function fetchArtifactBytes( - target: ArtifactTarget, - artifactId: string, -): Promise { - const artifact = await rpc.artifacts.get({ ...target, artifactId }); - return decodeArtifactBase64(artifact.contentBase64); +export async function fetchArtifactBytes(artifactId: string): Promise { + const response = await fetch(artifactUrl(artifactId), { credentials: "include" }); + if (!response.ok) throw new Error(t`Could not load this file.`); + return new Uint8Array(await response.arrayBuffer()); } export function downloadArtifactBytes(name: string, mimeType: string, bytes: Uint8Array): void { const blob = new Blob([new Uint8Array(bytes)], { type: mimeType }); const url = URL.createObjectURL(blob); + saveUrl(url, name); + window.setTimeout(() => URL.revokeObjectURL(url), 60_000); +} + +/** Lets the browser stream the download from the artifact route instead of buffering it. */ +export function downloadArtifact(artifactId: string, name: string): void { + saveUrl(artifactUrl(artifactId, { download: true }), name); +} + +function saveUrl(url: string, name: string) { const anchor = document.createElement("a"); anchor.href = url; anchor.download = name; anchor.click(); - window.setTimeout(() => URL.revokeObjectURL(url), 60_000); -} - -export async function downloadArtifact( - target: ArtifactTarget, - artifactId: string, - name: string, - mimeType: string, -): Promise { - const bytes = await fetchArtifactBytes(target, artifactId); - downloadArtifactBytes(name, mimeType, bytes); } diff --git a/apps/web/src/lib/artifact-upload.test.ts b/apps/web/src/lib/artifact-upload.test.ts new file mode 100644 index 0000000..5c0440b --- /dev/null +++ b/apps/web/src/lib/artifact-upload.test.ts @@ -0,0 +1,83 @@ +import { ATTACHMENT_MAX_BYTES } from "@rakazo/contracts"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@lingui/core/macro", () => ({ + t: (parts: TemplateStringsArray, ...values: unknown[]) => + parts.reduce( + (text, part, index) => text + part + (index < values.length ? values[index] : ""), + "", + ), +})); +const create = vi.hoisted(() => vi.fn()); +vi.mock("./rpc", () => ({ + rpc: { artifacts: { create } }, + withSpaceHeaders: (init: HeadersInit) => new Headers(init), +})); + +import { + attachmentMaxBytes, + uploadArtifactFile, + uploadAttachment, + usesStreamingUpload, +} from "./artifact-upload"; + +const artifact = { + id: "art-1", + botId: "bot-1", + groupId: null, + runId: null, + name: "clip.mp4", + mimeType: "video/mp4", + size: 4, + createdAt: "2026-10-03T12:00:00.000Z", +}; + +afterEach(() => { + vi.unstubAllGlobals(); + create.mockReset(); +}); + +describe("artifact uploads", () => { + it("streams media and large files and keeps small files on the inline RPC", () => { + expect(usesStreamingUpload(10, "video/mp4")).toBe(true); + expect(usesStreamingUpload(10, "audio/mpeg")).toBe(true); + expect(usesStreamingUpload(10, "image/png")).toBe(false); + expect(usesStreamingUpload(ATTACHMENT_MAX_BYTES + 1, "image/png")).toBe(true); + expect(attachmentMaxBytes("video/mp4")).toBe(500 * 1024 * 1024); + expect(attachmentMaxBytes("text/plain")).toBe(25 * 1024 * 1024); + }); + + it("posts the raw file with its target, name and type", async () => { + const fetch = vi.fn(async () => Response.json(artifact, { status: 201 })); + vi.stubGlobal("fetch", fetch); + const file = new File(["data"], "clip.mp4", { type: "video/mp4" }); + + await expect(uploadAttachment({ groupId: "group-1" }, file, "video/mp4")).resolves.toEqual( + artifact, + ); + const [url, init] = fetch.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe("/api/artifacts?groupId=group-1&name=clip.mp4"); + expect(init.method).toBe("POST"); + expect(init.credentials).toBe("include"); + expect(new Headers(init.headers).get("content-type")).toBe("video/mp4"); + expect(init.body).toBe(file); + expect(create).not.toHaveBeenCalled(); + }); + + it("surfaces the server's reason, or a generic one", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ error: "File is too large." }, { status: 413 })), + ); + await expect( + uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"), + ).rejects.toThrow("File is too large."); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("bad gateway", { status: 502 })), + ); + await expect( + uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"), + ).rejects.toThrow("Could not upload big.mp4. Try again."); + }); +}); diff --git a/apps/web/src/lib/artifact-upload.ts b/apps/web/src/lib/artifact-upload.ts new file mode 100644 index 0000000..12d11e4 --- /dev/null +++ b/apps/web/src/lib/artifact-upload.ts @@ -0,0 +1,68 @@ +import { t } from "@lingui/core/macro"; +import { + ARTIFACT_MAX_BYTES_BY_KIND, + ArtifactSchema, + ATTACHMENT_MAX_BYTES, + attachmentKindOf, +} from "@rakazo/contracts"; +import type { ArtifactTarget } from "./artifact-open"; +import { rpc, withSpaceHeaders } from "./rpc"; + +type UploadedArtifact = ReturnType; + +/** Largest file of this type the composer accepts. */ +export function attachmentMaxBytes(mimeType: string): number { + return ARTIFACT_MAX_BYTES_BY_KIND[attachmentKindOf(mimeType)]; +} + +/** Media and anything above the inline RPC cap upload as a raw stream instead of base64. */ +export function usesStreamingUpload(size: number, mimeType: string): boolean { + const kind = attachmentKindOf(mimeType); + return size > ATTACHMENT_MAX_BYTES || kind === "video" || kind === "audio"; +} + +export async function uploadAttachment( + target: ArtifactTarget, + file: File, + mimeType: string, +): Promise { + if (usesStreamingUpload(file.size, mimeType)) { + return uploadArtifactFile(target, file, file.name, mimeType); + } + const contentBase64 = await readFileAsBase64(file); + return rpc.artifacts.create({ ...target, name: file.name, mimeType, contentBase64 }); +} + +/** Uploads a file as a raw streamed body to the artifact route. */ +export async function uploadArtifactFile( + target: ArtifactTarget, + file: Blob, + name: string, + mimeType: string, +): Promise { + const query = new URLSearchParams({ ...target, name }); + const response = await fetch(`/api/artifacts?${query}`, { + method: "POST", + headers: withSpaceHeaders({ "content-type": mimeType }), + credentials: "include", + body: file, + }); + const body: unknown = await response.json().catch(() => null); + const artifact = ArtifactSchema.safeParse(body); + if (response.ok && artifact.success) return artifact.data; + const error = (body as { error?: unknown } | null)?.error; + throw new Error(typeof error === "string" ? error : t`Could not upload ${name}. Try again.`); +} + +function readFileAsBase64(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const result = typeof reader.result === "string" ? reader.result : ""; + const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result; + resolve(base64); + }; + reader.onerror = () => reject(reader.error ?? new Error("Failed to read file")); + reader.readAsDataURL(file); + }); +} diff --git a/apps/web/src/lib/artifact-url.ts b/apps/web/src/lib/artifact-url.ts index a87f4d4..5dff453 100644 --- a/apps/web/src/lib/artifact-url.ts +++ b/apps/web/src/lib/artifact-url.ts @@ -1,38 +1,8 @@ -import { t } from "@lingui/core/macro"; -import { ArtifactSchema } from "@rakazo/contracts"; -import type { ArtifactTarget } from "./artifact-open"; -import { withSpaceHeaders } from "./rpc"; - -const ARTIFACT_PATH = "/api/artifacts"; - /** * Same-origin URL that streams an artifact with Range support, for `
+ ); + } + if (block.kind === "video" || block.kind === "audio") { + return ( +
+
); @@ -7856,7 +7873,6 @@ const MessageView = memo(function MessageView({ className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`} > { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const result = typeof reader.result === "string" ? reader.result : ""; - const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result; - resolve(base64); - }; - reader.onerror = () => reject(reader.error ?? new Error("Failed to read file")); - reader.readAsDataURL(file); - }); -} diff --git a/apps/web/src/pages/shell/message-cards.tsx b/apps/web/src/pages/shell/message-cards.tsx index 14131d6..294bbd4 100644 --- a/apps/web/src/pages/shell/message-cards.tsx +++ b/apps/web/src/pages/shell/message-cards.tsx @@ -1,11 +1,18 @@ import { Trans, useLingui } from "@lingui/react/macro"; import type { MessageBlock } from "@rakazo/contracts"; import { abortableDelay } from "@rakazo/core"; -import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web"; -import { X } from "lucide-react"; +import { + Button, + buttonVariants, + Dialog, + DialogClose, + DialogContent, + DialogTitle, +} from "@rakazo/ui-web"; +import { Download, X } from "lucide-react"; import { useEffect, useRef, useState } from "react"; import { BuiCard, SuccessPop } from "../../components/ai/primitives"; -import { type ArtifactTarget, decodeArtifactBase64 } from "../../lib/artifact-open"; +import { artifactUrl } from "../../lib/artifact-url"; import { chartViewport } from "../../lib/chart-viewport"; import { connectMcpOauth } from "../../lib/mcp-connect"; import { rpc } from "../../lib/rpc"; @@ -461,104 +468,65 @@ export function ChartBlockView({ ); } -export function ArtifactImage({ - target, - artifactId, - name, -}: { - target: ArtifactTarget; - artifactId: string; - name: string; -}) { +export function ArtifactImage({ artifactId, name }: { artifactId: string; name: string }) { const { t } = useLingui(); - const [src, setSrc] = useState(null); const [open, setOpen] = useState(false); - const [visible, setVisible] = useState(false); - const container = useRef(null); - const targetBotId = "botId" in target ? target.botId : undefined; - const targetGroupId = "groupId" in target ? target.groupId : undefined; + const [failed, setFailed] = useState(false); + const src = artifactUrl(artifactId); - useEffect(() => { - const element = container.current; - if (!element || typeof IntersectionObserver === "undefined") { - setVisible(true); - return; - } - const observer = new IntersectionObserver( - (entries) => { - if (entries.some((entry) => entry.isIntersecting)) { - setVisible(true); - observer.disconnect(); - } - }, - { rootMargin: "320px" }, + if (failed) { + return ( +
+ {name} +
); - observer.observe(element); - return () => observer.disconnect(); - }, []); - - useEffect(() => { - if (!visible) return; - let cancelled = false; - let objectUrl: string | null = null; - setSrc(null); - void rpc.artifacts - .get( - targetBotId - ? { botId: targetBotId, artifactId } - : { groupId: targetGroupId ?? "", artifactId }, - ) - .then((artifact) => { - const bytes = decodeArtifactBase64(artifact.contentBase64); - objectUrl = URL.createObjectURL( - new Blob([new Uint8Array(bytes)], { type: artifact.mimeType }), - ); - if (cancelled) URL.revokeObjectURL(objectUrl); - else setSrc(objectUrl); - }) - .catch(() => undefined); - return () => { - cancelled = true; - if (objectUrl) URL.revokeObjectURL(objectUrl); - }; - }, [artifactId, targetBotId, targetGroupId, visible]); - + } return ( -
- {src ? ( - + + - {name} - - ) : ( -
- {name} -
- )} - {src ? ( - - - {name} + {name} +
+ + + } + render={
 - ) : null} +
+ {name} + + ); }