diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index d3ff4e7..f9f0e2f 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -609,6 +609,7 @@ export async function createApp( artifacts, sessionUserId: async (request) => (await auth.api.getSession({ headers: sessionHeaders(request) }))?.user?.id ?? null, + trustedOrigin: (origin) => isTrustedOrigin(origin, env), }); mountWebhookHttpRoutes(app, { prisma, secrets, events, jobs }); // Shared with stop so a shutdown during retry delays does not restart polling. diff --git a/apps/api/src/artifact-http.test.ts b/apps/api/src/artifact-http.test.ts index a1d6d44..0a2a6fc 100644 --- a/apps/api/src/artifact-http.test.ts +++ b/apps/api/src/artifact-http.test.ts @@ -77,6 +77,7 @@ describe("artifact HTTP routes", () => { prisma: prisma as unknown as PrismaClient, artifacts: artifacts as unknown as ArtifactStore, sessionUserId, + trustedOrigin: (origin) => origin === "http://127.0.0.1:5173", }); return { hono, prisma, artifacts }; } @@ -95,6 +96,19 @@ describe("artifact HTTP routes", () => { expect(artifacts.putStream).not.toHaveBeenCalled(); }); + it("refuses uploads from untrusted origins before reading the session", async () => { + const sessionUserId = vi.fn(async () => "user-1"); + const { hono, artifacts } = app(sessionUserId); + const response = await hono.request("/api/artifacts?botId=bot-1&name=a.txt", { + method: "POST", + headers: { "content-type": "text/plain", origin: "https://evil.example" }, + body: "data", + }); + expect(response.status).toBe(403); + expect(sessionUserId).not.toHaveBeenCalled(); + expect(artifacts.putStream).not.toHaveBeenCalled(); + }); + it("answers ids that are not artifact ids with 404 without a lookup", async () => { const { hono, prisma } = app(async () => "user-1"); const response = await hono.request(`/api/artifacts/${encodeURIComponent("../etc/passwd")}`); diff --git a/apps/api/src/artifact-http.ts b/apps/api/src/artifact-http.ts index b0c048c..d8b9df5 100644 --- a/apps/api/src/artifact-http.ts +++ b/apps/api/src/artifact-http.ts @@ -20,6 +20,8 @@ export type ArtifactHttpDeps = { artifacts: ArtifactStore; /** The signed-in user (cookie or bearer session), or null. */ sessionUserId: (request: Request) => Promise; + /** Same rule as CORS. Raw uploads can be CORS "simple" requests, so the origin is checked. */ + trustedOrigin: (origin: string) => boolean; }; /** Generated ids only; anything else is not an artifact. */ @@ -208,10 +210,12 @@ export function mountArtifactHttpRoutes(app: Hono, deps: ArtifactHttpDeps) { app.post(ARTIFACT_HTTP_PATH, async (c) => { const request = c.req.raw; - const reject = (status: 400 | 401 | 404 | 413 | 415, error: string) => { + const reject = (status: 400 | 401 | 403 | 404 | 413 | 415, error: string) => { cancelBody(request.body); return c.json({ error }, status); }; + const origin = c.req.header("origin"); + if (origin && !deps.trustedOrigin(origin)) return reject(403, "Forbidden"); const userId = await deps.sessionUserId(request); if (!userId) return reject(401, "Unauthorized"); const actor = await requireMembership( diff --git a/apps/api/src/artifacts.ts b/apps/api/src/artifacts.ts index b0804a0..9dec9c4 100644 --- a/apps/api/src/artifacts.ts +++ b/apps/api/src/artifacts.ts @@ -11,6 +11,12 @@ import { } from "@rakazo/core"; import { IsolationError, type PrismaClient } from "@rakazo/db"; +/** + * Base64 reads load the whole file into memory; larger media is served by + * GET /api/artifacts/:id instead. + */ +export const ARTIFACT_INLINE_READ_MAX_BYTES = 25 * 1024 * 1024; + function adapterContext(actor: Actor, botId: string, operationId: string) { return { operationId, @@ -131,6 +137,9 @@ async function readArtifact( }, contextBotId: string, ) { + if (row.size > ARTIFACT_INLINE_READ_MAX_BYTES) { + throw new AttachmentValidationError("This file is too large to load inline."); + } const bytes = await artifacts.get( row.storageKey, adapterContext(actor, contextBotId, `artifact-get:${row.id}`), diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 4255c3d..f14a76f 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -4670,24 +4670,26 @@ export function createRouter(deps: RouterDeps) { } }), get: authed.artifacts.get.handler(async ({ context, input }) => { - if (input.groupId) { - const group = await groupRepos.getGroupTarget(context.actor, input.groupId); - const contextBotId = group.members[0]?.bot.id; - if (!contextBotId) throw new IsolationError(); - return getSpaceArtifact(deps, context.actor, { - artifactId: input.artifactId, - groupId: input.groupId, - contextBotId, - }); - } - await repos.getBot(context.actor, input.botId!); try { + if (input.groupId) { + const group = await groupRepos.getGroupTarget(context.actor, input.groupId); + const contextBotId = group.members[0]?.bot.id; + if (!contextBotId) throw new IsolationError(); + return await getSpaceArtifact(deps, context.actor, { + artifactId: input.artifactId, + groupId: input.groupId, + contextBotId, + }); + } + await repos.getBot(context.actor, input.botId!); return await getOwnedArtifact(deps, context.actor, { botId: input.botId!, artifactId: input.artifactId, }); } catch (error) { - if (error instanceof IsolationError) throw error; + if (error instanceof AttachmentValidationError) { + throw new ORPCError("BAD_REQUEST", { message: error.message }); + } throw error; } }), diff --git a/apps/mobile/app/thread.tsx b/apps/mobile/app/thread.tsx index c2eb734..c8ae7d4 100644 --- a/apps/mobile/app/thread.tsx +++ b/apps/mobile/app/thread.tsx @@ -77,6 +77,7 @@ import { type MobileMessagePage, type MobileSnapshot, mergeMobileSnapshot, + messageAttachments, messagingProviderLabel, prependMobileMessagePage, rpc, @@ -2243,7 +2244,11 @@ function previewMessageText(message: MobileMessage): string { .join(" ") .trim(); if (text) return text; - if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) { + if ( + message.blocks.some((block) => + ["image", "file", "video", "audio", "media"].includes(block.kind), + ) + ) { return t("Attachment"); } return t("Message"); @@ -2676,9 +2681,7 @@ const MessageBubble = memo(function MessageBubble({ ); } - const attachments = message.blocks.filter( - (block) => block.kind === "image" || block.kind === "file", - ); + const attachments = messageAttachments(message.blocks); const caption = message.blocks .flatMap((block) => { if (block.kind === "channel_message" && block.text) { @@ -2686,6 +2689,7 @@ const MessageBubble = memo(function MessageBubble({ `${messagingProviderLabel(block.provider, block.transport)} ยท ${block.fromLabel}: ${block.text}`, ]; } + if (block.kind === "media") return [block.title]; return block.kind === "text" && block.text ? [block.text] : []; }) .join("\n"); @@ -2797,7 +2801,8 @@ const MessageBubble = memo(function MessageBubble({ fontSize: 15, }} > - ๐Ÿ“Ž {attachment.name ?? t("File")} + {attachment.kind === "video" ? "๐ŸŽฌ" : attachment.kind === "audio" ? "๐ŸŽต" : "๐Ÿ“Ž"}{" "} + {attachment.name ?? t("File")} {attachment.size ? ( { ).toBe("Slack ยท Alex: Hello from the group"); }); + it("falls back to plain text and file rows for video, audio and media cards", () => { + const blocks = [ + { + kind: "video" as const, + artifactId: "vid-1", + mimeType: "video/mp4", + name: "clip.mp4", + size: 10, + width: 1080, + height: 1920, + }, + { + kind: "audio" as const, + artifactId: "aud-1", + mimeType: "audio/mpeg", + name: "memo.mp3", + size: 5, + }, + { + kind: "media" as const, + jobId: "job-1", + operation: "export_video" as const, + status: "done" as const, + title: "Vertical cut", + outputs: [ + { + artifactId: "out-1", + kind: "video" as const, + mimeType: "video/mp4", + name: "cut.mp4", + size: 7, + }, + ], + }, + ]; + expect(blockText(mobileMessage("media-1", blocks))).toBe( + "[video: clip.mp4]\n[audio: memo.mp3]\nVertical cut: done", + ); + expect(messageAttachments(blocks).map((item) => [item.kind, item.artifactId])).toEqual([ + ["video", "vid-1"], + ["audio", "aud-1"], + ["video", "out-1"], + ]); + }); + it("deduplicates durable messages and replaces matching transient subagent state", () => { const initial = snapshot([ mobileMessage("message-1", [{ kind: "text", text: "old" }]), diff --git a/apps/mobile/lib/api.ts b/apps/mobile/lib/api.ts index f328abc..ac39cf1 100644 --- a/apps/mobile/lib/api.ts +++ b/apps/mobile/lib/api.ts @@ -860,6 +860,24 @@ export function copyableMobileMessageText(message: MobileMessage): string { .trim(); } +export type MobileAttachment = { + kind: "image" | "video" | "audio" | "file"; + artifactId: string; + name: string; + mimeType: string; + size?: number; +}; + +/** Files a message carries, including the outputs of a finished media card, as tappable rows. */ +export function messageAttachments(blocks: readonly MessageBlock[]): MobileAttachment[] { + return blocks.flatMap((block): MobileAttachment[] => { + if (block.kind === "image") return [{ ...block, kind: "image" }]; + if (block.kind === "video" || block.kind === "audio" || block.kind === "file") return [block]; + if (block.kind === "media") return block.outputs; + return []; + }); +} + export function blockText(message: MobileMessage) { return message.blocks .map((block) => { @@ -879,6 +897,10 @@ export function blockText(message: MobileMessage) { if (block.kind === "file") { return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`; } + if (block.kind === "video" || block.kind === "audio") { + return `[${block.kind}: ${block.name ?? "attachment"}]`; + } + if (block.kind === "media") return `${block.title}: ${block.status}`; if (block.kind === "steps") { return (block.steps ?? []) .map((step) => `${step.label}${step.count > 1 ? ` ร—${step.count}` : ""}`) diff --git a/apps/mobile/lib/artifact-open.ts b/apps/mobile/lib/artifact-open.ts index 17f0e00..ab18d9a 100644 --- a/apps/mobile/lib/artifact-open.ts +++ b/apps/mobile/lib/artifact-open.ts @@ -1,6 +1,7 @@ +import { attachmentKindOf } from "@rakazo/contracts"; import { File, Paths } from "expo-file-system"; import * as Sharing from "expo-sharing"; -import { rpc } from "./api"; +import { captureApiRequestContext, rpc } from "./api"; import { artifactCacheFileName } from "./artifact-file"; import { t } from "./i18n"; @@ -11,6 +12,16 @@ async function cacheMobileArtifact( artifactId: string, mimeType: string, ): Promise { + const kind = attachmentKindOf(mimeType); + if (kind === "video" || kind === "audio") { + // Media can be hundreds of megabytes: stream it to disk instead of a base64 RPC payload. + const { apiBase, headers } = await captureApiRequestContext(); + return File.downloadFileAsync( + `${apiBase}/api/artifacts/${encodeURIComponent(artifactId)}`, + new File(Paths.cache, artifactCacheFileName(artifactId, mimeType)), + { headers, idempotent: true }, + ); + } const artifact = await rpc<{ contentBase64: string }>("artifacts/get", { ...target, artifactId, diff --git a/apps/web/e2e/fixtures/media-clip.mp4 b/apps/web/e2e/fixtures/media-clip.mp4 new file mode 100644 index 0000000..61abe46 Binary files /dev/null and b/apps/web/e2e/fixtures/media-clip.mp4 differ diff --git a/apps/web/e2e/media-pipeline.spec.ts b/apps/web/e2e/media-pipeline.spec.ts new file mode 100644 index 0000000..eb63fe8 --- /dev/null +++ b/apps/web/e2e/media-pipeline.spec.ts @@ -0,0 +1,82 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "@playwright/test"; +import { captureScreenshot, completeOnboarding, signup } from "./helpers"; + +test.describe.configure({ mode: "serial" }); + +// VP9 in MP4: Playwright's Chromium has no H.264 decoder, real browsers play both. +const clip = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "media-clip.mp4"); +const pixel = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", + "base64", +); + +test("an uploaded video streams from the artifact route and plays inline", async ({ + page, +}, testInfo) => { + const stamp = Date.now(); + await signup(page, `media-video-${stamp}@rakazo.test`, "password12", "Media Video"); + await completeOnboarding(page); + + const upload = page.waitForResponse( + (response) => + response.request().method() === "POST" && /\/api\/artifacts\?/.test(response.url()), + ); + await page.locator('input[type="file"]').setInputFiles(clip); + await expect(page.getByRole("button", { name: "Remove media-clip.mp4" })).toBeVisible(); + await page.getByRole("combobox", { name: /Message/ }).fill("here is the clip"); + await page.keyboard.press("Enter"); + expect((await upload).status()).toBe(201); + + const video = page.getByTestId("media-video").locator("video"); + await expect(video).toBeVisible({ timeout: 30_000 }); + await expect(video).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/); + await expect + .poll(() => video.evaluate((element: HTMLVideoElement) => element.readyState), { + timeout: 15_000, + }) + .toBeGreaterThanOrEqual(1); + expect(await video.evaluate((element: HTMLVideoElement) => element.duration)).toBeCloseTo(1, 0); + + await video.evaluate(async (element: HTMLVideoElement) => { + element.muted = true; + await element.play(); + }); + await expect + .poll(() => video.evaluate((element: HTMLVideoElement) => element.currentTime)) + .toBeGreaterThan(0); + await video.evaluate((element: HTMLVideoElement) => { + element.pause(); + element.currentTime = 0.5; + element.scrollIntoView({ block: "center" }); + }); + await captureScreenshot(page, testInfo, "media-video-block"); +}); + +test("an image in the main conversation opens and downloads from the artifact route", async ({ + page, +}) => { + const stamp = Date.now(); + await signup(page, `media-image-${stamp}@rakazo.test`, "password12", "Media Image"); + // The main conversation is a group thread, so this covers group uploads and reads. + await completeOnboarding(page, undefined, { mainConversation: true }); + + await page + .locator('input[type="file"]') + .setInputFiles({ name: "pixel.png", mimeType: "image/png", buffer: pixel }); + await page.getByRole("combobox", { name: /Message/ }).fill("a pixel"); + await page.keyboard.press("Enter"); + + const thumbnail = page.getByTestId("transcript").getByRole("img", { name: "pixel.png" }); + await expect(thumbnail).toBeVisible({ timeout: 30_000 }); + await expect(thumbnail).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/); + await thumbnail.click(); + + const dialog = page.getByRole("dialog", { name: "pixel.png" }); + const download = dialog.getByRole("link", { name: "Download pixel.png" }); + await expect(download).toHaveAttribute("href", /\/api\/artifacts\/[^/?]+\?download=1$/); + const saved = page.waitForEvent("download"); + await download.click(); + expect((await saved).suggestedFilename()).toBe("pixel.png"); +}); diff --git a/apps/web/src/components/ArtifactFileCard.tsx b/apps/web/src/components/ArtifactFileCard.tsx index b36eb65..41c11c1 100644 --- a/apps/web/src/components/ArtifactFileCard.tsx +++ b/apps/web/src/components/ArtifactFileCard.tsx @@ -1,19 +1,12 @@ -import { i18n } from "@lingui/core"; -import { t } from "@lingui/core/macro"; import { Trans, useLingui } from "@lingui/react/macro"; import { ChatMarkdown } from "@rakazo/chat-ui/web"; import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web"; import { Download, FileText, X } from "lucide-react"; import { type RefObject, useEffect, useRef, useState } from "react"; -import { - type ArtifactTarget, - downloadArtifact, - downloadArtifactBytes, - fetchArtifactBytes, -} from "../lib/artifact-open"; +import { downloadArtifact, fetchArtifactBytes } from "../lib/artifact-open"; +import { formatBytes } from "../lib/format-bytes"; type ArtifactFileCardProps = { - target: ArtifactTarget; artifactId: string; name: string; mimeType: string; @@ -26,23 +19,14 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) { const previewButton = useRef(null); const closeButton = useRef(null); const [previewOpen, setPreviewOpen] = useState(false); - const [downloadError, setDownloadError] = useState(null); - - async function startDownload() { - setDownloadError(null); - try { - await downloadArtifact(props.target, props.artifactId, props.name, props.mimeType); - } catch { - setDownloadError(t`Could not download ${props.name}. Try again.`); - } - } + const startDownload = () => downloadArtifact(props.artifactId, props.name); if (!markdown) { return (
- {downloadError ? : null} ); } @@ -80,13 +63,12 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) { type="button" aria-label={t`Download ${props.name}`} title={t`Download ${props.name}`} - onClick={() => void startDownload()} + onClick={startDownload} className="grid w-14 shrink-0 place-items-center border-l border-border text-muted-foreground hover:bg-accent hover:text-foreground" > - {downloadError ? : null} }) { const { t } = useLingui(); - const [downloadError, setDownloadError] = useState(null); const [state, setState] = useState< | { status: "loading" } - | { status: "ready"; bytes: Uint8Array; markdown: string } + | { status: "ready"; markdown: string } | { status: "error"; message: string } >({ status: "loading" }); - const targetBotId = "botId" in target ? target.botId : undefined; - const targetGroupId = "groupId" in target ? target.groupId : undefined; useEffect(() => { let cancelled = false; - const artifactTarget: ArtifactTarget = - targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! }; - void fetchArtifactBytes(artifactTarget, artifactId) + void fetchArtifactBytes(artifactId) .then((bytes) => { if (cancelled) return; try { const markdown = new TextDecoder("utf-8", { fatal: true }).decode(bytes); - setState({ status: "ready", bytes, markdown }); + setState({ status: "ready", markdown }); } catch { setState({ status: "error", message: t`This file is not valid UTF-8 Markdown.` }); } @@ -143,7 +118,7 @@ function MarkdownPreview({ return () => { cancelled = true; }; - }, [artifactId, targetBotId, targetGroupId, t]); + }, [artifactId, t]); return ( <> @@ -157,17 +132,7 @@ function MarkdownPreview({ className="rounded-full text-muted-foreground" aria-label={t`Download ${name}`} title={t`Download ${name}`} - onClick={() => - void (async () => { - setDownloadError(null); - try { - if (state.status === "ready") downloadArtifactBytes(name, mimeType, state.bytes); - else await downloadArtifact(target, artifactId, name, mimeType); - } catch { - setDownloadError(t`Could not download ${name}. Try again.`); - } - })() - } + onClick={() => downloadArtifact(artifactId, name)} > @@ -181,11 +146,6 @@ function MarkdownPreview({ - {downloadError ? ( -
- -
- ) : null}
{state.status === "loading" ? ( @@ -204,23 +164,3 @@ function MarkdownPreview({ ); } - -function DownloadError({ message }: { message: string }) { - return ( -
- {message} -
- ); -} - -function formatBytes(size: number) { - const locale = i18n.locale || "en"; - if (size < 1024) return t`${size} B`; - const format = (value: number) => - new Intl.NumberFormat(locale, { - minimumFractionDigits: 1, - maximumFractionDigits: 1, - }).format(value); - if (size < 1024 * 1024) return t`${format(size / 1024)} KB`; - return t`${format(size / (1024 * 1024))} MB`; -} diff --git a/apps/web/src/components/media/MediaPlayer.test.tsx b/apps/web/src/components/media/MediaPlayer.test.tsx new file mode 100644 index 0000000..c1d9f5a --- /dev/null +++ b/apps/web/src/components/media/MediaPlayer.test.tsx @@ -0,0 +1,99 @@ +// @vitest-environment jsdom + +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@lingui/react/macro", () => { + const t = (parts: TemplateStringsArray, ...values: unknown[]) => + parts.reduce( + (text, part, index) => text + part + (index < values.length ? values[index] : ""), + "", + ); + return { useLingui: () => ({ t }) }; +}); + +import { MediaPlayer } from "./MediaPlayer"; + +let container: HTMLDivElement; +let root: Root; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +describe("MediaPlayer", () => { + it("loads nothing before it is near the viewport and fits vertical video to the column", () => { + const html = renderToStaticMarkup( + , + ); + expect(html).toContain('aria-label="clip.mp4"'); + expect(html).not.toContain("src="); + expect(html).toContain("width:270px"); + expect(html).toContain("aspect-ratio:270 / 480"); + }); + + it("streams video with its poster from the artifact route once visible", async () => { + vi.stubGlobal( + "IntersectionObserver", + class { + constructor(private readonly callback: IntersectionObserverCallback) {} + observe() { + this.callback([{ isIntersecting: true } as IntersectionObserverEntry], this as never); + } + disconnect() {} + }, + ); + await act(async () => + root.render( + , + ), + ); + const video = container.querySelector("video")!; + expect(video.getAttribute("src")).toBe("/api/artifacts/art-1"); + expect(video.getAttribute("poster")).toBe("/api/artifacts/art-2"); + expect(video.getAttribute("preload")).toBe("metadata"); + expect(video.hasAttribute("controls")).toBe(true); + }); + + it("sizes the frame from the file when the block has no dimensions", async () => { + await act(async () => + root.render(), + ); + const frame = container.querySelector('[data-testid="media-video"]')!; + expect(frame.style.width).toBe("420px"); + const video = container.querySelector("video")!; + Object.defineProperty(video, "videoWidth", { value: 1080 }); + Object.defineProperty(video, "videoHeight", { value: 1920 }); + await act(async () => { + video.dispatchEvent(new Event("loadedmetadata")); + }); + expect(frame.style.width).toBe("270px"); + }); + + it("falls back to a download link when the browser cannot play the file", async () => { + await act(async () => + root.render(), + ); + const audio = container.querySelector("audio")!; + expect(audio.getAttribute("src")).toBe("/api/artifacts/art-3"); + await act(async () => { + audio.dispatchEvent(new Event("error")); + }); + expect(container.querySelector("audio")).toBeNull(); + const link = container.querySelector("a")!; + expect(link.getAttribute("href")).toBe("/api/artifacts/art-3?download=1"); + expect(link.getAttribute("download")).toBe("memo.ogg"); + expect(link.getAttribute("aria-label")).toBe("Download memo.ogg"); + }); +}); diff --git a/apps/web/src/components/media/MediaPlayer.tsx b/apps/web/src/components/media/MediaPlayer.tsx new file mode 100644 index 0000000..0df489d --- /dev/null +++ b/apps/web/src/components/media/MediaPlayer.tsx @@ -0,0 +1,129 @@ +import { useLingui } from "@lingui/react/macro"; +import { buttonVariants } from "@rakazo/ui-web"; +import { Download } from "lucide-react"; +import { type RefObject, useEffect, useRef, useState } from "react"; +import { artifactUrl } from "../../lib/artifact-url"; + +export type MediaPlayerProps = { + artifactId: string; + kind: "video" | "audio"; + name: string; + posterArtifactId?: string; + width?: number; + height?: number; +}; + +const MAX_VIDEO_WIDTH = 420; +const MAX_VIDEO_HEIGHT = 480; + +/** Plays a video or audio artifact in place; loads nothing until it is near the viewport. */ +export function MediaPlayer({ + artifactId, + kind, + name, + posterArtifactId, + width, + height, +}: MediaPlayerProps) { + const container = useRef(null); + const near = useNearViewport(container); + const [failed, setFailed] = useState(false); + const [frame, setFrame] = useState<{ width: number; height: number } | null>(null); + const src = near ? artifactUrl(artifactId) : undefined; + + if (failed) return ; + + if (kind === "audio") { + return ( +
+ {/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */} +
+ ); + } + + // Uploads carry no dimensions; size the frame from the file once its metadata loads. + const box = videoBox(width ?? frame?.width, height ?? frame?.height); + return ( +
+ {/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */} +
+ ); +} + +/** Fits the frame into the chat column, defaulting to 16:9 until the size is known. */ +function videoBox(width?: number, height?: number) { + const w = width && width > 0 ? width : 16; + const h = height && height > 0 ? height : 9; + const scale = Math.min(MAX_VIDEO_WIDTH / w, MAX_VIDEO_HEIGHT / h); + return { width: Math.round(w * scale), height: Math.round(h * scale) }; +} + +function MediaFallback({ artifactId, name }: { artifactId: string; name: string }) { + const { t } = useLingui(); + return ( +
+ {name} + + + +
+ ); +} + +function useNearViewport(ref: RefObject) { + const [near, setNear] = useState(false); + useEffect(() => { + const element = ref.current; + if (!element || typeof IntersectionObserver === "undefined") { + setNear(true); + return; + } + const observer = new IntersectionObserver( + (entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + setNear(true); + observer.disconnect(); + } + }, + { rootMargin: "320px" }, + ); + observer.observe(element); + return () => observer.disconnect(); + }, [ref]); + return near; +} diff --git a/apps/web/src/lib/artifact-open.ts b/apps/web/src/lib/artifact-open.ts index a1ed8eb..e832642 100644 --- a/apps/web/src/lib/artifact-open.ts +++ b/apps/web/src/lib/artifact-open.ts @@ -1,40 +1,29 @@ -import { rpc } from "./rpc"; +import { t } from "@lingui/core/macro"; +import { artifactUrl } from "./artifact-url"; export type ArtifactTarget = { botId: string } | { groupId: string }; -export function decodeArtifactBase64(contentBase64: string): Uint8Array { - const binary = atob(contentBase64); - const bytes = new Uint8Array(binary.length); - for (let index = 0; index < binary.length; index += 1) { - bytes[index] = binary.charCodeAt(index); - } - return bytes; -} - -export async function fetchArtifactBytes( - target: ArtifactTarget, - artifactId: string, -): Promise { - const artifact = await rpc.artifacts.get({ ...target, artifactId }); - return decodeArtifactBase64(artifact.contentBase64); +export async function fetchArtifactBytes(artifactId: string): Promise { + const response = await fetch(artifactUrl(artifactId), { credentials: "include" }); + if (!response.ok) throw new Error(t`Could not load this file.`); + return new Uint8Array(await response.arrayBuffer()); } export function downloadArtifactBytes(name: string, mimeType: string, bytes: Uint8Array): void { const blob = new Blob([new Uint8Array(bytes)], { type: mimeType }); const url = URL.createObjectURL(blob); + saveUrl(url, name); + window.setTimeout(() => URL.revokeObjectURL(url), 60_000); +} + +/** Lets the browser stream the download from the artifact route instead of buffering it. */ +export function downloadArtifact(artifactId: string, name: string): void { + saveUrl(artifactUrl(artifactId, { download: true }), name); +} + +function saveUrl(url: string, name: string) { const anchor = document.createElement("a"); anchor.href = url; anchor.download = name; anchor.click(); - window.setTimeout(() => URL.revokeObjectURL(url), 60_000); -} - -export async function downloadArtifact( - target: ArtifactTarget, - artifactId: string, - name: string, - mimeType: string, -): Promise { - const bytes = await fetchArtifactBytes(target, artifactId); - downloadArtifactBytes(name, mimeType, bytes); } diff --git a/apps/web/src/lib/artifact-upload.test.ts b/apps/web/src/lib/artifact-upload.test.ts new file mode 100644 index 0000000..5c0440b --- /dev/null +++ b/apps/web/src/lib/artifact-upload.test.ts @@ -0,0 +1,83 @@ +import { ATTACHMENT_MAX_BYTES } from "@rakazo/contracts"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@lingui/core/macro", () => ({ + t: (parts: TemplateStringsArray, ...values: unknown[]) => + parts.reduce( + (text, part, index) => text + part + (index < values.length ? values[index] : ""), + "", + ), +})); +const create = vi.hoisted(() => vi.fn()); +vi.mock("./rpc", () => ({ + rpc: { artifacts: { create } }, + withSpaceHeaders: (init: HeadersInit) => new Headers(init), +})); + +import { + attachmentMaxBytes, + uploadArtifactFile, + uploadAttachment, + usesStreamingUpload, +} from "./artifact-upload"; + +const artifact = { + id: "art-1", + botId: "bot-1", + groupId: null, + runId: null, + name: "clip.mp4", + mimeType: "video/mp4", + size: 4, + createdAt: "2026-10-03T12:00:00.000Z", +}; + +afterEach(() => { + vi.unstubAllGlobals(); + create.mockReset(); +}); + +describe("artifact uploads", () => { + it("streams media and large files and keeps small files on the inline RPC", () => { + expect(usesStreamingUpload(10, "video/mp4")).toBe(true); + expect(usesStreamingUpload(10, "audio/mpeg")).toBe(true); + expect(usesStreamingUpload(10, "image/png")).toBe(false); + expect(usesStreamingUpload(ATTACHMENT_MAX_BYTES + 1, "image/png")).toBe(true); + expect(attachmentMaxBytes("video/mp4")).toBe(500 * 1024 * 1024); + expect(attachmentMaxBytes("text/plain")).toBe(25 * 1024 * 1024); + }); + + it("posts the raw file with its target, name and type", async () => { + const fetch = vi.fn(async () => Response.json(artifact, { status: 201 })); + vi.stubGlobal("fetch", fetch); + const file = new File(["data"], "clip.mp4", { type: "video/mp4" }); + + await expect(uploadAttachment({ groupId: "group-1" }, file, "video/mp4")).resolves.toEqual( + artifact, + ); + const [url, init] = fetch.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe("/api/artifacts?groupId=group-1&name=clip.mp4"); + expect(init.method).toBe("POST"); + expect(init.credentials).toBe("include"); + expect(new Headers(init.headers).get("content-type")).toBe("video/mp4"); + expect(init.body).toBe(file); + expect(create).not.toHaveBeenCalled(); + }); + + it("surfaces the server's reason, or a generic one", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ error: "File is too large." }, { status: 413 })), + ); + await expect( + uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"), + ).rejects.toThrow("File is too large."); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("bad gateway", { status: 502 })), + ); + await expect( + uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"), + ).rejects.toThrow("Could not upload big.mp4. Try again."); + }); +}); diff --git a/apps/web/src/lib/artifact-upload.ts b/apps/web/src/lib/artifact-upload.ts new file mode 100644 index 0000000..12d11e4 --- /dev/null +++ b/apps/web/src/lib/artifact-upload.ts @@ -0,0 +1,68 @@ +import { t } from "@lingui/core/macro"; +import { + ARTIFACT_MAX_BYTES_BY_KIND, + ArtifactSchema, + ATTACHMENT_MAX_BYTES, + attachmentKindOf, +} from "@rakazo/contracts"; +import type { ArtifactTarget } from "./artifact-open"; +import { rpc, withSpaceHeaders } from "./rpc"; + +type UploadedArtifact = ReturnType; + +/** Largest file of this type the composer accepts. */ +export function attachmentMaxBytes(mimeType: string): number { + return ARTIFACT_MAX_BYTES_BY_KIND[attachmentKindOf(mimeType)]; +} + +/** Media and anything above the inline RPC cap upload as a raw stream instead of base64. */ +export function usesStreamingUpload(size: number, mimeType: string): boolean { + const kind = attachmentKindOf(mimeType); + return size > ATTACHMENT_MAX_BYTES || kind === "video" || kind === "audio"; +} + +export async function uploadAttachment( + target: ArtifactTarget, + file: File, + mimeType: string, +): Promise { + if (usesStreamingUpload(file.size, mimeType)) { + return uploadArtifactFile(target, file, file.name, mimeType); + } + const contentBase64 = await readFileAsBase64(file); + return rpc.artifacts.create({ ...target, name: file.name, mimeType, contentBase64 }); +} + +/** Uploads a file as a raw streamed body to the artifact route. */ +export async function uploadArtifactFile( + target: ArtifactTarget, + file: Blob, + name: string, + mimeType: string, +): Promise { + const query = new URLSearchParams({ ...target, name }); + const response = await fetch(`/api/artifacts?${query}`, { + method: "POST", + headers: withSpaceHeaders({ "content-type": mimeType }), + credentials: "include", + body: file, + }); + const body: unknown = await response.json().catch(() => null); + const artifact = ArtifactSchema.safeParse(body); + if (response.ok && artifact.success) return artifact.data; + const error = (body as { error?: unknown } | null)?.error; + throw new Error(typeof error === "string" ? error : t`Could not upload ${name}. Try again.`); +} + +function readFileAsBase64(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const result = typeof reader.result === "string" ? reader.result : ""; + const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result; + resolve(base64); + }; + reader.onerror = () => reject(reader.error ?? new Error("Failed to read file")); + reader.readAsDataURL(file); + }); +} diff --git a/apps/web/src/lib/artifact-url.ts b/apps/web/src/lib/artifact-url.ts index a87f4d4..5dff453 100644 --- a/apps/web/src/lib/artifact-url.ts +++ b/apps/web/src/lib/artifact-url.ts @@ -1,38 +1,8 @@ -import { t } from "@lingui/core/macro"; -import { ArtifactSchema } from "@rakazo/contracts"; -import type { ArtifactTarget } from "./artifact-open"; -import { withSpaceHeaders } from "./rpc"; - -const ARTIFACT_PATH = "/api/artifacts"; - /** * Same-origin URL that streams an artifact with Range support, for `
+ ); + } + if (block.kind === "video" || block.kind === "audio") { + return ( +
+
); @@ -7856,7 +7873,6 @@ const MessageView = memo(function MessageView({ className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`} > { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const result = typeof reader.result === "string" ? reader.result : ""; - const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result; - resolve(base64); - }; - reader.onerror = () => reject(reader.error ?? new Error("Failed to read file")); - reader.readAsDataURL(file); - }); -} diff --git a/apps/web/src/pages/shell/message-cards.tsx b/apps/web/src/pages/shell/message-cards.tsx index 14131d6..294bbd4 100644 --- a/apps/web/src/pages/shell/message-cards.tsx +++ b/apps/web/src/pages/shell/message-cards.tsx @@ -1,11 +1,18 @@ import { Trans, useLingui } from "@lingui/react/macro"; import type { MessageBlock } from "@rakazo/contracts"; import { abortableDelay } from "@rakazo/core"; -import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web"; -import { X } from "lucide-react"; +import { + Button, + buttonVariants, + Dialog, + DialogClose, + DialogContent, + DialogTitle, +} from "@rakazo/ui-web"; +import { Download, X } from "lucide-react"; import { useEffect, useRef, useState } from "react"; import { BuiCard, SuccessPop } from "../../components/ai/primitives"; -import { type ArtifactTarget, decodeArtifactBase64 } from "../../lib/artifact-open"; +import { artifactUrl } from "../../lib/artifact-url"; import { chartViewport } from "../../lib/chart-viewport"; import { connectMcpOauth } from "../../lib/mcp-connect"; import { rpc } from "../../lib/rpc"; @@ -461,104 +468,65 @@ export function ChartBlockView({ ); } -export function ArtifactImage({ - target, - artifactId, - name, -}: { - target: ArtifactTarget; - artifactId: string; - name: string; -}) { +export function ArtifactImage({ artifactId, name }: { artifactId: string; name: string }) { const { t } = useLingui(); - const [src, setSrc] = useState(null); const [open, setOpen] = useState(false); - const [visible, setVisible] = useState(false); - const container = useRef(null); - const targetBotId = "botId" in target ? target.botId : undefined; - const targetGroupId = "groupId" in target ? target.groupId : undefined; + const [failed, setFailed] = useState(false); + const src = artifactUrl(artifactId); - useEffect(() => { - const element = container.current; - if (!element || typeof IntersectionObserver === "undefined") { - setVisible(true); - return; - } - const observer = new IntersectionObserver( - (entries) => { - if (entries.some((entry) => entry.isIntersecting)) { - setVisible(true); - observer.disconnect(); - } - }, - { rootMargin: "320px" }, + if (failed) { + return ( +
+ {name} +
); - observer.observe(element); - return () => observer.disconnect(); - }, []); - - useEffect(() => { - if (!visible) return; - let cancelled = false; - let objectUrl: string | null = null; - setSrc(null); - void rpc.artifacts - .get( - targetBotId - ? { botId: targetBotId, artifactId } - : { groupId: targetGroupId ?? "", artifactId }, - ) - .then((artifact) => { - const bytes = decodeArtifactBase64(artifact.contentBase64); - objectUrl = URL.createObjectURL( - new Blob([new Uint8Array(bytes)], { type: artifact.mimeType }), - ); - if (cancelled) URL.revokeObjectURL(objectUrl); - else setSrc(objectUrl); - }) - .catch(() => undefined); - return () => { - cancelled = true; - if (objectUrl) URL.revokeObjectURL(objectUrl); - }; - }, [artifactId, targetBotId, targetGroupId, visible]); - + } return ( -
- {src ? ( - + + - {name} - - ) : ( -
- {name} -
- )} - {src ? ( - - - {name} + {name} +
+ + + } + render={
 - ) : null} +
+ {name} + + ); } diff --git a/packages/adapters/src/artifact-files.ts b/packages/adapters/src/artifact-files.ts index ad88094..b228bf1 100644 --- a/packages/adapters/src/artifact-files.ts +++ b/packages/adapters/src/artifact-files.ts @@ -47,6 +47,7 @@ export async function storeArtifactStream( stream: ArtifactStreamPut["stream"]; maxBytes?: number; signal?: AbortSignal; + operationId?: string; }, ) { validateAttachmentMimeType(input.mimeType); @@ -54,7 +55,8 @@ export async function storeArtifactStream( input.maxBytes ?? Number.POSITIVE_INFINITY, artifactMaxBytes(input.mimeType), ); - const operationId = `artifact-store:${input.runId ?? input.botId ?? input.userId}`; + const operationId = + input.operationId ?? `artifact-store:${input.runId ?? input.botId ?? input.userId}`; const context: AdapterContext = { operationId, traceId: operationId, diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index 9907789..7d68b26 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -162,7 +162,7 @@ export const builtinAgentTools: ConnectorTool[] = [ { name: "attach_file", description: - "Attach a workspace file from this bot's home to the chat thread as an image or common file. The file stays in place; users can open it from the message.", + "Attach a workspace file from this bot's home to the chat thread as an image, video, audio or common file. The file stays in place; users can open or play it from the message.", inputSchema: { type: "object", properties: { path: { type: "string" } }, diff --git a/packages/adapters/src/computer-support.ts b/packages/adapters/src/computer-support.ts index 62deb83..a6e6519 100644 --- a/packages/adapters/src/computer-support.ts +++ b/packages/adapters/src/computer-support.ts @@ -17,6 +17,18 @@ export function toComputerRef(computer: { }; } +/** + * Remote computers move files as base64 JSON (the Docker supervisor accepts 16 MiB requests), so + * files copied to or read from them stay below this size. Local computers have no transfer cap. + */ +export const REMOTE_COMPUTER_FILE_TRANSFER_MAX_BYTES = 12 * 1024 * 1024; + +export function computerFileTransferMaxBytes(computer: Pick): number { + return computer.kind === "desktop" || computer.kind === "fake" + ? Number.POSITIVE_INFINITY + : REMOTE_COMPUTER_FILE_TRANSFER_MAX_BYTES; +} + const EMPTY_PNG = Uint8Array.from( Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", diff --git a/packages/adapters/src/executor.ts b/packages/adapters/src/executor.ts index 31b094d..3d18ea0 100644 --- a/packages/adapters/src/executor.ts +++ b/packages/adapters/src/executor.ts @@ -197,6 +197,7 @@ import { withComputerScreenAvailability } from "./computer-screens.js"; import { assertTeamShellCommandAllowed, assertTeamWorkspacePathAllowed, + computerFileTransferMaxBytes, displayBotWorkspacePath, filterTeamWorkspaceEntries, resolveBotWorkspaceCwd, @@ -369,8 +370,10 @@ import { } from "./takeover-resume.js"; import { getActiveTeachingSession, parsePlaybook } from "./teaching-session.js"; import { + attachableFileMaxBytes, attachWorkspaceFileToThread, currentTurnFilesInstruction, + formatMiB, materializeCurrentTurnFiles, } from "./thread-artifacts.js"; import { advanceToolCallLoopGuard } from "./tool-loop.js"; @@ -2368,6 +2371,7 @@ export function createRunExecutor(deps: ExecutorDeps) { spaceId: run.spaceId, userId: run.userId, botId: bot.id, + groupId: thread.groupId ?? undefined, runId: run.id, filePath: outPath, bytes: png, @@ -2393,18 +2397,23 @@ export function createRunExecutor(deps: ExecutorDeps) { return finish({ error: "artifact storage unavailable", path: filePath }); } const storedPath = resolveStoredWorkspacePath(filePath); - let bytes: Uint8Array; - try { - bytes = await deps.sandbox.readFile(computer, storedPath, context, { - maxBytes: ATTACHMENT_MAX_BYTES, - }); - } catch { - return finish({ error: "file not found or unreadable", path: filePath }); - } const mimeType = inferAttachmentMimeType(filePath); if (!mimeType) { return finish({ error: "unsupported attachment type", path: filePath }); } + const maxBytes = Math.min( + attachableFileMaxBytes(deps.artifacts, mimeType), + computerFileTransferMaxBytes(computer), + ); + let bytes: Uint8Array; + try { + bytes = await deps.sandbox.readFile(computer, storedPath, context, { maxBytes }); + } catch { + return finish({ + error: `file not found, unreadable or larger than ${formatMiB(maxBytes)}`, + path: filePath, + }); + } try { const attached = await attachWorkspaceFileToThread( { prisma: deps.prisma, artifacts: deps.artifacts }, @@ -3906,7 +3915,7 @@ export function createRunExecutor(deps: ExecutorDeps) { item.blocks, context.signal, ); - workspaceCheckpoint.markFiles(files); + workspaceCheckpoint.markFiles(files.filter((file) => file.path)); const filesInstruction = currentTurnFilesInstruction(files); return { id: item.id, diff --git a/packages/adapters/src/history-compaction.test.ts b/packages/adapters/src/history-compaction.test.ts index ef8ad64..b130201 100644 --- a/packages/adapters/src/history-compaction.test.ts +++ b/packages/adapters/src/history-compaction.test.ts @@ -498,7 +498,7 @@ describe("compactHistory", () => { const [request] = harness.runtime.run.mock.calls[0]!; expect(request.prompt).toContain("[file: plan.pdf (application/pdf, 123 bytes)]"); - expect(request.prompt).toContain("[image: diagram.png]"); + expect(request.prompt).toContain("[image: diagram.png (artifact artifact-2)]"); }); it("compacts locally without a semantic memory provider", async () => { diff --git a/packages/adapters/src/reply-context.test.ts b/packages/adapters/src/reply-context.test.ts index 136dea7..9b0db83 100644 --- a/packages/adapters/src/reply-context.test.ts +++ b/packages/adapters/src/reply-context.test.ts @@ -55,11 +55,11 @@ describe("reply context", () => { ...target, blocks: [ { kind: "text", text: "" }, - { kind: "image", name: "example.png" }, + { kind: "image", artifactId: "artifact-1", name: "example.png" }, ], }); const context = await loadReplyContext(prisma, "thread-1", "user-reply"); - expect(context).toContain("[image: example.png]"); + expect(context).toContain("[image: example.png (artifact artifact-1)]"); expect(context).toContain("\\u003c/reply_target\\u003e"); expect(context?.match(/<\/reply_target>/g)).toHaveLength(1); }); diff --git a/packages/adapters/src/thread-artifacts.test.ts b/packages/adapters/src/thread-artifacts.test.ts index 5d164bd..fb79536 100644 --- a/packages/adapters/src/thread-artifacts.test.ts +++ b/packages/adapters/src/thread-artifacts.test.ts @@ -8,6 +8,7 @@ import type { MessageBlock } from "@rakazo/contracts"; import type { PrismaClient } from "@rakazo/db"; import { describe, expect, it, vi } from "vitest"; import { + attachableFileMaxBytes, attachWorkspaceFileToThread, currentTurnFilesInstruction, materializeCurrentTurnFiles, @@ -25,7 +26,8 @@ describe("current-turn thread files", () => { artifact: { create: vi.fn().mockRejectedValue(failure) }, } as unknown as PrismaClient, artifacts: { - put: vi.fn().mockResolvedValue({ id: "stored-1", hash: "hash" }), + describe: () => ({ capabilities: { stream: true } }), + putStream: vi.fn().mockResolvedValue({ key: "stored-1", size: 3, sha256: "hash" }), remove, } as unknown as ArtifactStore, }, @@ -111,6 +113,7 @@ describe("current-turn thread files", () => { ); expect(files).toEqual([ { + artifactId: "artifact-1", name: "../quarterly report.pdf", mimeType: "application/pdf", size: 4, @@ -120,7 +123,73 @@ describe("current-turn thread files", () => { expect(currentTurnFilesInstruction(files)).toContain('"attachments/artifact-1.pdf"'); }); - it("does not load images as computer files", async () => { + it("copies images, video and audio too, and keeps files over a remote computer's cap in storage", async () => { + const mb = 1024 * 1024; + const findMany = vi.fn().mockResolvedValue([ + { id: "img-1", name: "photo.png", mimeType: "image/png", size: 3, storageKey: "k-img" }, + { id: "aud-1", name: "memo.mp3", mimeType: "audio/mpeg", size: 3, storageKey: "k-aud" }, + { id: "vid-1", name: "clip.mp4", mimeType: "video/mp4", size: 40 * mb, storageKey: "k-vid" }, + ]); + const get = vi.fn().mockResolvedValue(new Uint8Array([1, 2, 3])); + const writeFile = vi.fn().mockResolvedValue(undefined); + const context: AdapterContext & { botId: string } = { + operationId: "run-1", + traceId: "run-1", + spaceId: "workspace-1", + userId: "user-1", + botId: "bot-1", + signal: new AbortController().signal, + }; + const blocks: MessageBlock[] = [ + { kind: "image", artifactId: "img-1", name: "photo.png", mimeType: "image/png" }, + { kind: "audio", artifactId: "aud-1", name: "memo.mp3", mimeType: "audio/mpeg", size: 3 }, + { + kind: "video", + artifactId: "vid-1", + name: "clip.mp4", + mimeType: "video/mp4", + size: 40 * mb, + }, + { kind: "text", text: "see these" }, + ]; + const deps = { + prisma: { artifact: { findMany } } as unknown as PrismaClient, + artifacts: { get } as unknown as ArtifactStore, + sandbox: { writeFile } as unknown as SandboxProvider, + }; + + const remote = await materializeCurrentTurnFiles(deps, blocks, { + context, + computer: { id: "computer-1", botId: "bot-1", kind: "docker", providerRef: "docker-1" }, + computerMode: "dedicated", + }); + expect(remote.map((file) => [file.artifactId, file.path])).toEqual([ + ["img-1", "attachments/img-1.png"], + ["aud-1", "attachments/aud-1.mp3"], + ["vid-1", null], + ]); + expect(get).not.toHaveBeenCalledWith("k-vid", context); + const instruction = currentTurnFilesInstruction(remote); + expect(instruction).toContain('"attachments/img-1.png"'); + expect(instruction).toContain("too large to copy to your computer"); + expect(instruction).toContain('"clip.mp4" (video/mp4, 41943040 bytes): artifact vid-1'); + + writeFile.mockClear(); + const local = await materializeCurrentTurnFiles(deps, blocks, { + context, + computer: { id: "computer-2", botId: "bot-1", kind: "desktop", providerRef: "desktop-1" }, + computerMode: "dedicated", + }); + expect(local.map((file) => file.path)).toEqual([ + "attachments/img-1.png", + "attachments/aud-1.mp3", + "attachments/vid-1.mp4", + ]); + expect(writeFile).toHaveBeenCalledTimes(3); + expect(currentTurnFilesInstruction(local)).not.toContain("too large"); + }); + + it("skips loading when a turn has no attachments", async () => { const findMany = vi.fn(); const files = await materializeCurrentTurnFiles( { @@ -128,14 +197,7 @@ describe("current-turn thread files", () => { artifacts: {} as ArtifactStore, sandbox: {} as SandboxProvider, }, - [ - { - kind: "image", - artifactId: "image-1", - name: "photo.png", - mimeType: "image/png", - }, - ], + [{ kind: "text", text: "hello" }], { context: { operationId: "run-1", @@ -158,4 +220,12 @@ describe("current-turn thread files", () => { expect(files).toEqual([]); expect(findMany).not.toHaveBeenCalled(); }); + + it("caps attachments by kind when the store streams and at the inline cap otherwise", () => { + const streaming = { describe: () => ({ capabilities: { stream: true } }) } as ArtifactStore; + const buffered = { describe: () => ({ capabilities: { stream: false } }) } as ArtifactStore; + expect(attachableFileMaxBytes(streaming, "video/mp4")).toBe(500 * 1024 * 1024); + expect(attachableFileMaxBytes(streaming, "text/plain")).toBe(25 * 1024 * 1024); + expect(attachableFileMaxBytes(buffered, "video/mp4")).toBe(10 * 1024 * 1024); + }); }); diff --git a/packages/adapters/src/thread-artifacts.ts b/packages/adapters/src/thread-artifacts.ts index 72bc432..f6630bd 100644 --- a/packages/adapters/src/thread-artifacts.ts +++ b/packages/adapters/src/thread-artifacts.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import path from "node:path"; import type { AdapterContext, @@ -12,18 +11,39 @@ import { attachmentExtensionForMimeType, inferAttachmentMimeType, messageBlockForArtifact, - validateAttachmentMimeType, } from "@rakazo/core"; import type { PrismaClient } from "@rakazo/db"; -import { assertTeamWorkspacePathAllowed, resolveBotWorkspacePath } from "./computer-support.js"; +import { artifactMaxBytes, storeArtifactStream } from "./artifact-files.js"; +import { + assertTeamWorkspacePathAllowed, + computerFileTransferMaxBytes, + resolveBotWorkspacePath, +} from "./computer-support.js"; export type MaterializedThreadFile = { + artifactId: string; name: string; mimeType: string; size: number; - path: string; + /** Workspace path, or null when the file is too large to copy to this computer. */ + path: string | null; }; +type AttachmentBlock = Extract; + +const ATTACHMENT_BLOCK_KINDS: ReadonlySet = new Set(["image", "video", "audio", "file"]); + +/** Media up to its kind cap when the store streams; otherwise the inline attachment cap. */ +export function attachableFileMaxBytes(artifacts: ArtifactStore, mimeType: string): number { + return artifacts.describe().capabilities.stream + ? artifactMaxBytes(mimeType) + : ATTACHMENT_MAX_BYTES; +} + +export function formatMiB(bytes: number): string { + return `${Math.floor(bytes / (1024 * 1024))} MiB`; +} + export async function attachWorkspaceFileToThread( deps: { prisma: PrismaClient; @@ -39,60 +59,39 @@ export async function attachWorkspaceFileToThread( bytes: Uint8Array; operationId: string; }, -): Promise<{ - artifactId: string; - block: Extract; -}> { +): Promise<{ artifactId: string; block: AttachmentBlock }> { const name = path.basename(input.filePath) || input.filePath; const mimeType = inferAttachmentMimeType(name); if (!mimeType) { throw new Error(`Unsupported attachment type for ${name}`); } - validateAttachmentMimeType(mimeType); - if (input.bytes.byteLength > ATTACHMENT_MAX_BYTES) { - throw new Error("file exceeds the 10 MiB attachment limit"); + const maxBytes = attachableFileMaxBytes(deps.artifacts, mimeType); + if (input.bytes.byteLength > maxBytes) { + throw new Error(`file exceeds the ${formatMiB(maxBytes)} attachment limit`); } - - const context = { - operationId: input.operationId, - traceId: input.operationId, + const bytes = input.bytes; + const row = await storeArtifactStream(deps, { spaceId: input.spaceId, userId: input.userId, botId: input.botId, - signal: new AbortController().signal, - }; - const stored = await deps.artifacts.put({ name, mimeType, bytes: input.bytes }, context); - const hash = createHash("sha256").update(input.bytes).digest("hex"); - const row = await deps.prisma.artifact - .create({ - data: { - spaceId: input.spaceId, - userId: input.userId, - botId: input.botId, - groupId: input.groupId, - runId: input.runId, - name, - mimeType, - size: input.bytes.byteLength, - hash, - storageKey: stored.id, - }, - }) - .catch(async (error) => { - await deps.artifacts.remove(stored.id, context).catch(() => undefined); - throw error; - }); - return { - artifactId: row.id, - block: messageBlockForArtifact({ - id: row.id, - name: row.name, - mimeType: row.mimeType, - size: row.size, - }), - }; + groupId: input.groupId ?? null, + runId: input.runId, + name, + mimeType, + stream: (async function* () { + yield bytes; + })(), + maxBytes, + operationId: input.operationId, + }); + return { artifactId: row.id, block: messageBlockForArtifact(row) }; } +/** + * Copies this turn's attachments (files, images, video and audio) into the bot's workspace so its + * tools can open them. Files above the computer's transfer cap stay in storage and are listed + * with their artifact id instead. + */ export async function materializeCurrentTurnFiles( deps: { prisma: PrismaClient; @@ -108,28 +107,31 @@ export async function materializeCurrentTurnFiles( markWorkspaceDirty?: () => void; }, ): Promise { - const fileBlocks = blocks?.filter( - (block): block is Extract => block.kind === "file", + const attachmentBlocks = blocks?.filter((block): block is AttachmentBlock => + ATTACHMENT_BLOCK_KINDS.has(block.kind), ); - if (!fileBlocks?.length) return []; + if (!attachmentBlocks?.length) return []; const rows = await deps.prisma.artifact.findMany({ where: { - id: { in: fileBlocks.map((block) => block.artifactId) }, + id: { in: attachmentBlocks.map((block) => block.artifactId) }, spaceId: input.context.spaceId, userId: input.context.userId, }, }); const byId = new Map(rows.map((row) => [row.id, row])); + const transferMaxBytes = computerFileTransferMaxBytes(input.computer); const materialized: MaterializedThreadFile[] = []; - for (const block of fileBlocks) { + for (const block of attachmentBlocks) { const row = byId.get(block.artifactId); if (!row) throw new Error(`Attached file is unavailable: ${block.name}`); - const bytes = await deps.artifacts.get(row.storageKey, input.context); - if (bytes.byteLength > ATTACHMENT_MAX_BYTES) { - throw new Error(`Attached file exceeds the 10 MiB limit: ${row.name}`); + const entry = { artifactId: row.id, name: row.name, mimeType: row.mimeType, size: row.size }; + if (row.size > Math.min(transferMaxBytes, artifactMaxBytes(row.mimeType))) { + materialized.push({ ...entry, path: null }); + continue; } + const bytes = await deps.artifacts.get(row.storageKey, input.context); const relativePath = `attachments/${row.id}${attachmentExtensionForMimeType(row.mimeType)}`; const storedPath = resolveBotWorkspacePath( input.computerMode, @@ -146,24 +148,29 @@ export async function materializeCurrentTurnFiles( }, input.context, ); - materialized.push({ - name: row.name, - mimeType: row.mimeType, - size: row.size, - path: relativePath, - }); + materialized.push({ ...entry, path: relativePath }); } return materialized; } export function currentTurnFilesInstruction(files: readonly MaterializedThreadFile[]): string { - if (!files.length) return ""; + const copied = files.filter((file) => file.path !== null); + const kept = files.filter((file) => file.path === null); + const describe = (file: MaterializedThreadFile) => + `- ${JSON.stringify(file.name)} (${file.mimeType}, ${file.size} bytes)`; return [ - "Current-turn attached files are available on your computer. Inspect them before answering:", - ...files.map( - (file) => - `- ${JSON.stringify(file.name)} (${file.mimeType}, ${file.size} bytes): ${JSON.stringify(file.path)}`, - ), + ...(copied.length + ? [ + "Current-turn attached files are available on your computer. Inspect them before answering:", + ...copied.map((file) => `${describe(file)}: ${JSON.stringify(file.path)}`), + ] + : []), + ...(kept.length + ? [ + "These current-turn attachments are too large to copy to your computer. Refer to them by artifact id; do not guess their contents:", + ...kept.map((file) => `${describe(file)}: artifact ${file.artifactId}`), + ] + : []), ].join("\n"); } diff --git a/packages/core/src/attachments.test.ts b/packages/core/src/attachments.test.ts index 6db2e57..b39c7d4 100644 --- a/packages/core/src/attachments.test.ts +++ b/packages/core/src/attachments.test.ts @@ -44,7 +44,7 @@ describe("attachment helpers", () => { size: 99, }, ]), - ).toBe("hello\n[image: shot.png]\n[file: brief.pdf (application/pdf, 99 bytes)]"); + ).toBe("hello\n[image: shot.png (artifact a1)]\n[file: brief.pdf (application/pdf, 99 bytes)]"); }); it("infers attachment mime types from extensions", () => { diff --git a/packages/core/src/attachments.ts b/packages/core/src/attachments.ts index cace390..0c2097f 100644 --- a/packages/core/src/attachments.ts +++ b/packages/core/src/attachments.ts @@ -117,12 +117,11 @@ export function blocksToAgentHistoryText(blocks: MessageBlock[]): string { .map((block) => { if (block.kind === "text") return block.text; if (block.kind === "chart") return `[chart: ${block.name}]`; - if (block.kind === "image") return `[image: ${block.name}]`; if (block.kind === "file") { return `[file: ${block.name} (${block.mimeType}, ${block.size} bytes)]`; } // Media keeps the artifact id so a later turn can edit or export what it refers to. - if (block.kind === "video" || block.kind === "audio") { + if (block.kind === "image" || block.kind === "video" || block.kind === "audio") { return `[${block.kind}: ${block.name} (artifact ${block.artifactId})]`; } if (block.kind === "media") { diff --git a/packages/db/src/events.test.ts b/packages/db/src/events.test.ts index e8ca709..afc0e39 100644 --- a/packages/db/src/events.test.ts +++ b/packages/db/src/events.test.ts @@ -1739,7 +1739,7 @@ describe("claimSteering", () => { { id: "steer-1", messageId: "message-1", - text: "First\n[image: chart.png]", + text: "First\n[image: chart.png (artifact artifact-1)]", blocks: [ { kind: "text", text: "First" }, { diff --git a/packages/testkit/src/artifact-routes.test.ts b/packages/testkit/src/artifact-routes.test.ts index bc33ca4..6e55fdc 100644 --- a/packages/testkit/src/artifact-routes.test.ts +++ b/packages/testkit/src/artifact-routes.test.ts @@ -148,6 +148,15 @@ describeRoutes("artifact HTTP routes", () => { (await upload(app, intruder, { groupId: group.id }, "x.mp4", "video/mp4", video)).status, ).toBe(404); + // A cross-site page cannot upload with the owner's cookie. + expect( + ( + await upload(app, owner, { botId: bot.id }, "x.txt", "text/plain", video, { + origin: "https://evil.example", + }) + ).status, + ).toBe(403); + // Signed out. expect((await app.request(`/api/artifacts/${botArtifact.id}`)).status).toBe(401); expect( @@ -283,6 +292,19 @@ describeRoutes("artifact HTTP routes", () => { const me = await rpc<{ spaceId: string }>(app, owner, "me"); const spaceDir = path.join(dataDir, "artifacts", me.spaceId); expect(readdirSafe(spaceDir)).toEqual([]); + + // Video far above the inline cap uploads, streams by range, and is refused as base64. + const large = Buffer.alloc(26 * 1024 * 1024, 1); + large.write("head", 0); + const uploaded = await upload(app, owner, { botId: bot.id }, "long.mp4", "video/mp4", large); + expect(uploaded.status).toBe(201); + const largeVideo = (await uploaded.json()) as { id: string }; + const head = await get(app, owner, largeVideo.id, { range: "bytes=0-3" }); + expect(head.status).toBe(206); + expect(await head.text()).toBe("head"); + await expect( + rpc(app, owner, "artifacts/get", { botId: bot.id, artifactId: largeVideo.id }), + ).rejects.toThrow(/400: .*too large to load inline/); }); });