Merge the media pipeline: inline players, streamed uploads, attached media

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude 2026-10-03 14:27:55 +00:00
commit 99c8ac14b5
No known key found for this signature in database
33 changed files with 937 additions and 359 deletions

View File

@ -609,6 +609,7 @@ export async function createApp(
artifacts,
sessionUserId: async (request) =>
(await auth.api.getSession({ headers: sessionHeaders(request) }))?.user?.id ?? null,
trustedOrigin: (origin) => isTrustedOrigin(origin, env),
});
mountWebhookHttpRoutes(app, { prisma, secrets, events, jobs });
// Shared with stop so a shutdown during retry delays does not restart polling.

View File

@ -77,6 +77,7 @@ describe("artifact HTTP routes", () => {
prisma: prisma as unknown as PrismaClient,
artifacts: artifacts as unknown as ArtifactStore,
sessionUserId,
trustedOrigin: (origin) => origin === "http://127.0.0.1:5173",
});
return { hono, prisma, artifacts };
}
@ -95,6 +96,19 @@ describe("artifact HTTP routes", () => {
expect(artifacts.putStream).not.toHaveBeenCalled();
});
it("refuses uploads from untrusted origins before reading the session", async () => {
const sessionUserId = vi.fn(async () => "user-1");
const { hono, artifacts } = app(sessionUserId);
const response = await hono.request("/api/artifacts?botId=bot-1&name=a.txt", {
method: "POST",
headers: { "content-type": "text/plain", origin: "https://evil.example" },
body: "data",
});
expect(response.status).toBe(403);
expect(sessionUserId).not.toHaveBeenCalled();
expect(artifacts.putStream).not.toHaveBeenCalled();
});
it("answers ids that are not artifact ids with 404 without a lookup", async () => {
const { hono, prisma } = app(async () => "user-1");
const response = await hono.request(`/api/artifacts/${encodeURIComponent("../etc/passwd")}`);

View File

@ -20,6 +20,8 @@ export type ArtifactHttpDeps = {
artifacts: ArtifactStore;
/** The signed-in user (cookie or bearer session), or null. */
sessionUserId: (request: Request) => Promise<string | null>;
/** Same rule as CORS. Raw uploads can be CORS "simple" requests, so the origin is checked. */
trustedOrigin: (origin: string) => boolean;
};
/** Generated ids only; anything else is not an artifact. */
@ -208,10 +210,12 @@ export function mountArtifactHttpRoutes(app: Hono, deps: ArtifactHttpDeps) {
app.post(ARTIFACT_HTTP_PATH, async (c) => {
const request = c.req.raw;
const reject = (status: 400 | 401 | 404 | 413 | 415, error: string) => {
const reject = (status: 400 | 401 | 403 | 404 | 413 | 415, error: string) => {
cancelBody(request.body);
return c.json({ error }, status);
};
const origin = c.req.header("origin");
if (origin && !deps.trustedOrigin(origin)) return reject(403, "Forbidden");
const userId = await deps.sessionUserId(request);
if (!userId) return reject(401, "Unauthorized");
const actor = await requireMembership(

View File

@ -11,6 +11,12 @@ import {
} from "@rakazo/core";
import { IsolationError, type PrismaClient } from "@rakazo/db";
/**
* Base64 reads load the whole file into memory; larger media is served by
* GET /api/artifacts/:id instead.
*/
export const ARTIFACT_INLINE_READ_MAX_BYTES = 25 * 1024 * 1024;
function adapterContext(actor: Actor, botId: string, operationId: string) {
return {
operationId,
@ -131,6 +137,9 @@ async function readArtifact(
},
contextBotId: string,
) {
if (row.size > ARTIFACT_INLINE_READ_MAX_BYTES) {
throw new AttachmentValidationError("This file is too large to load inline.");
}
const bytes = await artifacts.get(
row.storageKey,
adapterContext(actor, contextBotId, `artifact-get:${row.id}`),

View File

@ -4670,24 +4670,26 @@ export function createRouter(deps: RouterDeps) {
}
}),
get: authed.artifacts.get.handler(async ({ context, input }) => {
if (input.groupId) {
const group = await groupRepos.getGroupTarget(context.actor, input.groupId);
const contextBotId = group.members[0]?.bot.id;
if (!contextBotId) throw new IsolationError();
return getSpaceArtifact(deps, context.actor, {
artifactId: input.artifactId,
groupId: input.groupId,
contextBotId,
});
}
await repos.getBot(context.actor, input.botId!);
try {
if (input.groupId) {
const group = await groupRepos.getGroupTarget(context.actor, input.groupId);
const contextBotId = group.members[0]?.bot.id;
if (!contextBotId) throw new IsolationError();
return await getSpaceArtifact(deps, context.actor, {
artifactId: input.artifactId,
groupId: input.groupId,
contextBotId,
});
}
await repos.getBot(context.actor, input.botId!);
return await getOwnedArtifact(deps, context.actor, {
botId: input.botId!,
artifactId: input.artifactId,
});
} catch (error) {
if (error instanceof IsolationError) throw error;
if (error instanceof AttachmentValidationError) {
throw new ORPCError("BAD_REQUEST", { message: error.message });
}
throw error;
}
}),

View File

@ -77,6 +77,7 @@ import {
type MobileMessagePage,
type MobileSnapshot,
mergeMobileSnapshot,
messageAttachments,
messagingProviderLabel,
prependMobileMessagePage,
rpc,
@ -2243,7 +2244,11 @@ function previewMessageText(message: MobileMessage): string {
.join(" ")
.trim();
if (text) return text;
if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) {
if (
message.blocks.some((block) =>
["image", "file", "video", "audio", "media"].includes(block.kind),
)
) {
return t("Attachment");
}
return t("Message");
@ -2676,9 +2681,7 @@ const MessageBubble = memo(function MessageBubble({
</View>
);
}
const attachments = message.blocks.filter(
(block) => block.kind === "image" || block.kind === "file",
);
const attachments = messageAttachments(message.blocks);
const caption = message.blocks
.flatMap((block) => {
if (block.kind === "channel_message" && block.text) {
@ -2686,6 +2689,7 @@ const MessageBubble = memo(function MessageBubble({
`${messagingProviderLabel(block.provider, block.transport)} · ${block.fromLabel}: ${block.text}`,
];
}
if (block.kind === "media") return [block.title];
return block.kind === "text" && block.text ? [block.text] : [];
})
.join("\n");
@ -2797,7 +2801,8 @@ const MessageBubble = memo(function MessageBubble({
fontSize: 15,
}}
>
📎 {attachment.name ?? t("File")}
{attachment.kind === "video" ? "🎬" : attachment.kind === "audio" ? "🎵" : "📎"}{" "}
{attachment.name ?? t("File")}
</Text>
{attachment.size ? (
<Text

View File

@ -16,6 +16,7 @@ import {
MAX_MOBILE_AUTH_RESPONSE_BYTES,
MAX_MOBILE_RPC_RESPONSE_BYTES,
mergeMobileSnapshot,
messageAttachments,
passwordResetCapabilities,
prependMobileMessagePage,
requestPasswordReset,
@ -1857,6 +1858,51 @@ describe("mobile thread event reduction", () => {
).toBe("Slack · Alex: Hello from the group");
});
it("falls back to plain text and file rows for video, audio and media cards", () => {
const blocks = [
{
kind: "video" as const,
artifactId: "vid-1",
mimeType: "video/mp4",
name: "clip.mp4",
size: 10,
width: 1080,
height: 1920,
},
{
kind: "audio" as const,
artifactId: "aud-1",
mimeType: "audio/mpeg",
name: "memo.mp3",
size: 5,
},
{
kind: "media" as const,
jobId: "job-1",
operation: "export_video" as const,
status: "done" as const,
title: "Vertical cut",
outputs: [
{
artifactId: "out-1",
kind: "video" as const,
mimeType: "video/mp4",
name: "cut.mp4",
size: 7,
},
],
},
];
expect(blockText(mobileMessage("media-1", blocks))).toBe(
"[video: clip.mp4]\n[audio: memo.mp3]\nVertical cut: done",
);
expect(messageAttachments(blocks).map((item) => [item.kind, item.artifactId])).toEqual([
["video", "vid-1"],
["audio", "aud-1"],
["video", "out-1"],
]);
});
it("deduplicates durable messages and replaces matching transient subagent state", () => {
const initial = snapshot([
mobileMessage("message-1", [{ kind: "text", text: "old" }]),

View File

@ -860,6 +860,24 @@ export function copyableMobileMessageText(message: MobileMessage): string {
.trim();
}
export type MobileAttachment = {
kind: "image" | "video" | "audio" | "file";
artifactId: string;
name: string;
mimeType: string;
size?: number;
};
/** Files a message carries, including the outputs of a finished media card, as tappable rows. */
export function messageAttachments(blocks: readonly MessageBlock[]): MobileAttachment[] {
return blocks.flatMap((block): MobileAttachment[] => {
if (block.kind === "image") return [{ ...block, kind: "image" }];
if (block.kind === "video" || block.kind === "audio" || block.kind === "file") return [block];
if (block.kind === "media") return block.outputs;
return [];
});
}
export function blockText(message: MobileMessage) {
return message.blocks
.map((block) => {
@ -879,6 +897,10 @@ export function blockText(message: MobileMessage) {
if (block.kind === "file") {
return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`;
}
if (block.kind === "video" || block.kind === "audio") {
return `[${block.kind}: ${block.name ?? "attachment"}]`;
}
if (block.kind === "media") return `${block.title}: ${block.status}`;
if (block.kind === "steps") {
return (block.steps ?? [])
.map((step) => `${step.label}${step.count > 1 ? ` ×${step.count}` : ""}`)

View File

@ -1,6 +1,7 @@
import { attachmentKindOf } from "@rakazo/contracts";
import { File, Paths } from "expo-file-system";
import * as Sharing from "expo-sharing";
import { rpc } from "./api";
import { captureApiRequestContext, rpc } from "./api";
import { artifactCacheFileName } from "./artifact-file";
import { t } from "./i18n";
@ -11,6 +12,16 @@ async function cacheMobileArtifact(
artifactId: string,
mimeType: string,
): Promise<File> {
const kind = attachmentKindOf(mimeType);
if (kind === "video" || kind === "audio") {
// Media can be hundreds of megabytes: stream it to disk instead of a base64 RPC payload.
const { apiBase, headers } = await captureApiRequestContext();
return File.downloadFileAsync(
`${apiBase}/api/artifacts/${encodeURIComponent(artifactId)}`,
new File(Paths.cache, artifactCacheFileName(artifactId, mimeType)),
{ headers, idempotent: true },
);
}
const artifact = await rpc<{ contentBase64: string }>("artifacts/get", {
...target,
artifactId,

Binary file not shown.

View File

@ -0,0 +1,82 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import { expect, test } from "@playwright/test";
import { captureScreenshot, completeOnboarding, signup } from "./helpers";
test.describe.configure({ mode: "serial" });
// VP9 in MP4: Playwright's Chromium has no H.264 decoder, real browsers play both.
const clip = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "media-clip.mp4");
const pixel = Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
"base64",
);
test("an uploaded video streams from the artifact route and plays inline", async ({
page,
}, testInfo) => {
const stamp = Date.now();
await signup(page, `media-video-${stamp}@rakazo.test`, "password12", "Media Video");
await completeOnboarding(page);
const upload = page.waitForResponse(
(response) =>
response.request().method() === "POST" && /\/api\/artifacts\?/.test(response.url()),
);
await page.locator('input[type="file"]').setInputFiles(clip);
await expect(page.getByRole("button", { name: "Remove media-clip.mp4" })).toBeVisible();
await page.getByRole("combobox", { name: /Message/ }).fill("here is the clip");
await page.keyboard.press("Enter");
expect((await upload).status()).toBe(201);
const video = page.getByTestId("media-video").locator("video");
await expect(video).toBeVisible({ timeout: 30_000 });
await expect(video).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/);
await expect
.poll(() => video.evaluate((element: HTMLVideoElement) => element.readyState), {
timeout: 15_000,
})
.toBeGreaterThanOrEqual(1);
expect(await video.evaluate((element: HTMLVideoElement) => element.duration)).toBeCloseTo(1, 0);
await video.evaluate(async (element: HTMLVideoElement) => {
element.muted = true;
await element.play();
});
await expect
.poll(() => video.evaluate((element: HTMLVideoElement) => element.currentTime))
.toBeGreaterThan(0);
await video.evaluate((element: HTMLVideoElement) => {
element.pause();
element.currentTime = 0.5;
element.scrollIntoView({ block: "center" });
});
await captureScreenshot(page, testInfo, "media-video-block");
});
test("an image in the main conversation opens and downloads from the artifact route", async ({
page,
}) => {
const stamp = Date.now();
await signup(page, `media-image-${stamp}@rakazo.test`, "password12", "Media Image");
// The main conversation is a group thread, so this covers group uploads and reads.
await completeOnboarding(page, undefined, { mainConversation: true });
await page
.locator('input[type="file"]')
.setInputFiles({ name: "pixel.png", mimeType: "image/png", buffer: pixel });
await page.getByRole("combobox", { name: /Message/ }).fill("a pixel");
await page.keyboard.press("Enter");
const thumbnail = page.getByTestId("transcript").getByRole("img", { name: "pixel.png" });
await expect(thumbnail).toBeVisible({ timeout: 30_000 });
await expect(thumbnail).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/);
await thumbnail.click();
const dialog = page.getByRole("dialog", { name: "pixel.png" });
const download = dialog.getByRole("link", { name: "Download pixel.png" });
await expect(download).toHaveAttribute("href", /\/api\/artifacts\/[^/?]+\?download=1$/);
const saved = page.waitForEvent("download");
await download.click();
expect((await saved).suggestedFilename()).toBe("pixel.png");
});

View File

@ -1,19 +1,12 @@
import { i18n } from "@lingui/core";
import { t } from "@lingui/core/macro";
import { Trans, useLingui } from "@lingui/react/macro";
import { ChatMarkdown } from "@rakazo/chat-ui/web";
import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web";
import { Download, FileText, X } from "lucide-react";
import { type RefObject, useEffect, useRef, useState } from "react";
import {
type ArtifactTarget,
downloadArtifact,
downloadArtifactBytes,
fetchArtifactBytes,
} from "../lib/artifact-open";
import { downloadArtifact, fetchArtifactBytes } from "../lib/artifact-open";
import { formatBytes } from "../lib/format-bytes";
type ArtifactFileCardProps = {
target: ArtifactTarget;
artifactId: string;
name: string;
mimeType: string;
@ -26,23 +19,14 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
const previewButton = useRef<HTMLButtonElement>(null);
const closeButton = useRef<HTMLButtonElement>(null);
const [previewOpen, setPreviewOpen] = useState(false);
const [downloadError, setDownloadError] = useState<string | null>(null);
async function startDownload() {
setDownloadError(null);
try {
await downloadArtifact(props.target, props.artifactId, props.name, props.mimeType);
} catch {
setDownloadError(t`Could not download ${props.name}. Try again.`);
}
}
const startDownload = () => downloadArtifact(props.artifactId, props.name);
if (!markdown) {
return (
<div>
<button
type="button"
onClick={() => void startDownload()}
onClick={startDownload}
className="rounded-2xl border border-border bg-card px-4 py-3 text-left text-ui-body text-foreground hover:bg-accent"
>
<div className="font-medium">{props.name}</div>
@ -50,7 +34,6 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
{props.mimeType} · {formatBytes(props.size)}
</div>
</button>
{downloadError ? <DownloadError message={downloadError} /> : null}
</div>
);
}
@ -80,13 +63,12 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
type="button"
aria-label={t`Download ${props.name}`}
title={t`Download ${props.name}`}
onClick={() => void startDownload()}
onClick={startDownload}
className="grid w-14 shrink-0 place-items-center border-l border-border text-muted-foreground hover:bg-accent hover:text-foreground"
>
<Download strokeWidth={1.8} className="size-icon-lg" />
</button>
</div>
{downloadError ? <DownloadError message={downloadError} /> : null}
</div>
<Dialog open={previewOpen} onOpenChange={setPreviewOpen}>
<DialogContent
@ -103,32 +85,25 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
}
function MarkdownPreview({
target,
artifactId,
name,
mimeType,
closeButtonRef,
}: ArtifactFileCardProps & { closeButtonRef: RefObject<HTMLButtonElement | null> }) {
const { t } = useLingui();
const [downloadError, setDownloadError] = useState<string | null>(null);
const [state, setState] = useState<
| { status: "loading" }
| { status: "ready"; bytes: Uint8Array; markdown: string }
| { status: "ready"; markdown: string }
| { status: "error"; message: string }
>({ status: "loading" });
const targetBotId = "botId" in target ? target.botId : undefined;
const targetGroupId = "groupId" in target ? target.groupId : undefined;
useEffect(() => {
let cancelled = false;
const artifactTarget: ArtifactTarget =
targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! };
void fetchArtifactBytes(artifactTarget, artifactId)
void fetchArtifactBytes(artifactId)
.then((bytes) => {
if (cancelled) return;
try {
const markdown = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
setState({ status: "ready", bytes, markdown });
setState({ status: "ready", markdown });
} catch {
setState({ status: "error", message: t`This file is not valid UTF-8 Markdown.` });
}
@ -143,7 +118,7 @@ function MarkdownPreview({
return () => {
cancelled = true;
};
}, [artifactId, targetBotId, targetGroupId, t]);
}, [artifactId, t]);
return (
<>
@ -157,17 +132,7 @@ function MarkdownPreview({
className="rounded-full text-muted-foreground"
aria-label={t`Download ${name}`}
title={t`Download ${name}`}
onClick={() =>
void (async () => {
setDownloadError(null);
try {
if (state.status === "ready") downloadArtifactBytes(name, mimeType, state.bytes);
else await downloadArtifact(target, artifactId, name, mimeType);
} catch {
setDownloadError(t`Could not download ${name}. Try again.`);
}
})()
}
onClick={() => downloadArtifact(artifactId, name)}
>
<Download />
</Button>
@ -181,11 +146,6 @@ function MarkdownPreview({
<X />
</DialogClose>
</header>
{downloadError ? (
<div className="shrink-0 px-5 pt-4">
<DownloadError message={downloadError} />
</div>
) : null}
<div className="min-h-0 flex-1 overflow-y-auto">
<article className="mx-auto w-full max-w-[760px] px-8 py-10 text-ui-body leading-7 text-foreground sm:px-12 sm:py-12">
{state.status === "loading" ? (
@ -204,23 +164,3 @@ function MarkdownPreview({
</>
);
}
function DownloadError({ message }: { message: string }) {
return (
<div role="alert" className="mt-2 text-left text-ui-body-sm text-destructive">
{message}
</div>
);
}
function formatBytes(size: number) {
const locale = i18n.locale || "en";
if (size < 1024) return t`${size} B`;
const format = (value: number) =>
new Intl.NumberFormat(locale, {
minimumFractionDigits: 1,
maximumFractionDigits: 1,
}).format(value);
if (size < 1024 * 1024) return t`${format(size / 1024)} KB`;
return t`${format(size / (1024 * 1024))} MB`;
}

View File

@ -0,0 +1,99 @@
// @vitest-environment jsdom
import { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { renderToStaticMarkup } from "react-dom/server";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("@lingui/react/macro", () => {
const t = (parts: TemplateStringsArray, ...values: unknown[]) =>
parts.reduce(
(text, part, index) => text + part + (index < values.length ? values[index] : ""),
"",
);
return { useLingui: () => ({ t }) };
});
import { MediaPlayer } from "./MediaPlayer";
let container: HTMLDivElement;
let root: Root;
beforeEach(() => {
vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
container = document.createElement("div");
document.body.append(container);
root = createRoot(container);
});
afterEach(async () => {
await act(async () => root.unmount());
container.remove();
vi.unstubAllGlobals();
});
describe("MediaPlayer", () => {
it("loads nothing before it is near the viewport and fits vertical video to the column", () => {
const html = renderToStaticMarkup(
<MediaPlayer artifactId="art-1" kind="video" name="clip.mp4" width={1080} height={1920} />,
);
expect(html).toContain('aria-label="clip.mp4"');
expect(html).not.toContain("src=");
expect(html).toContain("width:270px");
expect(html).toContain("aspect-ratio:270 / 480");
});
it("streams video with its poster from the artifact route once visible", async () => {
vi.stubGlobal(
"IntersectionObserver",
class {
constructor(private readonly callback: IntersectionObserverCallback) {}
observe() {
this.callback([{ isIntersecting: true } as IntersectionObserverEntry], this as never);
}
disconnect() {}
},
);
await act(async () =>
root.render(
<MediaPlayer artifactId="art-1" kind="video" name="clip.mp4" posterArtifactId="art-2" />,
),
);
const video = container.querySelector("video")!;
expect(video.getAttribute("src")).toBe("/api/artifacts/art-1");
expect(video.getAttribute("poster")).toBe("/api/artifacts/art-2");
expect(video.getAttribute("preload")).toBe("metadata");
expect(video.hasAttribute("controls")).toBe(true);
});
it("sizes the frame from the file when the block has no dimensions", async () => {
await act(async () =>
root.render(<MediaPlayer artifactId="art-4" kind="video" name="up.mp4" />),
);
const frame = container.querySelector<HTMLDivElement>('[data-testid="media-video"]')!;
expect(frame.style.width).toBe("420px");
const video = container.querySelector("video")!;
Object.defineProperty(video, "videoWidth", { value: 1080 });
Object.defineProperty(video, "videoHeight", { value: 1920 });
await act(async () => {
video.dispatchEvent(new Event("loadedmetadata"));
});
expect(frame.style.width).toBe("270px");
});
it("falls back to a download link when the browser cannot play the file", async () => {
await act(async () =>
root.render(<MediaPlayer artifactId="art-3" kind="audio" name="memo.ogg" />),
);
const audio = container.querySelector("audio")!;
expect(audio.getAttribute("src")).toBe("/api/artifacts/art-3");
await act(async () => {
audio.dispatchEvent(new Event("error"));
});
expect(container.querySelector("audio")).toBeNull();
const link = container.querySelector("a")!;
expect(link.getAttribute("href")).toBe("/api/artifacts/art-3?download=1");
expect(link.getAttribute("download")).toBe("memo.ogg");
expect(link.getAttribute("aria-label")).toBe("Download memo.ogg");
});
});

View File

@ -0,0 +1,129 @@
import { useLingui } from "@lingui/react/macro";
import { buttonVariants } from "@rakazo/ui-web";
import { Download } from "lucide-react";
import { type RefObject, useEffect, useRef, useState } from "react";
import { artifactUrl } from "../../lib/artifact-url";
export type MediaPlayerProps = {
artifactId: string;
kind: "video" | "audio";
name: string;
posterArtifactId?: string;
width?: number;
height?: number;
};
const MAX_VIDEO_WIDTH = 420;
const MAX_VIDEO_HEIGHT = 480;
/** Plays a video or audio artifact in place; loads nothing until it is near the viewport. */
export function MediaPlayer({
artifactId,
kind,
name,
posterArtifactId,
width,
height,
}: MediaPlayerProps) {
const container = useRef<HTMLDivElement>(null);
const near = useNearViewport(container);
const [failed, setFailed] = useState(false);
const [frame, setFrame] = useState<{ width: number; height: number } | null>(null);
const src = near ? artifactUrl(artifactId) : undefined;
if (failed) return <MediaFallback artifactId={artifactId} name={name} />;
if (kind === "audio") {
return (
<div ref={container} className="w-72 max-w-full" data-testid="media-audio">
{/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */}
<audio
src={src}
controls
preload="metadata"
aria-label={name}
className="w-full"
onError={() => setFailed(true)}
/>
</div>
);
}
// Uploads carry no dimensions; size the frame from the file once its metadata loads.
const box = videoBox(width ?? frame?.width, height ?? frame?.height);
return (
<div
ref={container}
className="max-w-full overflow-hidden rounded-[20px] bg-muted"
style={{ width: box.width, aspectRatio: `${box.width} / ${box.height}` }}
data-testid="media-video"
>
{/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */}
<video
src={src}
poster={near && posterArtifactId ? artifactUrl(posterArtifactId) : undefined}
controls
playsInline
preload="metadata"
aria-label={name}
className="block size-full object-contain"
onError={() => setFailed(true)}
onLoadedMetadata={(event) => {
const { videoWidth, videoHeight } = event.currentTarget;
if (videoWidth > 0 && videoHeight > 0) {
setFrame({ width: videoWidth, height: videoHeight });
}
}}
/>
</div>
);
}
/** Fits the frame into the chat column, defaulting to 16:9 until the size is known. */
function videoBox(width?: number, height?: number) {
const w = width && width > 0 ? width : 16;
const h = height && height > 0 ? height : 9;
const scale = Math.min(MAX_VIDEO_WIDTH / w, MAX_VIDEO_HEIGHT / h);
return { width: Math.round(w * scale), height: Math.round(h * scale) };
}
function MediaFallback({ artifactId, name }: { artifactId: string; name: string }) {
const { t } = useLingui();
return (
<div className="flex max-w-full items-center gap-2 rounded-[20px] border border-border bg-muted py-2 ps-4 pe-2 text-ui-body text-muted-foreground">
<span className="min-w-0 truncate">{name}</span>
<a
href={artifactUrl(artifactId, { download: true })}
download={name}
aria-label={t`Download ${name}`}
title={t`Download ${name}`}
className={buttonVariants({ variant: "ghost", size: "icon-sm" })}
>
<Download />
</a>
</div>
);
}
function useNearViewport(ref: RefObject<HTMLElement | null>) {
const [near, setNear] = useState(false);
useEffect(() => {
const element = ref.current;
if (!element || typeof IntersectionObserver === "undefined") {
setNear(true);
return;
}
const observer = new IntersectionObserver(
(entries) => {
if (entries.some((entry) => entry.isIntersecting)) {
setNear(true);
observer.disconnect();
}
},
{ rootMargin: "320px" },
);
observer.observe(element);
return () => observer.disconnect();
}, [ref]);
return near;
}

View File

@ -1,40 +1,29 @@
import { rpc } from "./rpc";
import { t } from "@lingui/core/macro";
import { artifactUrl } from "./artifact-url";
export type ArtifactTarget = { botId: string } | { groupId: string };
export function decodeArtifactBase64(contentBase64: string): Uint8Array {
const binary = atob(contentBase64);
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) {
bytes[index] = binary.charCodeAt(index);
}
return bytes;
}
export async function fetchArtifactBytes(
target: ArtifactTarget,
artifactId: string,
): Promise<Uint8Array> {
const artifact = await rpc.artifacts.get({ ...target, artifactId });
return decodeArtifactBase64(artifact.contentBase64);
export async function fetchArtifactBytes(artifactId: string): Promise<Uint8Array> {
const response = await fetch(artifactUrl(artifactId), { credentials: "include" });
if (!response.ok) throw new Error(t`Could not load this file.`);
return new Uint8Array(await response.arrayBuffer());
}
export function downloadArtifactBytes(name: string, mimeType: string, bytes: Uint8Array): void {
const blob = new Blob([new Uint8Array(bytes)], { type: mimeType });
const url = URL.createObjectURL(blob);
saveUrl(url, name);
window.setTimeout(() => URL.revokeObjectURL(url), 60_000);
}
/** Lets the browser stream the download from the artifact route instead of buffering it. */
export function downloadArtifact(artifactId: string, name: string): void {
saveUrl(artifactUrl(artifactId, { download: true }), name);
}
function saveUrl(url: string, name: string) {
const anchor = document.createElement("a");
anchor.href = url;
anchor.download = name;
anchor.click();
window.setTimeout(() => URL.revokeObjectURL(url), 60_000);
}
export async function downloadArtifact(
target: ArtifactTarget,
artifactId: string,
name: string,
mimeType: string,
): Promise<void> {
const bytes = await fetchArtifactBytes(target, artifactId);
downloadArtifactBytes(name, mimeType, bytes);
}

View File

@ -0,0 +1,83 @@
import { ATTACHMENT_MAX_BYTES } from "@rakazo/contracts";
import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("@lingui/core/macro", () => ({
t: (parts: TemplateStringsArray, ...values: unknown[]) =>
parts.reduce(
(text, part, index) => text + part + (index < values.length ? values[index] : ""),
"",
),
}));
const create = vi.hoisted(() => vi.fn());
vi.mock("./rpc", () => ({
rpc: { artifacts: { create } },
withSpaceHeaders: (init: HeadersInit) => new Headers(init),
}));
import {
attachmentMaxBytes,
uploadArtifactFile,
uploadAttachment,
usesStreamingUpload,
} from "./artifact-upload";
const artifact = {
id: "art-1",
botId: "bot-1",
groupId: null,
runId: null,
name: "clip.mp4",
mimeType: "video/mp4",
size: 4,
createdAt: "2026-10-03T12:00:00.000Z",
};
afterEach(() => {
vi.unstubAllGlobals();
create.mockReset();
});
describe("artifact uploads", () => {
it("streams media and large files and keeps small files on the inline RPC", () => {
expect(usesStreamingUpload(10, "video/mp4")).toBe(true);
expect(usesStreamingUpload(10, "audio/mpeg")).toBe(true);
expect(usesStreamingUpload(10, "image/png")).toBe(false);
expect(usesStreamingUpload(ATTACHMENT_MAX_BYTES + 1, "image/png")).toBe(true);
expect(attachmentMaxBytes("video/mp4")).toBe(500 * 1024 * 1024);
expect(attachmentMaxBytes("text/plain")).toBe(25 * 1024 * 1024);
});
it("posts the raw file with its target, name and type", async () => {
const fetch = vi.fn(async () => Response.json(artifact, { status: 201 }));
vi.stubGlobal("fetch", fetch);
const file = new File(["data"], "clip.mp4", { type: "video/mp4" });
await expect(uploadAttachment({ groupId: "group-1" }, file, "video/mp4")).resolves.toEqual(
artifact,
);
const [url, init] = fetch.mock.calls[0] as unknown as [string, RequestInit];
expect(url).toBe("/api/artifacts?groupId=group-1&name=clip.mp4");
expect(init.method).toBe("POST");
expect(init.credentials).toBe("include");
expect(new Headers(init.headers).get("content-type")).toBe("video/mp4");
expect(init.body).toBe(file);
expect(create).not.toHaveBeenCalled();
});
it("surfaces the server's reason, or a generic one", async () => {
vi.stubGlobal(
"fetch",
vi.fn(async () => Response.json({ error: "File is too large." }, { status: 413 })),
);
await expect(
uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"),
).rejects.toThrow("File is too large.");
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response("bad gateway", { status: 502 })),
);
await expect(
uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"),
).rejects.toThrow("Could not upload big.mp4. Try again.");
});
});

View File

@ -0,0 +1,68 @@
import { t } from "@lingui/core/macro";
import {
ARTIFACT_MAX_BYTES_BY_KIND,
ArtifactSchema,
ATTACHMENT_MAX_BYTES,
attachmentKindOf,
} from "@rakazo/contracts";
import type { ArtifactTarget } from "./artifact-open";
import { rpc, withSpaceHeaders } from "./rpc";
type UploadedArtifact = ReturnType<typeof ArtifactSchema.parse>;
/** Largest file of this type the composer accepts. */
export function attachmentMaxBytes(mimeType: string): number {
return ARTIFACT_MAX_BYTES_BY_KIND[attachmentKindOf(mimeType)];
}
/** Media and anything above the inline RPC cap upload as a raw stream instead of base64. */
export function usesStreamingUpload(size: number, mimeType: string): boolean {
const kind = attachmentKindOf(mimeType);
return size > ATTACHMENT_MAX_BYTES || kind === "video" || kind === "audio";
}
export async function uploadAttachment(
target: ArtifactTarget,
file: File,
mimeType: string,
): Promise<UploadedArtifact> {
if (usesStreamingUpload(file.size, mimeType)) {
return uploadArtifactFile(target, file, file.name, mimeType);
}
const contentBase64 = await readFileAsBase64(file);
return rpc.artifacts.create({ ...target, name: file.name, mimeType, contentBase64 });
}
/** Uploads a file as a raw streamed body to the artifact route. */
export async function uploadArtifactFile(
target: ArtifactTarget,
file: Blob,
name: string,
mimeType: string,
): Promise<UploadedArtifact> {
const query = new URLSearchParams({ ...target, name });
const response = await fetch(`/api/artifacts?${query}`, {
method: "POST",
headers: withSpaceHeaders({ "content-type": mimeType }),
credentials: "include",
body: file,
});
const body: unknown = await response.json().catch(() => null);
const artifact = ArtifactSchema.safeParse(body);
if (response.ok && artifact.success) return artifact.data;
const error = (body as { error?: unknown } | null)?.error;
throw new Error(typeof error === "string" ? error : t`Could not upload ${name}. Try again.`);
}
function readFileAsBase64(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const result = typeof reader.result === "string" ? reader.result : "";
const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result;
resolve(base64);
};
reader.onerror = () => reject(reader.error ?? new Error("Failed to read file"));
reader.readAsDataURL(file);
});
}

View File

@ -1,38 +1,8 @@
import { t } from "@lingui/core/macro";
import { ArtifactSchema } from "@rakazo/contracts";
import type { ArtifactTarget } from "./artifact-open";
import { withSpaceHeaders } from "./rpc";
const ARTIFACT_PATH = "/api/artifacts";
/**
* Same-origin URL that streams an artifact with Range support, for `<video>`, `<audio>`, `<img>`
* and downloads. The session cookie authorizes it; `download` forces a save instead of inline.
*/
export function artifactUrl(artifactId: string, options: { download?: boolean } = {}): string {
const path = `${ARTIFACT_PATH}/${encodeURIComponent(artifactId)}`;
const path = `/api/artifacts/${encodeURIComponent(artifactId)}`;
return options.download ? `${path}?download=1` : path;
}
/** Uploads a file as a raw streamed body, for media and files above the inline RPC cap. */
export async function uploadArtifactFile(
target: ArtifactTarget,
file: Blob,
name: string,
mimeType: string,
): Promise<ReturnType<typeof ArtifactSchema.parse>> {
const query = new URLSearchParams(
"botId" in target ? { botId: target.botId, name } : { groupId: target.groupId, name },
);
const response = await fetch(`${ARTIFACT_PATH}?${query}`, {
method: "POST",
headers: withSpaceHeaders({ "content-type": mimeType }),
credentials: "include",
body: file,
});
const body: unknown = await response.json().catch(() => null);
const artifact = ArtifactSchema.safeParse(body);
if (response.ok && artifact.success) return artifact.data;
const error = (body as { error?: unknown } | null)?.error;
throw new Error(typeof error === "string" ? error : t`Could not upload ${name}. Try again.`);
}

View File

@ -0,0 +1,12 @@
import { i18n } from "@lingui/core";
import { t } from "@lingui/core/macro";
/** Human file size in the UI locale: B, KB or MB with at most one decimal. */
export function formatBytes(size: number): string {
const locale = i18n.locale || "en";
if (size < 1024) return t`${size} B`;
const format = (value: number) =>
new Intl.NumberFormat(locale, { maximumFractionDigits: 1 }).format(value);
if (size < 1024 * 1024) return t`${format(size / 1024)} KB`;
return t`${format(size / (1024 * 1024))} MB`;
}

View File

@ -41,7 +41,6 @@ import type {
} from "@rakazo/contracts";
import {
ATTACHMENT_ALLOWED_MIME_TYPES,
ATTACHMENT_MAX_BYTES,
ATTACHMENT_MAX_COUNT,
canReactToThreadMessage,
MESSAGE_REACTIONS,
@ -169,6 +168,7 @@ import { ComputerUpdateProgress } from "../components/ComputerUpdateProgress";
import { DataTrustPanel } from "../components/DataTrustPanel";
import { MainInbox } from "../components/MainInbox";
import { MessageHoverMetadata } from "../components/MessageHoverMetadata";
import { MediaPlayer } from "../components/media/MediaPlayer";
import { OoGroupStage, OoStage } from "../components/stage/OoStage";
import { TradingModeIndicator } from "../components/TradingModeIndicator";
import { SkillDraftCard } from "../components/teach/SkillDraftCard";
@ -180,6 +180,7 @@ import { useSidebarResize } from "../hooks/useSidebarResize";
import { readActivityMode, writeActivityMode } from "../lib/activity-mode";
import { needsYouChipLabel, rosterActivitySubtitle, rosterStatusDot } from "../lib/agent-activity";
import type { ArtifactTarget } from "../lib/artifact-open";
import { attachmentMaxBytes, uploadAttachment } from "../lib/artifact-upload";
import { authClient } from "../lib/auth";
import { takeInitialBootstrap } from "../lib/bootstrap";
import {
@ -197,6 +198,7 @@ import { browserPreviewActivity } from "../lib/browser-preview-context";
import { loadComputerScreen } from "../lib/computer-screen";
import { desktopBridge } from "../lib/desktop";
import { scheduleFocusPrompt } from "../lib/focus-prompt";
import { formatBytes } from "../lib/format-bytes";
import {
isWorkingRunStatus,
resolveActiveGroupSpeaker,
@ -2711,15 +2713,16 @@ export function ShellPage() {
skipped.push(t`${file.name} (max ${ATTACHMENT_MAX_COUNT} attachments)`);
continue;
}
if (file.size > ATTACHMENT_MAX_BYTES) {
skipped.push(t`${file.name} (over 10 MiB)`);
continue;
}
const mimeType = inferAttachmentMimeType(file.name, file.type);
if (!mimeType) {
skipped.push(file.name);
continue;
}
const maxBytes = attachmentMaxBytes(mimeType);
if (file.size > maxBytes) {
skipped.push(t`${file.name} (over ${formatBytes(maxBytes)})`);
continue;
}
next.push({
id: `${file.name}-${file.size}-${file.lastModified}-${next.length}`,
threadKey,
@ -2859,11 +2862,10 @@ export function ShellPage() {
if (!mimeType) {
throw new Error(t`Unsupported file type: ${pending.file.name}`);
}
const contentBase64 = await readFileAsBase64(pending.file);
const artifact = await rpc.artifacts.create(
groupTarget
? { groupId: groupTarget, name: pending.file.name, mimeType, contentBase64 }
: { botId: botTarget!, name: pending.file.name, mimeType, contentBase64 },
const artifact = await uploadAttachment(
groupTarget ? { groupId: groupTarget } : { botId: botTarget! },
pending.file,
mimeType,
);
artifactIds.push(artifact.id);
}
@ -7186,6 +7188,8 @@ function MentionChipIcon({ mention }: { mention: ComposerMention }) {
);
}
const ATTACHMENT_BLOCK_KINDS = new Set(["image", "file", "video", "audio", "media"]);
function previewMessageText(message: ThreadMessage): string {
const text = message.blocks
.map((block) => (block.kind === "text" || block.kind === "channel_message" ? block.text : ""))
@ -7193,7 +7197,7 @@ function previewMessageText(message: ThreadMessage): string {
.join(" ")
.trim();
if (text) return text;
if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) {
if (message.blocks.some((block) => ATTACHMENT_BLOCK_KINDS.has(block.kind))) {
return t`Attachment`;
}
return t`Message`;
@ -7841,10 +7845,23 @@ const MessageView = memo(function MessageView({
key={i}
className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`}
>
<ArtifactImage
target={artifactTarget}
<ArtifactImage artifactId={block.artifactId} name={block.name} />
</div>
);
}
if (block.kind === "video" || block.kind === "audio") {
return (
<div
key={i}
className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`}
>
<MediaPlayer
artifactId={block.artifactId}
kind={block.kind}
name={block.name}
posterArtifactId={block.kind === "video" ? block.posterArtifactId : undefined}
width={block.kind === "video" ? block.width : undefined}
height={block.kind === "video" ? block.height : undefined}
/>
</div>
);
@ -7856,7 +7873,6 @@ const MessageView = memo(function MessageView({
className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`}
>
<ArtifactFileCard
target={artifactTarget}
artifactId={block.artifactId}
name={block.name}
mimeType={block.mimeType}
@ -8059,16 +8075,3 @@ function newClientNonce(): string {
}
return `m-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
}
function readFileAsBase64(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const result = typeof reader.result === "string" ? reader.result : "";
const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result;
resolve(base64);
};
reader.onerror = () => reject(reader.error ?? new Error("Failed to read file"));
reader.readAsDataURL(file);
});
}

View File

@ -1,11 +1,18 @@
import { Trans, useLingui } from "@lingui/react/macro";
import type { MessageBlock } from "@rakazo/contracts";
import { abortableDelay } from "@rakazo/core";
import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web";
import { X } from "lucide-react";
import {
Button,
buttonVariants,
Dialog,
DialogClose,
DialogContent,
DialogTitle,
} from "@rakazo/ui-web";
import { Download, X } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { BuiCard, SuccessPop } from "../../components/ai/primitives";
import { type ArtifactTarget, decodeArtifactBase64 } from "../../lib/artifact-open";
import { artifactUrl } from "../../lib/artifact-url";
import { chartViewport } from "../../lib/chart-viewport";
import { connectMcpOauth } from "../../lib/mcp-connect";
import { rpc } from "../../lib/rpc";
@ -461,104 +468,65 @@ export function ChartBlockView({
);
}
export function ArtifactImage({
target,
artifactId,
name,
}: {
target: ArtifactTarget;
artifactId: string;
name: string;
}) {
export function ArtifactImage({ artifactId, name }: { artifactId: string; name: string }) {
const { t } = useLingui();
const [src, setSrc] = useState<string | null>(null);
const [open, setOpen] = useState(false);
const [visible, setVisible] = useState(false);
const container = useRef<HTMLDivElement>(null);
const targetBotId = "botId" in target ? target.botId : undefined;
const targetGroupId = "groupId" in target ? target.groupId : undefined;
const [failed, setFailed] = useState(false);
const src = artifactUrl(artifactId);
useEffect(() => {
const element = container.current;
if (!element || typeof IntersectionObserver === "undefined") {
setVisible(true);
return;
}
const observer = new IntersectionObserver(
(entries) => {
if (entries.some((entry) => entry.isIntersecting)) {
setVisible(true);
observer.disconnect();
}
},
{ rootMargin: "320px" },
if (failed) {
return (
<div className="rounded-[20px] border border-border bg-muted px-4 py-3 text-ui-body text-muted-foreground">
{name}
</div>
);
observer.observe(element);
return () => observer.disconnect();
}, []);
useEffect(() => {
if (!visible) return;
let cancelled = false;
let objectUrl: string | null = null;
setSrc(null);
void rpc.artifacts
.get(
targetBotId
? { botId: targetBotId, artifactId }
: { groupId: targetGroupId ?? "", artifactId },
)
.then((artifact) => {
const bytes = decodeArtifactBase64(artifact.contentBase64);
objectUrl = URL.createObjectURL(
new Blob([new Uint8Array(bytes)], { type: artifact.mimeType }),
);
if (cancelled) URL.revokeObjectURL(objectUrl);
else setSrc(objectUrl);
})
.catch(() => undefined);
return () => {
cancelled = true;
if (objectUrl) URL.revokeObjectURL(objectUrl);
};
}, [artifactId, targetBotId, targetGroupId, visible]);
}
return (
<div ref={container}>
{src ? (
<button
type="button"
onClick={() => setOpen(true)}
className="max-w-[240px] overflow-hidden rounded-[20px]"
<div>
<button
type="button"
onClick={() => setOpen(true)}
className="max-w-[240px] overflow-hidden rounded-[20px]"
>
<img
src={src}
alt={name}
loading="lazy"
decoding="async"
onError={() => setFailed(true)}
className="max-h-48 w-full object-cover"
/>
</button>
<Dialog open={open} onOpenChange={setOpen}>
<DialogContent
showCloseButton={false}
className="w-auto max-w-none bg-transparent p-0 ring-0 sm:max-w-none"
>
<img src={src} alt={name} className="max-h-48 w-full object-cover" />
</button>
) : (
<div className="rounded-[20px] border border-border bg-muted px-4 py-3 text-ui-body text-muted-foreground">
{name}
</div>
)}
{src ? (
<Dialog open={open} onOpenChange={setOpen}>
<DialogContent
showCloseButton={false}
className="w-auto max-w-none bg-transparent p-0 ring-0 sm:max-w-none"
>
<DialogTitle className="sr-only">{name}</DialogTitle>
<DialogTitle className="sr-only">{name}</DialogTitle>
<div className="absolute end-2 top-2 flex gap-1">
<a
href={artifactUrl(artifactId, { download: true })}
download={name}
aria-label={t`Download ${name}`}
title={t`Download ${name}`}
className={buttonVariants({ variant: "ghost", size: "icon-sm" })}
>
<Download />
</a>
<DialogClose
aria-label={t`Close image preview`}
render={<Button variant="ghost" size="icon-sm" className="absolute end-2 top-2" />}
render={<Button variant="ghost" size="icon-sm" />}
>
<X />
</DialogClose>
<img
src={src}
alt={name}
className="max-h-[85vh] max-w-[90vw] rounded-xl object-contain"
/>
</DialogContent>
</Dialog>
) : null}
</div>
<img
src={src}
alt={name}
className="max-h-[85vh] max-w-[90vw] rounded-xl object-contain"
/>
</DialogContent>
</Dialog>
</div>
);
}

View File

@ -47,6 +47,7 @@ export async function storeArtifactStream(
stream: ArtifactStreamPut["stream"];
maxBytes?: number;
signal?: AbortSignal;
operationId?: string;
},
) {
validateAttachmentMimeType(input.mimeType);
@ -54,7 +55,8 @@ export async function storeArtifactStream(
input.maxBytes ?? Number.POSITIVE_INFINITY,
artifactMaxBytes(input.mimeType),
);
const operationId = `artifact-store:${input.runId ?? input.botId ?? input.userId}`;
const operationId =
input.operationId ?? `artifact-store:${input.runId ?? input.botId ?? input.userId}`;
const context: AdapterContext = {
operationId,
traceId: operationId,

View File

@ -162,7 +162,7 @@ export const builtinAgentTools: ConnectorTool[] = [
{
name: "attach_file",
description:
"Attach a workspace file from this bot's home to the chat thread as an image or common file. The file stays in place; users can open it from the message.",
"Attach a workspace file from this bot's home to the chat thread as an image, video, audio or common file. The file stays in place; users can open or play it from the message.",
inputSchema: {
type: "object",
properties: { path: { type: "string" } },

View File

@ -17,6 +17,18 @@ export function toComputerRef(computer: {
};
}
/**
* Remote computers move files as base64 JSON (the Docker supervisor accepts 16 MiB requests), so
* files copied to or read from them stay below this size. Local computers have no transfer cap.
*/
export const REMOTE_COMPUTER_FILE_TRANSFER_MAX_BYTES = 12 * 1024 * 1024;
export function computerFileTransferMaxBytes(computer: Pick<ComputerRef, "kind">): number {
return computer.kind === "desktop" || computer.kind === "fake"
? Number.POSITIVE_INFINITY
: REMOTE_COMPUTER_FILE_TRANSFER_MAX_BYTES;
}
const EMPTY_PNG = Uint8Array.from(
Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",

View File

@ -197,6 +197,7 @@ import { withComputerScreenAvailability } from "./computer-screens.js";
import {
assertTeamShellCommandAllowed,
assertTeamWorkspacePathAllowed,
computerFileTransferMaxBytes,
displayBotWorkspacePath,
filterTeamWorkspaceEntries,
resolveBotWorkspaceCwd,
@ -369,8 +370,10 @@ import {
} from "./takeover-resume.js";
import { getActiveTeachingSession, parsePlaybook } from "./teaching-session.js";
import {
attachableFileMaxBytes,
attachWorkspaceFileToThread,
currentTurnFilesInstruction,
formatMiB,
materializeCurrentTurnFiles,
} from "./thread-artifacts.js";
import { advanceToolCallLoopGuard } from "./tool-loop.js";
@ -2368,6 +2371,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
spaceId: run.spaceId,
userId: run.userId,
botId: bot.id,
groupId: thread.groupId ?? undefined,
runId: run.id,
filePath: outPath,
bytes: png,
@ -2393,18 +2397,23 @@ export function createRunExecutor(deps: ExecutorDeps) {
return finish({ error: "artifact storage unavailable", path: filePath });
}
const storedPath = resolveStoredWorkspacePath(filePath);
let bytes: Uint8Array;
try {
bytes = await deps.sandbox.readFile(computer, storedPath, context, {
maxBytes: ATTACHMENT_MAX_BYTES,
});
} catch {
return finish({ error: "file not found or unreadable", path: filePath });
}
const mimeType = inferAttachmentMimeType(filePath);
if (!mimeType) {
return finish({ error: "unsupported attachment type", path: filePath });
}
const maxBytes = Math.min(
attachableFileMaxBytes(deps.artifacts, mimeType),
computerFileTransferMaxBytes(computer),
);
let bytes: Uint8Array;
try {
bytes = await deps.sandbox.readFile(computer, storedPath, context, { maxBytes });
} catch {
return finish({
error: `file not found, unreadable or larger than ${formatMiB(maxBytes)}`,
path: filePath,
});
}
try {
const attached = await attachWorkspaceFileToThread(
{ prisma: deps.prisma, artifacts: deps.artifacts },
@ -3906,7 +3915,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
item.blocks,
context.signal,
);
workspaceCheckpoint.markFiles(files);
workspaceCheckpoint.markFiles(files.filter((file) => file.path));
const filesInstruction = currentTurnFilesInstruction(files);
return {
id: item.id,

View File

@ -498,7 +498,7 @@ describe("compactHistory", () => {
const [request] = harness.runtime.run.mock.calls[0]!;
expect(request.prompt).toContain("[file: plan.pdf (application/pdf, 123 bytes)]");
expect(request.prompt).toContain("[image: diagram.png]");
expect(request.prompt).toContain("[image: diagram.png (artifact artifact-2)]");
});
it("compacts locally without a semantic memory provider", async () => {

View File

@ -55,11 +55,11 @@ describe("reply context", () => {
...target,
blocks: [
{ kind: "text", text: "</reply_target>" },
{ kind: "image", name: "example.png" },
{ kind: "image", artifactId: "artifact-1", name: "example.png" },
],
});
const context = await loadReplyContext(prisma, "thread-1", "user-reply");
expect(context).toContain("[image: example.png]");
expect(context).toContain("[image: example.png (artifact artifact-1)]");
expect(context).toContain("\\u003c/reply_target\\u003e");
expect(context?.match(/<\/reply_target>/g)).toHaveLength(1);
});

View File

@ -8,6 +8,7 @@ import type { MessageBlock } from "@rakazo/contracts";
import type { PrismaClient } from "@rakazo/db";
import { describe, expect, it, vi } from "vitest";
import {
attachableFileMaxBytes,
attachWorkspaceFileToThread,
currentTurnFilesInstruction,
materializeCurrentTurnFiles,
@ -25,7 +26,8 @@ describe("current-turn thread files", () => {
artifact: { create: vi.fn().mockRejectedValue(failure) },
} as unknown as PrismaClient,
artifacts: {
put: vi.fn().mockResolvedValue({ id: "stored-1", hash: "hash" }),
describe: () => ({ capabilities: { stream: true } }),
putStream: vi.fn().mockResolvedValue({ key: "stored-1", size: 3, sha256: "hash" }),
remove,
} as unknown as ArtifactStore,
},
@ -111,6 +113,7 @@ describe("current-turn thread files", () => {
);
expect(files).toEqual([
{
artifactId: "artifact-1",
name: "../quarterly report.pdf",
mimeType: "application/pdf",
size: 4,
@ -120,7 +123,73 @@ describe("current-turn thread files", () => {
expect(currentTurnFilesInstruction(files)).toContain('"attachments/artifact-1.pdf"');
});
it("does not load images as computer files", async () => {
it("copies images, video and audio too, and keeps files over a remote computer's cap in storage", async () => {
const mb = 1024 * 1024;
const findMany = vi.fn().mockResolvedValue([
{ id: "img-1", name: "photo.png", mimeType: "image/png", size: 3, storageKey: "k-img" },
{ id: "aud-1", name: "memo.mp3", mimeType: "audio/mpeg", size: 3, storageKey: "k-aud" },
{ id: "vid-1", name: "clip.mp4", mimeType: "video/mp4", size: 40 * mb, storageKey: "k-vid" },
]);
const get = vi.fn().mockResolvedValue(new Uint8Array([1, 2, 3]));
const writeFile = vi.fn().mockResolvedValue(undefined);
const context: AdapterContext & { botId: string } = {
operationId: "run-1",
traceId: "run-1",
spaceId: "workspace-1",
userId: "user-1",
botId: "bot-1",
signal: new AbortController().signal,
};
const blocks: MessageBlock[] = [
{ kind: "image", artifactId: "img-1", name: "photo.png", mimeType: "image/png" },
{ kind: "audio", artifactId: "aud-1", name: "memo.mp3", mimeType: "audio/mpeg", size: 3 },
{
kind: "video",
artifactId: "vid-1",
name: "clip.mp4",
mimeType: "video/mp4",
size: 40 * mb,
},
{ kind: "text", text: "see these" },
];
const deps = {
prisma: { artifact: { findMany } } as unknown as PrismaClient,
artifacts: { get } as unknown as ArtifactStore,
sandbox: { writeFile } as unknown as SandboxProvider,
};
const remote = await materializeCurrentTurnFiles(deps, blocks, {
context,
computer: { id: "computer-1", botId: "bot-1", kind: "docker", providerRef: "docker-1" },
computerMode: "dedicated",
});
expect(remote.map((file) => [file.artifactId, file.path])).toEqual([
["img-1", "attachments/img-1.png"],
["aud-1", "attachments/aud-1.mp3"],
["vid-1", null],
]);
expect(get).not.toHaveBeenCalledWith("k-vid", context);
const instruction = currentTurnFilesInstruction(remote);
expect(instruction).toContain('"attachments/img-1.png"');
expect(instruction).toContain("too large to copy to your computer");
expect(instruction).toContain('"clip.mp4" (video/mp4, 41943040 bytes): artifact vid-1');
writeFile.mockClear();
const local = await materializeCurrentTurnFiles(deps, blocks, {
context,
computer: { id: "computer-2", botId: "bot-1", kind: "desktop", providerRef: "desktop-1" },
computerMode: "dedicated",
});
expect(local.map((file) => file.path)).toEqual([
"attachments/img-1.png",
"attachments/aud-1.mp3",
"attachments/vid-1.mp4",
]);
expect(writeFile).toHaveBeenCalledTimes(3);
expect(currentTurnFilesInstruction(local)).not.toContain("too large");
});
it("skips loading when a turn has no attachments", async () => {
const findMany = vi.fn();
const files = await materializeCurrentTurnFiles(
{
@ -128,14 +197,7 @@ describe("current-turn thread files", () => {
artifacts: {} as ArtifactStore,
sandbox: {} as SandboxProvider,
},
[
{
kind: "image",
artifactId: "image-1",
name: "photo.png",
mimeType: "image/png",
},
],
[{ kind: "text", text: "hello" }],
{
context: {
operationId: "run-1",
@ -158,4 +220,12 @@ describe("current-turn thread files", () => {
expect(files).toEqual([]);
expect(findMany).not.toHaveBeenCalled();
});
it("caps attachments by kind when the store streams and at the inline cap otherwise", () => {
const streaming = { describe: () => ({ capabilities: { stream: true } }) } as ArtifactStore;
const buffered = { describe: () => ({ capabilities: { stream: false } }) } as ArtifactStore;
expect(attachableFileMaxBytes(streaming, "video/mp4")).toBe(500 * 1024 * 1024);
expect(attachableFileMaxBytes(streaming, "text/plain")).toBe(25 * 1024 * 1024);
expect(attachableFileMaxBytes(buffered, "video/mp4")).toBe(10 * 1024 * 1024);
});
});

View File

@ -1,4 +1,3 @@
import { createHash } from "node:crypto";
import path from "node:path";
import type {
AdapterContext,
@ -12,18 +11,39 @@ import {
attachmentExtensionForMimeType,
inferAttachmentMimeType,
messageBlockForArtifact,
validateAttachmentMimeType,
} from "@rakazo/core";
import type { PrismaClient } from "@rakazo/db";
import { assertTeamWorkspacePathAllowed, resolveBotWorkspacePath } from "./computer-support.js";
import { artifactMaxBytes, storeArtifactStream } from "./artifact-files.js";
import {
assertTeamWorkspacePathAllowed,
computerFileTransferMaxBytes,
resolveBotWorkspacePath,
} from "./computer-support.js";
export type MaterializedThreadFile = {
artifactId: string;
name: string;
mimeType: string;
size: number;
path: string;
/** Workspace path, or null when the file is too large to copy to this computer. */
path: string | null;
};
type AttachmentBlock = Extract<MessageBlock, { kind: "image" | "video" | "audio" | "file" }>;
const ATTACHMENT_BLOCK_KINDS: ReadonlySet<string> = new Set(["image", "video", "audio", "file"]);
/** Media up to its kind cap when the store streams; otherwise the inline attachment cap. */
export function attachableFileMaxBytes(artifacts: ArtifactStore, mimeType: string): number {
return artifacts.describe().capabilities.stream
? artifactMaxBytes(mimeType)
: ATTACHMENT_MAX_BYTES;
}
export function formatMiB(bytes: number): string {
return `${Math.floor(bytes / (1024 * 1024))} MiB`;
}
export async function attachWorkspaceFileToThread(
deps: {
prisma: PrismaClient;
@ -39,60 +59,39 @@ export async function attachWorkspaceFileToThread(
bytes: Uint8Array;
operationId: string;
},
): Promise<{
artifactId: string;
block: Extract<MessageBlock, { kind: "image" | "video" | "audio" | "file" }>;
}> {
): Promise<{ artifactId: string; block: AttachmentBlock }> {
const name = path.basename(input.filePath) || input.filePath;
const mimeType = inferAttachmentMimeType(name);
if (!mimeType) {
throw new Error(`Unsupported attachment type for ${name}`);
}
validateAttachmentMimeType(mimeType);
if (input.bytes.byteLength > ATTACHMENT_MAX_BYTES) {
throw new Error("file exceeds the 10 MiB attachment limit");
const maxBytes = attachableFileMaxBytes(deps.artifacts, mimeType);
if (input.bytes.byteLength > maxBytes) {
throw new Error(`file exceeds the ${formatMiB(maxBytes)} attachment limit`);
}
const context = {
operationId: input.operationId,
traceId: input.operationId,
const bytes = input.bytes;
const row = await storeArtifactStream(deps, {
spaceId: input.spaceId,
userId: input.userId,
botId: input.botId,
signal: new AbortController().signal,
};
const stored = await deps.artifacts.put({ name, mimeType, bytes: input.bytes }, context);
const hash = createHash("sha256").update(input.bytes).digest("hex");
const row = await deps.prisma.artifact
.create({
data: {
spaceId: input.spaceId,
userId: input.userId,
botId: input.botId,
groupId: input.groupId,
runId: input.runId,
name,
mimeType,
size: input.bytes.byteLength,
hash,
storageKey: stored.id,
},
})
.catch(async (error) => {
await deps.artifacts.remove(stored.id, context).catch(() => undefined);
throw error;
});
return {
artifactId: row.id,
block: messageBlockForArtifact({
id: row.id,
name: row.name,
mimeType: row.mimeType,
size: row.size,
}),
};
groupId: input.groupId ?? null,
runId: input.runId,
name,
mimeType,
stream: (async function* () {
yield bytes;
})(),
maxBytes,
operationId: input.operationId,
});
return { artifactId: row.id, block: messageBlockForArtifact(row) };
}
/**
* Copies this turn's attachments (files, images, video and audio) into the bot's workspace so its
* tools can open them. Files above the computer's transfer cap stay in storage and are listed
* with their artifact id instead.
*/
export async function materializeCurrentTurnFiles(
deps: {
prisma: PrismaClient;
@ -108,28 +107,31 @@ export async function materializeCurrentTurnFiles(
markWorkspaceDirty?: () => void;
},
): Promise<MaterializedThreadFile[]> {
const fileBlocks = blocks?.filter(
(block): block is Extract<MessageBlock, { kind: "file" }> => block.kind === "file",
const attachmentBlocks = blocks?.filter((block): block is AttachmentBlock =>
ATTACHMENT_BLOCK_KINDS.has(block.kind),
);
if (!fileBlocks?.length) return [];
if (!attachmentBlocks?.length) return [];
const rows = await deps.prisma.artifact.findMany({
where: {
id: { in: fileBlocks.map((block) => block.artifactId) },
id: { in: attachmentBlocks.map((block) => block.artifactId) },
spaceId: input.context.spaceId,
userId: input.context.userId,
},
});
const byId = new Map(rows.map((row) => [row.id, row]));
const transferMaxBytes = computerFileTransferMaxBytes(input.computer);
const materialized: MaterializedThreadFile[] = [];
for (const block of fileBlocks) {
for (const block of attachmentBlocks) {
const row = byId.get(block.artifactId);
if (!row) throw new Error(`Attached file is unavailable: ${block.name}`);
const bytes = await deps.artifacts.get(row.storageKey, input.context);
if (bytes.byteLength > ATTACHMENT_MAX_BYTES) {
throw new Error(`Attached file exceeds the 10 MiB limit: ${row.name}`);
const entry = { artifactId: row.id, name: row.name, mimeType: row.mimeType, size: row.size };
if (row.size > Math.min(transferMaxBytes, artifactMaxBytes(row.mimeType))) {
materialized.push({ ...entry, path: null });
continue;
}
const bytes = await deps.artifacts.get(row.storageKey, input.context);
const relativePath = `attachments/${row.id}${attachmentExtensionForMimeType(row.mimeType)}`;
const storedPath = resolveBotWorkspacePath(
input.computerMode,
@ -146,24 +148,29 @@ export async function materializeCurrentTurnFiles(
},
input.context,
);
materialized.push({
name: row.name,
mimeType: row.mimeType,
size: row.size,
path: relativePath,
});
materialized.push({ ...entry, path: relativePath });
}
return materialized;
}
export function currentTurnFilesInstruction(files: readonly MaterializedThreadFile[]): string {
if (!files.length) return "";
const copied = files.filter((file) => file.path !== null);
const kept = files.filter((file) => file.path === null);
const describe = (file: MaterializedThreadFile) =>
`- ${JSON.stringify(file.name)} (${file.mimeType}, ${file.size} bytes)`;
return [
"Current-turn attached files are available on your computer. Inspect them before answering:",
...files.map(
(file) =>
`- ${JSON.stringify(file.name)} (${file.mimeType}, ${file.size} bytes): ${JSON.stringify(file.path)}`,
),
...(copied.length
? [
"Current-turn attached files are available on your computer. Inspect them before answering:",
...copied.map((file) => `${describe(file)}: ${JSON.stringify(file.path)}`),
]
: []),
...(kept.length
? [
"These current-turn attachments are too large to copy to your computer. Refer to them by artifact id; do not guess their contents:",
...kept.map((file) => `${describe(file)}: artifact ${file.artifactId}`),
]
: []),
].join("\n");
}

View File

@ -44,7 +44,7 @@ describe("attachment helpers", () => {
size: 99,
},
]),
).toBe("hello\n[image: shot.png]\n[file: brief.pdf (application/pdf, 99 bytes)]");
).toBe("hello\n[image: shot.png (artifact a1)]\n[file: brief.pdf (application/pdf, 99 bytes)]");
});
it("infers attachment mime types from extensions", () => {

View File

@ -117,12 +117,11 @@ export function blocksToAgentHistoryText(blocks: MessageBlock[]): string {
.map((block) => {
if (block.kind === "text") return block.text;
if (block.kind === "chart") return `[chart: ${block.name}]`;
if (block.kind === "image") return `[image: ${block.name}]`;
if (block.kind === "file") {
return `[file: ${block.name} (${block.mimeType}, ${block.size} bytes)]`;
}
// Media keeps the artifact id so a later turn can edit or export what it refers to.
if (block.kind === "video" || block.kind === "audio") {
if (block.kind === "image" || block.kind === "video" || block.kind === "audio") {
return `[${block.kind}: ${block.name} (artifact ${block.artifactId})]`;
}
if (block.kind === "media") {

View File

@ -1739,7 +1739,7 @@ describe("claimSteering", () => {
{
id: "steer-1",
messageId: "message-1",
text: "First\n[image: chart.png]",
text: "First\n[image: chart.png (artifact artifact-1)]",
blocks: [
{ kind: "text", text: "First" },
{

View File

@ -148,6 +148,15 @@ describeRoutes("artifact HTTP routes", () => {
(await upload(app, intruder, { groupId: group.id }, "x.mp4", "video/mp4", video)).status,
).toBe(404);
// A cross-site page cannot upload with the owner's cookie.
expect(
(
await upload(app, owner, { botId: bot.id }, "x.txt", "text/plain", video, {
origin: "https://evil.example",
})
).status,
).toBe(403);
// Signed out.
expect((await app.request(`/api/artifacts/${botArtifact.id}`)).status).toBe(401);
expect(
@ -283,6 +292,19 @@ describeRoutes("artifact HTTP routes", () => {
const me = await rpc<{ spaceId: string }>(app, owner, "me");
const spaceDir = path.join(dataDir, "artifacts", me.spaceId);
expect(readdirSafe(spaceDir)).toEqual([]);
// Video far above the inline cap uploads, streams by range, and is refused as base64.
const large = Buffer.alloc(26 * 1024 * 1024, 1);
large.write("head", 0);
const uploaded = await upload(app, owner, { botId: bot.id }, "long.mp4", "video/mp4", large);
expect(uploaded.status).toBe(201);
const largeVideo = (await uploaded.json()) as { id: string };
const head = await get(app, owner, largeVideo.id, { range: "bytes=0-3" });
expect(head.status).toBe(206);
expect(await head.text()).toBe("head");
await expect(
rpc(app, owner, "artifacts/get", { botId: bot.id, artifactId: largeVideo.id }),
).rejects.toThrow(/400: .*too large to load inline/);
});
});