Merge the media pipeline: inline players, streamed uploads, attached media
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
commit
99c8ac14b5
@ -609,6 +609,7 @@ export async function createApp(
|
||||
artifacts,
|
||||
sessionUserId: async (request) =>
|
||||
(await auth.api.getSession({ headers: sessionHeaders(request) }))?.user?.id ?? null,
|
||||
trustedOrigin: (origin) => isTrustedOrigin(origin, env),
|
||||
});
|
||||
mountWebhookHttpRoutes(app, { prisma, secrets, events, jobs });
|
||||
// Shared with stop so a shutdown during retry delays does not restart polling.
|
||||
|
||||
@ -77,6 +77,7 @@ describe("artifact HTTP routes", () => {
|
||||
prisma: prisma as unknown as PrismaClient,
|
||||
artifacts: artifacts as unknown as ArtifactStore,
|
||||
sessionUserId,
|
||||
trustedOrigin: (origin) => origin === "http://127.0.0.1:5173",
|
||||
});
|
||||
return { hono, prisma, artifacts };
|
||||
}
|
||||
@ -95,6 +96,19 @@ describe("artifact HTTP routes", () => {
|
||||
expect(artifacts.putStream).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses uploads from untrusted origins before reading the session", async () => {
|
||||
const sessionUserId = vi.fn(async () => "user-1");
|
||||
const { hono, artifacts } = app(sessionUserId);
|
||||
const response = await hono.request("/api/artifacts?botId=bot-1&name=a.txt", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "text/plain", origin: "https://evil.example" },
|
||||
body: "data",
|
||||
});
|
||||
expect(response.status).toBe(403);
|
||||
expect(sessionUserId).not.toHaveBeenCalled();
|
||||
expect(artifacts.putStream).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("answers ids that are not artifact ids with 404 without a lookup", async () => {
|
||||
const { hono, prisma } = app(async () => "user-1");
|
||||
const response = await hono.request(`/api/artifacts/${encodeURIComponent("../etc/passwd")}`);
|
||||
|
||||
@ -20,6 +20,8 @@ export type ArtifactHttpDeps = {
|
||||
artifacts: ArtifactStore;
|
||||
/** The signed-in user (cookie or bearer session), or null. */
|
||||
sessionUserId: (request: Request) => Promise<string | null>;
|
||||
/** Same rule as CORS. Raw uploads can be CORS "simple" requests, so the origin is checked. */
|
||||
trustedOrigin: (origin: string) => boolean;
|
||||
};
|
||||
|
||||
/** Generated ids only; anything else is not an artifact. */
|
||||
@ -208,10 +210,12 @@ export function mountArtifactHttpRoutes(app: Hono, deps: ArtifactHttpDeps) {
|
||||
|
||||
app.post(ARTIFACT_HTTP_PATH, async (c) => {
|
||||
const request = c.req.raw;
|
||||
const reject = (status: 400 | 401 | 404 | 413 | 415, error: string) => {
|
||||
const reject = (status: 400 | 401 | 403 | 404 | 413 | 415, error: string) => {
|
||||
cancelBody(request.body);
|
||||
return c.json({ error }, status);
|
||||
};
|
||||
const origin = c.req.header("origin");
|
||||
if (origin && !deps.trustedOrigin(origin)) return reject(403, "Forbidden");
|
||||
const userId = await deps.sessionUserId(request);
|
||||
if (!userId) return reject(401, "Unauthorized");
|
||||
const actor = await requireMembership(
|
||||
|
||||
@ -11,6 +11,12 @@ import {
|
||||
} from "@rakazo/core";
|
||||
import { IsolationError, type PrismaClient } from "@rakazo/db";
|
||||
|
||||
/**
|
||||
* Base64 reads load the whole file into memory; larger media is served by
|
||||
* GET /api/artifacts/:id instead.
|
||||
*/
|
||||
export const ARTIFACT_INLINE_READ_MAX_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
function adapterContext(actor: Actor, botId: string, operationId: string) {
|
||||
return {
|
||||
operationId,
|
||||
@ -131,6 +137,9 @@ async function readArtifact(
|
||||
},
|
||||
contextBotId: string,
|
||||
) {
|
||||
if (row.size > ARTIFACT_INLINE_READ_MAX_BYTES) {
|
||||
throw new AttachmentValidationError("This file is too large to load inline.");
|
||||
}
|
||||
const bytes = await artifacts.get(
|
||||
row.storageKey,
|
||||
adapterContext(actor, contextBotId, `artifact-get:${row.id}`),
|
||||
|
||||
@ -4670,24 +4670,26 @@ export function createRouter(deps: RouterDeps) {
|
||||
}
|
||||
}),
|
||||
get: authed.artifacts.get.handler(async ({ context, input }) => {
|
||||
if (input.groupId) {
|
||||
const group = await groupRepos.getGroupTarget(context.actor, input.groupId);
|
||||
const contextBotId = group.members[0]?.bot.id;
|
||||
if (!contextBotId) throw new IsolationError();
|
||||
return getSpaceArtifact(deps, context.actor, {
|
||||
artifactId: input.artifactId,
|
||||
groupId: input.groupId,
|
||||
contextBotId,
|
||||
});
|
||||
}
|
||||
await repos.getBot(context.actor, input.botId!);
|
||||
try {
|
||||
if (input.groupId) {
|
||||
const group = await groupRepos.getGroupTarget(context.actor, input.groupId);
|
||||
const contextBotId = group.members[0]?.bot.id;
|
||||
if (!contextBotId) throw new IsolationError();
|
||||
return await getSpaceArtifact(deps, context.actor, {
|
||||
artifactId: input.artifactId,
|
||||
groupId: input.groupId,
|
||||
contextBotId,
|
||||
});
|
||||
}
|
||||
await repos.getBot(context.actor, input.botId!);
|
||||
return await getOwnedArtifact(deps, context.actor, {
|
||||
botId: input.botId!,
|
||||
artifactId: input.artifactId,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof IsolationError) throw error;
|
||||
if (error instanceof AttachmentValidationError) {
|
||||
throw new ORPCError("BAD_REQUEST", { message: error.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}),
|
||||
|
||||
@ -77,6 +77,7 @@ import {
|
||||
type MobileMessagePage,
|
||||
type MobileSnapshot,
|
||||
mergeMobileSnapshot,
|
||||
messageAttachments,
|
||||
messagingProviderLabel,
|
||||
prependMobileMessagePage,
|
||||
rpc,
|
||||
@ -2243,7 +2244,11 @@ function previewMessageText(message: MobileMessage): string {
|
||||
.join(" ")
|
||||
.trim();
|
||||
if (text) return text;
|
||||
if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) {
|
||||
if (
|
||||
message.blocks.some((block) =>
|
||||
["image", "file", "video", "audio", "media"].includes(block.kind),
|
||||
)
|
||||
) {
|
||||
return t("Attachment");
|
||||
}
|
||||
return t("Message");
|
||||
@ -2676,9 +2681,7 @@ const MessageBubble = memo(function MessageBubble({
|
||||
</View>
|
||||
);
|
||||
}
|
||||
const attachments = message.blocks.filter(
|
||||
(block) => block.kind === "image" || block.kind === "file",
|
||||
);
|
||||
const attachments = messageAttachments(message.blocks);
|
||||
const caption = message.blocks
|
||||
.flatMap((block) => {
|
||||
if (block.kind === "channel_message" && block.text) {
|
||||
@ -2686,6 +2689,7 @@ const MessageBubble = memo(function MessageBubble({
|
||||
`${messagingProviderLabel(block.provider, block.transport)} · ${block.fromLabel}: ${block.text}`,
|
||||
];
|
||||
}
|
||||
if (block.kind === "media") return [block.title];
|
||||
return block.kind === "text" && block.text ? [block.text] : [];
|
||||
})
|
||||
.join("\n");
|
||||
@ -2797,7 +2801,8 @@ const MessageBubble = memo(function MessageBubble({
|
||||
fontSize: 15,
|
||||
}}
|
||||
>
|
||||
📎 {attachment.name ?? t("File")}
|
||||
{attachment.kind === "video" ? "🎬" : attachment.kind === "audio" ? "🎵" : "📎"}{" "}
|
||||
{attachment.name ?? t("File")}
|
||||
</Text>
|
||||
{attachment.size ? (
|
||||
<Text
|
||||
|
||||
@ -16,6 +16,7 @@ import {
|
||||
MAX_MOBILE_AUTH_RESPONSE_BYTES,
|
||||
MAX_MOBILE_RPC_RESPONSE_BYTES,
|
||||
mergeMobileSnapshot,
|
||||
messageAttachments,
|
||||
passwordResetCapabilities,
|
||||
prependMobileMessagePage,
|
||||
requestPasswordReset,
|
||||
@ -1857,6 +1858,51 @@ describe("mobile thread event reduction", () => {
|
||||
).toBe("Slack · Alex: Hello from the group");
|
||||
});
|
||||
|
||||
it("falls back to plain text and file rows for video, audio and media cards", () => {
|
||||
const blocks = [
|
||||
{
|
||||
kind: "video" as const,
|
||||
artifactId: "vid-1",
|
||||
mimeType: "video/mp4",
|
||||
name: "clip.mp4",
|
||||
size: 10,
|
||||
width: 1080,
|
||||
height: 1920,
|
||||
},
|
||||
{
|
||||
kind: "audio" as const,
|
||||
artifactId: "aud-1",
|
||||
mimeType: "audio/mpeg",
|
||||
name: "memo.mp3",
|
||||
size: 5,
|
||||
},
|
||||
{
|
||||
kind: "media" as const,
|
||||
jobId: "job-1",
|
||||
operation: "export_video" as const,
|
||||
status: "done" as const,
|
||||
title: "Vertical cut",
|
||||
outputs: [
|
||||
{
|
||||
artifactId: "out-1",
|
||||
kind: "video" as const,
|
||||
mimeType: "video/mp4",
|
||||
name: "cut.mp4",
|
||||
size: 7,
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
expect(blockText(mobileMessage("media-1", blocks))).toBe(
|
||||
"[video: clip.mp4]\n[audio: memo.mp3]\nVertical cut: done",
|
||||
);
|
||||
expect(messageAttachments(blocks).map((item) => [item.kind, item.artifactId])).toEqual([
|
||||
["video", "vid-1"],
|
||||
["audio", "aud-1"],
|
||||
["video", "out-1"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("deduplicates durable messages and replaces matching transient subagent state", () => {
|
||||
const initial = snapshot([
|
||||
mobileMessage("message-1", [{ kind: "text", text: "old" }]),
|
||||
|
||||
@ -860,6 +860,24 @@ export function copyableMobileMessageText(message: MobileMessage): string {
|
||||
.trim();
|
||||
}
|
||||
|
||||
export type MobileAttachment = {
|
||||
kind: "image" | "video" | "audio" | "file";
|
||||
artifactId: string;
|
||||
name: string;
|
||||
mimeType: string;
|
||||
size?: number;
|
||||
};
|
||||
|
||||
/** Files a message carries, including the outputs of a finished media card, as tappable rows. */
|
||||
export function messageAttachments(blocks: readonly MessageBlock[]): MobileAttachment[] {
|
||||
return blocks.flatMap((block): MobileAttachment[] => {
|
||||
if (block.kind === "image") return [{ ...block, kind: "image" }];
|
||||
if (block.kind === "video" || block.kind === "audio" || block.kind === "file") return [block];
|
||||
if (block.kind === "media") return block.outputs;
|
||||
return [];
|
||||
});
|
||||
}
|
||||
|
||||
export function blockText(message: MobileMessage) {
|
||||
return message.blocks
|
||||
.map((block) => {
|
||||
@ -879,6 +897,10 @@ export function blockText(message: MobileMessage) {
|
||||
if (block.kind === "file") {
|
||||
return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`;
|
||||
}
|
||||
if (block.kind === "video" || block.kind === "audio") {
|
||||
return `[${block.kind}: ${block.name ?? "attachment"}]`;
|
||||
}
|
||||
if (block.kind === "media") return `${block.title}: ${block.status}`;
|
||||
if (block.kind === "steps") {
|
||||
return (block.steps ?? [])
|
||||
.map((step) => `${step.label}${step.count > 1 ? ` ×${step.count}` : ""}`)
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { attachmentKindOf } from "@rakazo/contracts";
|
||||
import { File, Paths } from "expo-file-system";
|
||||
import * as Sharing from "expo-sharing";
|
||||
import { rpc } from "./api";
|
||||
import { captureApiRequestContext, rpc } from "./api";
|
||||
import { artifactCacheFileName } from "./artifact-file";
|
||||
import { t } from "./i18n";
|
||||
|
||||
@ -11,6 +12,16 @@ async function cacheMobileArtifact(
|
||||
artifactId: string,
|
||||
mimeType: string,
|
||||
): Promise<File> {
|
||||
const kind = attachmentKindOf(mimeType);
|
||||
if (kind === "video" || kind === "audio") {
|
||||
// Media can be hundreds of megabytes: stream it to disk instead of a base64 RPC payload.
|
||||
const { apiBase, headers } = await captureApiRequestContext();
|
||||
return File.downloadFileAsync(
|
||||
`${apiBase}/api/artifacts/${encodeURIComponent(artifactId)}`,
|
||||
new File(Paths.cache, artifactCacheFileName(artifactId, mimeType)),
|
||||
{ headers, idempotent: true },
|
||||
);
|
||||
}
|
||||
const artifact = await rpc<{ contentBase64: string }>("artifacts/get", {
|
||||
...target,
|
||||
artifactId,
|
||||
|
||||
BIN
apps/web/e2e/fixtures/media-clip.mp4
Normal file
BIN
apps/web/e2e/fixtures/media-clip.mp4
Normal file
Binary file not shown.
82
apps/web/e2e/media-pipeline.spec.ts
Normal file
82
apps/web/e2e/media-pipeline.spec.ts
Normal file
@ -0,0 +1,82 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { captureScreenshot, completeOnboarding, signup } from "./helpers";
|
||||
|
||||
test.describe.configure({ mode: "serial" });
|
||||
|
||||
// VP9 in MP4: Playwright's Chromium has no H.264 decoder, real browsers play both.
|
||||
const clip = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "media-clip.mp4");
|
||||
const pixel = Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
|
||||
"base64",
|
||||
);
|
||||
|
||||
test("an uploaded video streams from the artifact route and plays inline", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
const stamp = Date.now();
|
||||
await signup(page, `media-video-${stamp}@rakazo.test`, "password12", "Media Video");
|
||||
await completeOnboarding(page);
|
||||
|
||||
const upload = page.waitForResponse(
|
||||
(response) =>
|
||||
response.request().method() === "POST" && /\/api\/artifacts\?/.test(response.url()),
|
||||
);
|
||||
await page.locator('input[type="file"]').setInputFiles(clip);
|
||||
await expect(page.getByRole("button", { name: "Remove media-clip.mp4" })).toBeVisible();
|
||||
await page.getByRole("combobox", { name: /Message/ }).fill("here is the clip");
|
||||
await page.keyboard.press("Enter");
|
||||
expect((await upload).status()).toBe(201);
|
||||
|
||||
const video = page.getByTestId("media-video").locator("video");
|
||||
await expect(video).toBeVisible({ timeout: 30_000 });
|
||||
await expect(video).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/);
|
||||
await expect
|
||||
.poll(() => video.evaluate((element: HTMLVideoElement) => element.readyState), {
|
||||
timeout: 15_000,
|
||||
})
|
||||
.toBeGreaterThanOrEqual(1);
|
||||
expect(await video.evaluate((element: HTMLVideoElement) => element.duration)).toBeCloseTo(1, 0);
|
||||
|
||||
await video.evaluate(async (element: HTMLVideoElement) => {
|
||||
element.muted = true;
|
||||
await element.play();
|
||||
});
|
||||
await expect
|
||||
.poll(() => video.evaluate((element: HTMLVideoElement) => element.currentTime))
|
||||
.toBeGreaterThan(0);
|
||||
await video.evaluate((element: HTMLVideoElement) => {
|
||||
element.pause();
|
||||
element.currentTime = 0.5;
|
||||
element.scrollIntoView({ block: "center" });
|
||||
});
|
||||
await captureScreenshot(page, testInfo, "media-video-block");
|
||||
});
|
||||
|
||||
test("an image in the main conversation opens and downloads from the artifact route", async ({
|
||||
page,
|
||||
}) => {
|
||||
const stamp = Date.now();
|
||||
await signup(page, `media-image-${stamp}@rakazo.test`, "password12", "Media Image");
|
||||
// The main conversation is a group thread, so this covers group uploads and reads.
|
||||
await completeOnboarding(page, undefined, { mainConversation: true });
|
||||
|
||||
await page
|
||||
.locator('input[type="file"]')
|
||||
.setInputFiles({ name: "pixel.png", mimeType: "image/png", buffer: pixel });
|
||||
await page.getByRole("combobox", { name: /Message/ }).fill("a pixel");
|
||||
await page.keyboard.press("Enter");
|
||||
|
||||
const thumbnail = page.getByTestId("transcript").getByRole("img", { name: "pixel.png" });
|
||||
await expect(thumbnail).toBeVisible({ timeout: 30_000 });
|
||||
await expect(thumbnail).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/);
|
||||
await thumbnail.click();
|
||||
|
||||
const dialog = page.getByRole("dialog", { name: "pixel.png" });
|
||||
const download = dialog.getByRole("link", { name: "Download pixel.png" });
|
||||
await expect(download).toHaveAttribute("href", /\/api\/artifacts\/[^/?]+\?download=1$/);
|
||||
const saved = page.waitForEvent("download");
|
||||
await download.click();
|
||||
expect((await saved).suggestedFilename()).toBe("pixel.png");
|
||||
});
|
||||
@ -1,19 +1,12 @@
|
||||
import { i18n } from "@lingui/core";
|
||||
import { t } from "@lingui/core/macro";
|
||||
import { Trans, useLingui } from "@lingui/react/macro";
|
||||
import { ChatMarkdown } from "@rakazo/chat-ui/web";
|
||||
import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web";
|
||||
import { Download, FileText, X } from "lucide-react";
|
||||
import { type RefObject, useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
type ArtifactTarget,
|
||||
downloadArtifact,
|
||||
downloadArtifactBytes,
|
||||
fetchArtifactBytes,
|
||||
} from "../lib/artifact-open";
|
||||
import { downloadArtifact, fetchArtifactBytes } from "../lib/artifact-open";
|
||||
import { formatBytes } from "../lib/format-bytes";
|
||||
|
||||
type ArtifactFileCardProps = {
|
||||
target: ArtifactTarget;
|
||||
artifactId: string;
|
||||
name: string;
|
||||
mimeType: string;
|
||||
@ -26,23 +19,14 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
|
||||
const previewButton = useRef<HTMLButtonElement>(null);
|
||||
const closeButton = useRef<HTMLButtonElement>(null);
|
||||
const [previewOpen, setPreviewOpen] = useState(false);
|
||||
const [downloadError, setDownloadError] = useState<string | null>(null);
|
||||
|
||||
async function startDownload() {
|
||||
setDownloadError(null);
|
||||
try {
|
||||
await downloadArtifact(props.target, props.artifactId, props.name, props.mimeType);
|
||||
} catch {
|
||||
setDownloadError(t`Could not download ${props.name}. Try again.`);
|
||||
}
|
||||
}
|
||||
const startDownload = () => downloadArtifact(props.artifactId, props.name);
|
||||
|
||||
if (!markdown) {
|
||||
return (
|
||||
<div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void startDownload()}
|
||||
onClick={startDownload}
|
||||
className="rounded-2xl border border-border bg-card px-4 py-3 text-left text-ui-body text-foreground hover:bg-accent"
|
||||
>
|
||||
<div className="font-medium">{props.name}</div>
|
||||
@ -50,7 +34,6 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
|
||||
{props.mimeType} · {formatBytes(props.size)}
|
||||
</div>
|
||||
</button>
|
||||
{downloadError ? <DownloadError message={downloadError} /> : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -80,13 +63,12 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
|
||||
type="button"
|
||||
aria-label={t`Download ${props.name}`}
|
||||
title={t`Download ${props.name}`}
|
||||
onClick={() => void startDownload()}
|
||||
onClick={startDownload}
|
||||
className="grid w-14 shrink-0 place-items-center border-l border-border text-muted-foreground hover:bg-accent hover:text-foreground"
|
||||
>
|
||||
<Download strokeWidth={1.8} className="size-icon-lg" />
|
||||
</button>
|
||||
</div>
|
||||
{downloadError ? <DownloadError message={downloadError} /> : null}
|
||||
</div>
|
||||
<Dialog open={previewOpen} onOpenChange={setPreviewOpen}>
|
||||
<DialogContent
|
||||
@ -103,32 +85,25 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
|
||||
}
|
||||
|
||||
function MarkdownPreview({
|
||||
target,
|
||||
artifactId,
|
||||
name,
|
||||
mimeType,
|
||||
closeButtonRef,
|
||||
}: ArtifactFileCardProps & { closeButtonRef: RefObject<HTMLButtonElement | null> }) {
|
||||
const { t } = useLingui();
|
||||
const [downloadError, setDownloadError] = useState<string | null>(null);
|
||||
const [state, setState] = useState<
|
||||
| { status: "loading" }
|
||||
| { status: "ready"; bytes: Uint8Array; markdown: string }
|
||||
| { status: "ready"; markdown: string }
|
||||
| { status: "error"; message: string }
|
||||
>({ status: "loading" });
|
||||
const targetBotId = "botId" in target ? target.botId : undefined;
|
||||
const targetGroupId = "groupId" in target ? target.groupId : undefined;
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const artifactTarget: ArtifactTarget =
|
||||
targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! };
|
||||
void fetchArtifactBytes(artifactTarget, artifactId)
|
||||
void fetchArtifactBytes(artifactId)
|
||||
.then((bytes) => {
|
||||
if (cancelled) return;
|
||||
try {
|
||||
const markdown = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
||||
setState({ status: "ready", bytes, markdown });
|
||||
setState({ status: "ready", markdown });
|
||||
} catch {
|
||||
setState({ status: "error", message: t`This file is not valid UTF-8 Markdown.` });
|
||||
}
|
||||
@ -143,7 +118,7 @@ function MarkdownPreview({
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [artifactId, targetBotId, targetGroupId, t]);
|
||||
}, [artifactId, t]);
|
||||
|
||||
return (
|
||||
<>
|
||||
@ -157,17 +132,7 @@ function MarkdownPreview({
|
||||
className="rounded-full text-muted-foreground"
|
||||
aria-label={t`Download ${name}`}
|
||||
title={t`Download ${name}`}
|
||||
onClick={() =>
|
||||
void (async () => {
|
||||
setDownloadError(null);
|
||||
try {
|
||||
if (state.status === "ready") downloadArtifactBytes(name, mimeType, state.bytes);
|
||||
else await downloadArtifact(target, artifactId, name, mimeType);
|
||||
} catch {
|
||||
setDownloadError(t`Could not download ${name}. Try again.`);
|
||||
}
|
||||
})()
|
||||
}
|
||||
onClick={() => downloadArtifact(artifactId, name)}
|
||||
>
|
||||
<Download />
|
||||
</Button>
|
||||
@ -181,11 +146,6 @@ function MarkdownPreview({
|
||||
<X />
|
||||
</DialogClose>
|
||||
</header>
|
||||
{downloadError ? (
|
||||
<div className="shrink-0 px-5 pt-4">
|
||||
<DownloadError message={downloadError} />
|
||||
</div>
|
||||
) : null}
|
||||
<div className="min-h-0 flex-1 overflow-y-auto">
|
||||
<article className="mx-auto w-full max-w-[760px] px-8 py-10 text-ui-body leading-7 text-foreground sm:px-12 sm:py-12">
|
||||
{state.status === "loading" ? (
|
||||
@ -204,23 +164,3 @@ function MarkdownPreview({
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function DownloadError({ message }: { message: string }) {
|
||||
return (
|
||||
<div role="alert" className="mt-2 text-left text-ui-body-sm text-destructive">
|
||||
{message}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function formatBytes(size: number) {
|
||||
const locale = i18n.locale || "en";
|
||||
if (size < 1024) return t`${size} B`;
|
||||
const format = (value: number) =>
|
||||
new Intl.NumberFormat(locale, {
|
||||
minimumFractionDigits: 1,
|
||||
maximumFractionDigits: 1,
|
||||
}).format(value);
|
||||
if (size < 1024 * 1024) return t`${format(size / 1024)} KB`;
|
||||
return t`${format(size / (1024 * 1024))} MB`;
|
||||
}
|
||||
|
||||
99
apps/web/src/components/media/MediaPlayer.test.tsx
Normal file
99
apps/web/src/components/media/MediaPlayer.test.tsx
Normal file
@ -0,0 +1,99 @@
|
||||
// @vitest-environment jsdom
|
||||
|
||||
import { act } from "react";
|
||||
import { createRoot, type Root } from "react-dom/client";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@lingui/react/macro", () => {
|
||||
const t = (parts: TemplateStringsArray, ...values: unknown[]) =>
|
||||
parts.reduce(
|
||||
(text, part, index) => text + part + (index < values.length ? values[index] : ""),
|
||||
"",
|
||||
);
|
||||
return { useLingui: () => ({ t }) };
|
||||
});
|
||||
|
||||
import { MediaPlayer } from "./MediaPlayer";
|
||||
|
||||
let container: HTMLDivElement;
|
||||
let root: Root;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
|
||||
container = document.createElement("div");
|
||||
document.body.append(container);
|
||||
root = createRoot(container);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await act(async () => root.unmount());
|
||||
container.remove();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe("MediaPlayer", () => {
|
||||
it("loads nothing before it is near the viewport and fits vertical video to the column", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<MediaPlayer artifactId="art-1" kind="video" name="clip.mp4" width={1080} height={1920} />,
|
||||
);
|
||||
expect(html).toContain('aria-label="clip.mp4"');
|
||||
expect(html).not.toContain("src=");
|
||||
expect(html).toContain("width:270px");
|
||||
expect(html).toContain("aspect-ratio:270 / 480");
|
||||
});
|
||||
|
||||
it("streams video with its poster from the artifact route once visible", async () => {
|
||||
vi.stubGlobal(
|
||||
"IntersectionObserver",
|
||||
class {
|
||||
constructor(private readonly callback: IntersectionObserverCallback) {}
|
||||
observe() {
|
||||
this.callback([{ isIntersecting: true } as IntersectionObserverEntry], this as never);
|
||||
}
|
||||
disconnect() {}
|
||||
},
|
||||
);
|
||||
await act(async () =>
|
||||
root.render(
|
||||
<MediaPlayer artifactId="art-1" kind="video" name="clip.mp4" posterArtifactId="art-2" />,
|
||||
),
|
||||
);
|
||||
const video = container.querySelector("video")!;
|
||||
expect(video.getAttribute("src")).toBe("/api/artifacts/art-1");
|
||||
expect(video.getAttribute("poster")).toBe("/api/artifacts/art-2");
|
||||
expect(video.getAttribute("preload")).toBe("metadata");
|
||||
expect(video.hasAttribute("controls")).toBe(true);
|
||||
});
|
||||
|
||||
it("sizes the frame from the file when the block has no dimensions", async () => {
|
||||
await act(async () =>
|
||||
root.render(<MediaPlayer artifactId="art-4" kind="video" name="up.mp4" />),
|
||||
);
|
||||
const frame = container.querySelector<HTMLDivElement>('[data-testid="media-video"]')!;
|
||||
expect(frame.style.width).toBe("420px");
|
||||
const video = container.querySelector("video")!;
|
||||
Object.defineProperty(video, "videoWidth", { value: 1080 });
|
||||
Object.defineProperty(video, "videoHeight", { value: 1920 });
|
||||
await act(async () => {
|
||||
video.dispatchEvent(new Event("loadedmetadata"));
|
||||
});
|
||||
expect(frame.style.width).toBe("270px");
|
||||
});
|
||||
|
||||
it("falls back to a download link when the browser cannot play the file", async () => {
|
||||
await act(async () =>
|
||||
root.render(<MediaPlayer artifactId="art-3" kind="audio" name="memo.ogg" />),
|
||||
);
|
||||
const audio = container.querySelector("audio")!;
|
||||
expect(audio.getAttribute("src")).toBe("/api/artifacts/art-3");
|
||||
await act(async () => {
|
||||
audio.dispatchEvent(new Event("error"));
|
||||
});
|
||||
expect(container.querySelector("audio")).toBeNull();
|
||||
const link = container.querySelector("a")!;
|
||||
expect(link.getAttribute("href")).toBe("/api/artifacts/art-3?download=1");
|
||||
expect(link.getAttribute("download")).toBe("memo.ogg");
|
||||
expect(link.getAttribute("aria-label")).toBe("Download memo.ogg");
|
||||
});
|
||||
});
|
||||
129
apps/web/src/components/media/MediaPlayer.tsx
Normal file
129
apps/web/src/components/media/MediaPlayer.tsx
Normal file
@ -0,0 +1,129 @@
|
||||
import { useLingui } from "@lingui/react/macro";
|
||||
import { buttonVariants } from "@rakazo/ui-web";
|
||||
import { Download } from "lucide-react";
|
||||
import { type RefObject, useEffect, useRef, useState } from "react";
|
||||
import { artifactUrl } from "../../lib/artifact-url";
|
||||
|
||||
export type MediaPlayerProps = {
|
||||
artifactId: string;
|
||||
kind: "video" | "audio";
|
||||
name: string;
|
||||
posterArtifactId?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
};
|
||||
|
||||
const MAX_VIDEO_WIDTH = 420;
|
||||
const MAX_VIDEO_HEIGHT = 480;
|
||||
|
||||
/** Plays a video or audio artifact in place; loads nothing until it is near the viewport. */
|
||||
export function MediaPlayer({
|
||||
artifactId,
|
||||
kind,
|
||||
name,
|
||||
posterArtifactId,
|
||||
width,
|
||||
height,
|
||||
}: MediaPlayerProps) {
|
||||
const container = useRef<HTMLDivElement>(null);
|
||||
const near = useNearViewport(container);
|
||||
const [failed, setFailed] = useState(false);
|
||||
const [frame, setFrame] = useState<{ width: number; height: number } | null>(null);
|
||||
const src = near ? artifactUrl(artifactId) : undefined;
|
||||
|
||||
if (failed) return <MediaFallback artifactId={artifactId} name={name} />;
|
||||
|
||||
if (kind === "audio") {
|
||||
return (
|
||||
<div ref={container} className="w-72 max-w-full" data-testid="media-audio">
|
||||
{/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */}
|
||||
<audio
|
||||
src={src}
|
||||
controls
|
||||
preload="metadata"
|
||||
aria-label={name}
|
||||
className="w-full"
|
||||
onError={() => setFailed(true)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Uploads carry no dimensions; size the frame from the file once its metadata loads.
|
||||
const box = videoBox(width ?? frame?.width, height ?? frame?.height);
|
||||
return (
|
||||
<div
|
||||
ref={container}
|
||||
className="max-w-full overflow-hidden rounded-[20px] bg-muted"
|
||||
style={{ width: box.width, aspectRatio: `${box.width} / ${box.height}` }}
|
||||
data-testid="media-video"
|
||||
>
|
||||
{/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */}
|
||||
<video
|
||||
src={src}
|
||||
poster={near && posterArtifactId ? artifactUrl(posterArtifactId) : undefined}
|
||||
controls
|
||||
playsInline
|
||||
preload="metadata"
|
||||
aria-label={name}
|
||||
className="block size-full object-contain"
|
||||
onError={() => setFailed(true)}
|
||||
onLoadedMetadata={(event) => {
|
||||
const { videoWidth, videoHeight } = event.currentTarget;
|
||||
if (videoWidth > 0 && videoHeight > 0) {
|
||||
setFrame({ width: videoWidth, height: videoHeight });
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** Fits the frame into the chat column, defaulting to 16:9 until the size is known. */
|
||||
function videoBox(width?: number, height?: number) {
|
||||
const w = width && width > 0 ? width : 16;
|
||||
const h = height && height > 0 ? height : 9;
|
||||
const scale = Math.min(MAX_VIDEO_WIDTH / w, MAX_VIDEO_HEIGHT / h);
|
||||
return { width: Math.round(w * scale), height: Math.round(h * scale) };
|
||||
}
|
||||
|
||||
function MediaFallback({ artifactId, name }: { artifactId: string; name: string }) {
|
||||
const { t } = useLingui();
|
||||
return (
|
||||
<div className="flex max-w-full items-center gap-2 rounded-[20px] border border-border bg-muted py-2 ps-4 pe-2 text-ui-body text-muted-foreground">
|
||||
<span className="min-w-0 truncate">{name}</span>
|
||||
<a
|
||||
href={artifactUrl(artifactId, { download: true })}
|
||||
download={name}
|
||||
aria-label={t`Download ${name}`}
|
||||
title={t`Download ${name}`}
|
||||
className={buttonVariants({ variant: "ghost", size: "icon-sm" })}
|
||||
>
|
||||
<Download />
|
||||
</a>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function useNearViewport(ref: RefObject<HTMLElement | null>) {
|
||||
const [near, setNear] = useState(false);
|
||||
useEffect(() => {
|
||||
const element = ref.current;
|
||||
if (!element || typeof IntersectionObserver === "undefined") {
|
||||
setNear(true);
|
||||
return;
|
||||
}
|
||||
const observer = new IntersectionObserver(
|
||||
(entries) => {
|
||||
if (entries.some((entry) => entry.isIntersecting)) {
|
||||
setNear(true);
|
||||
observer.disconnect();
|
||||
}
|
||||
},
|
||||
{ rootMargin: "320px" },
|
||||
);
|
||||
observer.observe(element);
|
||||
return () => observer.disconnect();
|
||||
}, [ref]);
|
||||
return near;
|
||||
}
|
||||
@ -1,40 +1,29 @@
|
||||
import { rpc } from "./rpc";
|
||||
import { t } from "@lingui/core/macro";
|
||||
import { artifactUrl } from "./artifact-url";
|
||||
|
||||
export type ArtifactTarget = { botId: string } | { groupId: string };
|
||||
|
||||
export function decodeArtifactBase64(contentBase64: string): Uint8Array {
|
||||
const binary = atob(contentBase64);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let index = 0; index < binary.length; index += 1) {
|
||||
bytes[index] = binary.charCodeAt(index);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
export async function fetchArtifactBytes(
|
||||
target: ArtifactTarget,
|
||||
artifactId: string,
|
||||
): Promise<Uint8Array> {
|
||||
const artifact = await rpc.artifacts.get({ ...target, artifactId });
|
||||
return decodeArtifactBase64(artifact.contentBase64);
|
||||
export async function fetchArtifactBytes(artifactId: string): Promise<Uint8Array> {
|
||||
const response = await fetch(artifactUrl(artifactId), { credentials: "include" });
|
||||
if (!response.ok) throw new Error(t`Could not load this file.`);
|
||||
return new Uint8Array(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
export function downloadArtifactBytes(name: string, mimeType: string, bytes: Uint8Array): void {
|
||||
const blob = new Blob([new Uint8Array(bytes)], { type: mimeType });
|
||||
const url = URL.createObjectURL(blob);
|
||||
saveUrl(url, name);
|
||||
window.setTimeout(() => URL.revokeObjectURL(url), 60_000);
|
||||
}
|
||||
|
||||
/** Lets the browser stream the download from the artifact route instead of buffering it. */
|
||||
export function downloadArtifact(artifactId: string, name: string): void {
|
||||
saveUrl(artifactUrl(artifactId, { download: true }), name);
|
||||
}
|
||||
|
||||
function saveUrl(url: string, name: string) {
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = name;
|
||||
anchor.click();
|
||||
window.setTimeout(() => URL.revokeObjectURL(url), 60_000);
|
||||
}
|
||||
|
||||
export async function downloadArtifact(
|
||||
target: ArtifactTarget,
|
||||
artifactId: string,
|
||||
name: string,
|
||||
mimeType: string,
|
||||
): Promise<void> {
|
||||
const bytes = await fetchArtifactBytes(target, artifactId);
|
||||
downloadArtifactBytes(name, mimeType, bytes);
|
||||
}
|
||||
|
||||
83
apps/web/src/lib/artifact-upload.test.ts
Normal file
83
apps/web/src/lib/artifact-upload.test.ts
Normal file
@ -0,0 +1,83 @@
|
||||
import { ATTACHMENT_MAX_BYTES } from "@rakazo/contracts";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@lingui/core/macro", () => ({
|
||||
t: (parts: TemplateStringsArray, ...values: unknown[]) =>
|
||||
parts.reduce(
|
||||
(text, part, index) => text + part + (index < values.length ? values[index] : ""),
|
||||
"",
|
||||
),
|
||||
}));
|
||||
const create = vi.hoisted(() => vi.fn());
|
||||
vi.mock("./rpc", () => ({
|
||||
rpc: { artifacts: { create } },
|
||||
withSpaceHeaders: (init: HeadersInit) => new Headers(init),
|
||||
}));
|
||||
|
||||
import {
|
||||
attachmentMaxBytes,
|
||||
uploadArtifactFile,
|
||||
uploadAttachment,
|
||||
usesStreamingUpload,
|
||||
} from "./artifact-upload";
|
||||
|
||||
const artifact = {
|
||||
id: "art-1",
|
||||
botId: "bot-1",
|
||||
groupId: null,
|
||||
runId: null,
|
||||
name: "clip.mp4",
|
||||
mimeType: "video/mp4",
|
||||
size: 4,
|
||||
createdAt: "2026-10-03T12:00:00.000Z",
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
create.mockReset();
|
||||
});
|
||||
|
||||
describe("artifact uploads", () => {
|
||||
it("streams media and large files and keeps small files on the inline RPC", () => {
|
||||
expect(usesStreamingUpload(10, "video/mp4")).toBe(true);
|
||||
expect(usesStreamingUpload(10, "audio/mpeg")).toBe(true);
|
||||
expect(usesStreamingUpload(10, "image/png")).toBe(false);
|
||||
expect(usesStreamingUpload(ATTACHMENT_MAX_BYTES + 1, "image/png")).toBe(true);
|
||||
expect(attachmentMaxBytes("video/mp4")).toBe(500 * 1024 * 1024);
|
||||
expect(attachmentMaxBytes("text/plain")).toBe(25 * 1024 * 1024);
|
||||
});
|
||||
|
||||
it("posts the raw file with its target, name and type", async () => {
|
||||
const fetch = vi.fn(async () => Response.json(artifact, { status: 201 }));
|
||||
vi.stubGlobal("fetch", fetch);
|
||||
const file = new File(["data"], "clip.mp4", { type: "video/mp4" });
|
||||
|
||||
await expect(uploadAttachment({ groupId: "group-1" }, file, "video/mp4")).resolves.toEqual(
|
||||
artifact,
|
||||
);
|
||||
const [url, init] = fetch.mock.calls[0] as unknown as [string, RequestInit];
|
||||
expect(url).toBe("/api/artifacts?groupId=group-1&name=clip.mp4");
|
||||
expect(init.method).toBe("POST");
|
||||
expect(init.credentials).toBe("include");
|
||||
expect(new Headers(init.headers).get("content-type")).toBe("video/mp4");
|
||||
expect(init.body).toBe(file);
|
||||
expect(create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("surfaces the server's reason, or a generic one", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => Response.json({ error: "File is too large." }, { status: 413 })),
|
||||
);
|
||||
await expect(
|
||||
uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"),
|
||||
).rejects.toThrow("File is too large.");
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response("bad gateway", { status: 502 })),
|
||||
);
|
||||
await expect(
|
||||
uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"),
|
||||
).rejects.toThrow("Could not upload big.mp4. Try again.");
|
||||
});
|
||||
});
|
||||
68
apps/web/src/lib/artifact-upload.ts
Normal file
68
apps/web/src/lib/artifact-upload.ts
Normal file
@ -0,0 +1,68 @@
|
||||
import { t } from "@lingui/core/macro";
|
||||
import {
|
||||
ARTIFACT_MAX_BYTES_BY_KIND,
|
||||
ArtifactSchema,
|
||||
ATTACHMENT_MAX_BYTES,
|
||||
attachmentKindOf,
|
||||
} from "@rakazo/contracts";
|
||||
import type { ArtifactTarget } from "./artifact-open";
|
||||
import { rpc, withSpaceHeaders } from "./rpc";
|
||||
|
||||
type UploadedArtifact = ReturnType<typeof ArtifactSchema.parse>;
|
||||
|
||||
/** Largest file of this type the composer accepts. */
|
||||
export function attachmentMaxBytes(mimeType: string): number {
|
||||
return ARTIFACT_MAX_BYTES_BY_KIND[attachmentKindOf(mimeType)];
|
||||
}
|
||||
|
||||
/** Media and anything above the inline RPC cap upload as a raw stream instead of base64. */
|
||||
export function usesStreamingUpload(size: number, mimeType: string): boolean {
|
||||
const kind = attachmentKindOf(mimeType);
|
||||
return size > ATTACHMENT_MAX_BYTES || kind === "video" || kind === "audio";
|
||||
}
|
||||
|
||||
export async function uploadAttachment(
|
||||
target: ArtifactTarget,
|
||||
file: File,
|
||||
mimeType: string,
|
||||
): Promise<UploadedArtifact> {
|
||||
if (usesStreamingUpload(file.size, mimeType)) {
|
||||
return uploadArtifactFile(target, file, file.name, mimeType);
|
||||
}
|
||||
const contentBase64 = await readFileAsBase64(file);
|
||||
return rpc.artifacts.create({ ...target, name: file.name, mimeType, contentBase64 });
|
||||
}
|
||||
|
||||
/** Uploads a file as a raw streamed body to the artifact route. */
|
||||
export async function uploadArtifactFile(
|
||||
target: ArtifactTarget,
|
||||
file: Blob,
|
||||
name: string,
|
||||
mimeType: string,
|
||||
): Promise<UploadedArtifact> {
|
||||
const query = new URLSearchParams({ ...target, name });
|
||||
const response = await fetch(`/api/artifacts?${query}`, {
|
||||
method: "POST",
|
||||
headers: withSpaceHeaders({ "content-type": mimeType }),
|
||||
credentials: "include",
|
||||
body: file,
|
||||
});
|
||||
const body: unknown = await response.json().catch(() => null);
|
||||
const artifact = ArtifactSchema.safeParse(body);
|
||||
if (response.ok && artifact.success) return artifact.data;
|
||||
const error = (body as { error?: unknown } | null)?.error;
|
||||
throw new Error(typeof error === "string" ? error : t`Could not upload ${name}. Try again.`);
|
||||
}
|
||||
|
||||
function readFileAsBase64(file: File): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
const result = typeof reader.result === "string" ? reader.result : "";
|
||||
const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result;
|
||||
resolve(base64);
|
||||
};
|
||||
reader.onerror = () => reject(reader.error ?? new Error("Failed to read file"));
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
@ -1,38 +1,8 @@
|
||||
import { t } from "@lingui/core/macro";
|
||||
import { ArtifactSchema } from "@rakazo/contracts";
|
||||
import type { ArtifactTarget } from "./artifact-open";
|
||||
import { withSpaceHeaders } from "./rpc";
|
||||
|
||||
const ARTIFACT_PATH = "/api/artifacts";
|
||||
|
||||
/**
|
||||
* Same-origin URL that streams an artifact with Range support, for `<video>`, `<audio>`, `<img>`
|
||||
* and downloads. The session cookie authorizes it; `download` forces a save instead of inline.
|
||||
*/
|
||||
export function artifactUrl(artifactId: string, options: { download?: boolean } = {}): string {
|
||||
const path = `${ARTIFACT_PATH}/${encodeURIComponent(artifactId)}`;
|
||||
const path = `/api/artifacts/${encodeURIComponent(artifactId)}`;
|
||||
return options.download ? `${path}?download=1` : path;
|
||||
}
|
||||
|
||||
/** Uploads a file as a raw streamed body, for media and files above the inline RPC cap. */
|
||||
export async function uploadArtifactFile(
|
||||
target: ArtifactTarget,
|
||||
file: Blob,
|
||||
name: string,
|
||||
mimeType: string,
|
||||
): Promise<ReturnType<typeof ArtifactSchema.parse>> {
|
||||
const query = new URLSearchParams(
|
||||
"botId" in target ? { botId: target.botId, name } : { groupId: target.groupId, name },
|
||||
);
|
||||
const response = await fetch(`${ARTIFACT_PATH}?${query}`, {
|
||||
method: "POST",
|
||||
headers: withSpaceHeaders({ "content-type": mimeType }),
|
||||
credentials: "include",
|
||||
body: file,
|
||||
});
|
||||
const body: unknown = await response.json().catch(() => null);
|
||||
const artifact = ArtifactSchema.safeParse(body);
|
||||
if (response.ok && artifact.success) return artifact.data;
|
||||
const error = (body as { error?: unknown } | null)?.error;
|
||||
throw new Error(typeof error === "string" ? error : t`Could not upload ${name}. Try again.`);
|
||||
}
|
||||
|
||||
12
apps/web/src/lib/format-bytes.ts
Normal file
12
apps/web/src/lib/format-bytes.ts
Normal file
@ -0,0 +1,12 @@
|
||||
import { i18n } from "@lingui/core";
|
||||
import { t } from "@lingui/core/macro";
|
||||
|
||||
/** Human file size in the UI locale: B, KB or MB with at most one decimal. */
|
||||
export function formatBytes(size: number): string {
|
||||
const locale = i18n.locale || "en";
|
||||
if (size < 1024) return t`${size} B`;
|
||||
const format = (value: number) =>
|
||||
new Intl.NumberFormat(locale, { maximumFractionDigits: 1 }).format(value);
|
||||
if (size < 1024 * 1024) return t`${format(size / 1024)} KB`;
|
||||
return t`${format(size / (1024 * 1024))} MB`;
|
||||
}
|
||||
@ -41,7 +41,6 @@ import type {
|
||||
} from "@rakazo/contracts";
|
||||
import {
|
||||
ATTACHMENT_ALLOWED_MIME_TYPES,
|
||||
ATTACHMENT_MAX_BYTES,
|
||||
ATTACHMENT_MAX_COUNT,
|
||||
canReactToThreadMessage,
|
||||
MESSAGE_REACTIONS,
|
||||
@ -169,6 +168,7 @@ import { ComputerUpdateProgress } from "../components/ComputerUpdateProgress";
|
||||
import { DataTrustPanel } from "../components/DataTrustPanel";
|
||||
import { MainInbox } from "../components/MainInbox";
|
||||
import { MessageHoverMetadata } from "../components/MessageHoverMetadata";
|
||||
import { MediaPlayer } from "../components/media/MediaPlayer";
|
||||
import { OoGroupStage, OoStage } from "../components/stage/OoStage";
|
||||
import { TradingModeIndicator } from "../components/TradingModeIndicator";
|
||||
import { SkillDraftCard } from "../components/teach/SkillDraftCard";
|
||||
@ -180,6 +180,7 @@ import { useSidebarResize } from "../hooks/useSidebarResize";
|
||||
import { readActivityMode, writeActivityMode } from "../lib/activity-mode";
|
||||
import { needsYouChipLabel, rosterActivitySubtitle, rosterStatusDot } from "../lib/agent-activity";
|
||||
import type { ArtifactTarget } from "../lib/artifact-open";
|
||||
import { attachmentMaxBytes, uploadAttachment } from "../lib/artifact-upload";
|
||||
import { authClient } from "../lib/auth";
|
||||
import { takeInitialBootstrap } from "../lib/bootstrap";
|
||||
import {
|
||||
@ -197,6 +198,7 @@ import { browserPreviewActivity } from "../lib/browser-preview-context";
|
||||
import { loadComputerScreen } from "../lib/computer-screen";
|
||||
import { desktopBridge } from "../lib/desktop";
|
||||
import { scheduleFocusPrompt } from "../lib/focus-prompt";
|
||||
import { formatBytes } from "../lib/format-bytes";
|
||||
import {
|
||||
isWorkingRunStatus,
|
||||
resolveActiveGroupSpeaker,
|
||||
@ -2711,15 +2713,16 @@ export function ShellPage() {
|
||||
skipped.push(t`${file.name} (max ${ATTACHMENT_MAX_COUNT} attachments)`);
|
||||
continue;
|
||||
}
|
||||
if (file.size > ATTACHMENT_MAX_BYTES) {
|
||||
skipped.push(t`${file.name} (over 10 MiB)`);
|
||||
continue;
|
||||
}
|
||||
const mimeType = inferAttachmentMimeType(file.name, file.type);
|
||||
if (!mimeType) {
|
||||
skipped.push(file.name);
|
||||
continue;
|
||||
}
|
||||
const maxBytes = attachmentMaxBytes(mimeType);
|
||||
if (file.size > maxBytes) {
|
||||
skipped.push(t`${file.name} (over ${formatBytes(maxBytes)})`);
|
||||
continue;
|
||||
}
|
||||
next.push({
|
||||
id: `${file.name}-${file.size}-${file.lastModified}-${next.length}`,
|
||||
threadKey,
|
||||
@ -2859,11 +2862,10 @@ export function ShellPage() {
|
||||
if (!mimeType) {
|
||||
throw new Error(t`Unsupported file type: ${pending.file.name}`);
|
||||
}
|
||||
const contentBase64 = await readFileAsBase64(pending.file);
|
||||
const artifact = await rpc.artifacts.create(
|
||||
groupTarget
|
||||
? { groupId: groupTarget, name: pending.file.name, mimeType, contentBase64 }
|
||||
: { botId: botTarget!, name: pending.file.name, mimeType, contentBase64 },
|
||||
const artifact = await uploadAttachment(
|
||||
groupTarget ? { groupId: groupTarget } : { botId: botTarget! },
|
||||
pending.file,
|
||||
mimeType,
|
||||
);
|
||||
artifactIds.push(artifact.id);
|
||||
}
|
||||
@ -7186,6 +7188,8 @@ function MentionChipIcon({ mention }: { mention: ComposerMention }) {
|
||||
);
|
||||
}
|
||||
|
||||
const ATTACHMENT_BLOCK_KINDS = new Set(["image", "file", "video", "audio", "media"]);
|
||||
|
||||
function previewMessageText(message: ThreadMessage): string {
|
||||
const text = message.blocks
|
||||
.map((block) => (block.kind === "text" || block.kind === "channel_message" ? block.text : ""))
|
||||
@ -7193,7 +7197,7 @@ function previewMessageText(message: ThreadMessage): string {
|
||||
.join(" ")
|
||||
.trim();
|
||||
if (text) return text;
|
||||
if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) {
|
||||
if (message.blocks.some((block) => ATTACHMENT_BLOCK_KINDS.has(block.kind))) {
|
||||
return t`Attachment`;
|
||||
}
|
||||
return t`Message`;
|
||||
@ -7841,10 +7845,23 @@ const MessageView = memo(function MessageView({
|
||||
key={i}
|
||||
className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`}
|
||||
>
|
||||
<ArtifactImage
|
||||
target={artifactTarget}
|
||||
<ArtifactImage artifactId={block.artifactId} name={block.name} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (block.kind === "video" || block.kind === "audio") {
|
||||
return (
|
||||
<div
|
||||
key={i}
|
||||
className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`}
|
||||
>
|
||||
<MediaPlayer
|
||||
artifactId={block.artifactId}
|
||||
kind={block.kind}
|
||||
name={block.name}
|
||||
posterArtifactId={block.kind === "video" ? block.posterArtifactId : undefined}
|
||||
width={block.kind === "video" ? block.width : undefined}
|
||||
height={block.kind === "video" ? block.height : undefined}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
@ -7856,7 +7873,6 @@ const MessageView = memo(function MessageView({
|
||||
className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`}
|
||||
>
|
||||
<ArtifactFileCard
|
||||
target={artifactTarget}
|
||||
artifactId={block.artifactId}
|
||||
name={block.name}
|
||||
mimeType={block.mimeType}
|
||||
@ -8059,16 +8075,3 @@ function newClientNonce(): string {
|
||||
}
|
||||
return `m-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
}
|
||||
|
||||
function readFileAsBase64(file: File): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
const result = typeof reader.result === "string" ? reader.result : "";
|
||||
const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result;
|
||||
resolve(base64);
|
||||
};
|
||||
reader.onerror = () => reject(reader.error ?? new Error("Failed to read file"));
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
@ -1,11 +1,18 @@
|
||||
import { Trans, useLingui } from "@lingui/react/macro";
|
||||
import type { MessageBlock } from "@rakazo/contracts";
|
||||
import { abortableDelay } from "@rakazo/core";
|
||||
import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web";
|
||||
import { X } from "lucide-react";
|
||||
import {
|
||||
Button,
|
||||
buttonVariants,
|
||||
Dialog,
|
||||
DialogClose,
|
||||
DialogContent,
|
||||
DialogTitle,
|
||||
} from "@rakazo/ui-web";
|
||||
import { Download, X } from "lucide-react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { BuiCard, SuccessPop } from "../../components/ai/primitives";
|
||||
import { type ArtifactTarget, decodeArtifactBase64 } from "../../lib/artifact-open";
|
||||
import { artifactUrl } from "../../lib/artifact-url";
|
||||
import { chartViewport } from "../../lib/chart-viewport";
|
||||
import { connectMcpOauth } from "../../lib/mcp-connect";
|
||||
import { rpc } from "../../lib/rpc";
|
||||
@ -461,104 +468,65 @@ export function ChartBlockView({
|
||||
);
|
||||
}
|
||||
|
||||
export function ArtifactImage({
|
||||
target,
|
||||
artifactId,
|
||||
name,
|
||||
}: {
|
||||
target: ArtifactTarget;
|
||||
artifactId: string;
|
||||
name: string;
|
||||
}) {
|
||||
export function ArtifactImage({ artifactId, name }: { artifactId: string; name: string }) {
|
||||
const { t } = useLingui();
|
||||
const [src, setSrc] = useState<string | null>(null);
|
||||
const [open, setOpen] = useState(false);
|
||||
const [visible, setVisible] = useState(false);
|
||||
const container = useRef<HTMLDivElement>(null);
|
||||
const targetBotId = "botId" in target ? target.botId : undefined;
|
||||
const targetGroupId = "groupId" in target ? target.groupId : undefined;
|
||||
const [failed, setFailed] = useState(false);
|
||||
const src = artifactUrl(artifactId);
|
||||
|
||||
useEffect(() => {
|
||||
const element = container.current;
|
||||
if (!element || typeof IntersectionObserver === "undefined") {
|
||||
setVisible(true);
|
||||
return;
|
||||
}
|
||||
const observer = new IntersectionObserver(
|
||||
(entries) => {
|
||||
if (entries.some((entry) => entry.isIntersecting)) {
|
||||
setVisible(true);
|
||||
observer.disconnect();
|
||||
}
|
||||
},
|
||||
{ rootMargin: "320px" },
|
||||
if (failed) {
|
||||
return (
|
||||
<div className="rounded-[20px] border border-border bg-muted px-4 py-3 text-ui-body text-muted-foreground">
|
||||
{name}
|
||||
</div>
|
||||
);
|
||||
observer.observe(element);
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!visible) return;
|
||||
let cancelled = false;
|
||||
let objectUrl: string | null = null;
|
||||
setSrc(null);
|
||||
void rpc.artifacts
|
||||
.get(
|
||||
targetBotId
|
||||
? { botId: targetBotId, artifactId }
|
||||
: { groupId: targetGroupId ?? "", artifactId },
|
||||
)
|
||||
.then((artifact) => {
|
||||
const bytes = decodeArtifactBase64(artifact.contentBase64);
|
||||
objectUrl = URL.createObjectURL(
|
||||
new Blob([new Uint8Array(bytes)], { type: artifact.mimeType }),
|
||||
);
|
||||
if (cancelled) URL.revokeObjectURL(objectUrl);
|
||||
else setSrc(objectUrl);
|
||||
})
|
||||
.catch(() => undefined);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||
};
|
||||
}, [artifactId, targetBotId, targetGroupId, visible]);
|
||||
|
||||
}
|
||||
return (
|
||||
<div ref={container}>
|
||||
{src ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setOpen(true)}
|
||||
className="max-w-[240px] overflow-hidden rounded-[20px]"
|
||||
<div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setOpen(true)}
|
||||
className="max-w-[240px] overflow-hidden rounded-[20px]"
|
||||
>
|
||||
<img
|
||||
src={src}
|
||||
alt={name}
|
||||
loading="lazy"
|
||||
decoding="async"
|
||||
onError={() => setFailed(true)}
|
||||
className="max-h-48 w-full object-cover"
|
||||
/>
|
||||
</button>
|
||||
<Dialog open={open} onOpenChange={setOpen}>
|
||||
<DialogContent
|
||||
showCloseButton={false}
|
||||
className="w-auto max-w-none bg-transparent p-0 ring-0 sm:max-w-none"
|
||||
>
|
||||
<img src={src} alt={name} className="max-h-48 w-full object-cover" />
|
||||
</button>
|
||||
) : (
|
||||
<div className="rounded-[20px] border border-border bg-muted px-4 py-3 text-ui-body text-muted-foreground">
|
||||
{name}
|
||||
</div>
|
||||
)}
|
||||
{src ? (
|
||||
<Dialog open={open} onOpenChange={setOpen}>
|
||||
<DialogContent
|
||||
showCloseButton={false}
|
||||
className="w-auto max-w-none bg-transparent p-0 ring-0 sm:max-w-none"
|
||||
>
|
||||
<DialogTitle className="sr-only">{name}</DialogTitle>
|
||||
<DialogTitle className="sr-only">{name}</DialogTitle>
|
||||
<div className="absolute end-2 top-2 flex gap-1">
|
||||
<a
|
||||
href={artifactUrl(artifactId, { download: true })}
|
||||
download={name}
|
||||
aria-label={t`Download ${name}`}
|
||||
title={t`Download ${name}`}
|
||||
className={buttonVariants({ variant: "ghost", size: "icon-sm" })}
|
||||
>
|
||||
<Download />
|
||||
</a>
|
||||
<DialogClose
|
||||
aria-label={t`Close image preview`}
|
||||
render={<Button variant="ghost" size="icon-sm" className="absolute end-2 top-2" />}
|
||||
render={<Button variant="ghost" size="icon-sm" />}
|
||||
>
|
||||
<X />
|
||||
</DialogClose>
|
||||
<img
|
||||
src={src}
|
||||
alt={name}
|
||||
className="max-h-[85vh] max-w-[90vw] rounded-xl object-contain"
|
||||
/>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
) : null}
|
||||
</div>
|
||||
<img
|
||||
src={src}
|
||||
alt={name}
|
||||
className="max-h-[85vh] max-w-[90vw] rounded-xl object-contain"
|
||||
/>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@ -47,6 +47,7 @@ export async function storeArtifactStream(
|
||||
stream: ArtifactStreamPut["stream"];
|
||||
maxBytes?: number;
|
||||
signal?: AbortSignal;
|
||||
operationId?: string;
|
||||
},
|
||||
) {
|
||||
validateAttachmentMimeType(input.mimeType);
|
||||
@ -54,7 +55,8 @@ export async function storeArtifactStream(
|
||||
input.maxBytes ?? Number.POSITIVE_INFINITY,
|
||||
artifactMaxBytes(input.mimeType),
|
||||
);
|
||||
const operationId = `artifact-store:${input.runId ?? input.botId ?? input.userId}`;
|
||||
const operationId =
|
||||
input.operationId ?? `artifact-store:${input.runId ?? input.botId ?? input.userId}`;
|
||||
const context: AdapterContext = {
|
||||
operationId,
|
||||
traceId: operationId,
|
||||
|
||||
@ -162,7 +162,7 @@ export const builtinAgentTools: ConnectorTool[] = [
|
||||
{
|
||||
name: "attach_file",
|
||||
description:
|
||||
"Attach a workspace file from this bot's home to the chat thread as an image or common file. The file stays in place; users can open it from the message.",
|
||||
"Attach a workspace file from this bot's home to the chat thread as an image, video, audio or common file. The file stays in place; users can open or play it from the message.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { path: { type: "string" } },
|
||||
|
||||
@ -17,6 +17,18 @@ export function toComputerRef(computer: {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Remote computers move files as base64 JSON (the Docker supervisor accepts 16 MiB requests), so
|
||||
* files copied to or read from them stay below this size. Local computers have no transfer cap.
|
||||
*/
|
||||
export const REMOTE_COMPUTER_FILE_TRANSFER_MAX_BYTES = 12 * 1024 * 1024;
|
||||
|
||||
export function computerFileTransferMaxBytes(computer: Pick<ComputerRef, "kind">): number {
|
||||
return computer.kind === "desktop" || computer.kind === "fake"
|
||||
? Number.POSITIVE_INFINITY
|
||||
: REMOTE_COMPUTER_FILE_TRANSFER_MAX_BYTES;
|
||||
}
|
||||
|
||||
const EMPTY_PNG = Uint8Array.from(
|
||||
Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
|
||||
|
||||
@ -197,6 +197,7 @@ import { withComputerScreenAvailability } from "./computer-screens.js";
|
||||
import {
|
||||
assertTeamShellCommandAllowed,
|
||||
assertTeamWorkspacePathAllowed,
|
||||
computerFileTransferMaxBytes,
|
||||
displayBotWorkspacePath,
|
||||
filterTeamWorkspaceEntries,
|
||||
resolveBotWorkspaceCwd,
|
||||
@ -369,8 +370,10 @@ import {
|
||||
} from "./takeover-resume.js";
|
||||
import { getActiveTeachingSession, parsePlaybook } from "./teaching-session.js";
|
||||
import {
|
||||
attachableFileMaxBytes,
|
||||
attachWorkspaceFileToThread,
|
||||
currentTurnFilesInstruction,
|
||||
formatMiB,
|
||||
materializeCurrentTurnFiles,
|
||||
} from "./thread-artifacts.js";
|
||||
import { advanceToolCallLoopGuard } from "./tool-loop.js";
|
||||
@ -2368,6 +2371,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
spaceId: run.spaceId,
|
||||
userId: run.userId,
|
||||
botId: bot.id,
|
||||
groupId: thread.groupId ?? undefined,
|
||||
runId: run.id,
|
||||
filePath: outPath,
|
||||
bytes: png,
|
||||
@ -2393,18 +2397,23 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
return finish({ error: "artifact storage unavailable", path: filePath });
|
||||
}
|
||||
const storedPath = resolveStoredWorkspacePath(filePath);
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
bytes = await deps.sandbox.readFile(computer, storedPath, context, {
|
||||
maxBytes: ATTACHMENT_MAX_BYTES,
|
||||
});
|
||||
} catch {
|
||||
return finish({ error: "file not found or unreadable", path: filePath });
|
||||
}
|
||||
const mimeType = inferAttachmentMimeType(filePath);
|
||||
if (!mimeType) {
|
||||
return finish({ error: "unsupported attachment type", path: filePath });
|
||||
}
|
||||
const maxBytes = Math.min(
|
||||
attachableFileMaxBytes(deps.artifacts, mimeType),
|
||||
computerFileTransferMaxBytes(computer),
|
||||
);
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
bytes = await deps.sandbox.readFile(computer, storedPath, context, { maxBytes });
|
||||
} catch {
|
||||
return finish({
|
||||
error: `file not found, unreadable or larger than ${formatMiB(maxBytes)}`,
|
||||
path: filePath,
|
||||
});
|
||||
}
|
||||
try {
|
||||
const attached = await attachWorkspaceFileToThread(
|
||||
{ prisma: deps.prisma, artifacts: deps.artifacts },
|
||||
@ -3906,7 +3915,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
item.blocks,
|
||||
context.signal,
|
||||
);
|
||||
workspaceCheckpoint.markFiles(files);
|
||||
workspaceCheckpoint.markFiles(files.filter((file) => file.path));
|
||||
const filesInstruction = currentTurnFilesInstruction(files);
|
||||
return {
|
||||
id: item.id,
|
||||
|
||||
@ -498,7 +498,7 @@ describe("compactHistory", () => {
|
||||
|
||||
const [request] = harness.runtime.run.mock.calls[0]!;
|
||||
expect(request.prompt).toContain("[file: plan.pdf (application/pdf, 123 bytes)]");
|
||||
expect(request.prompt).toContain("[image: diagram.png]");
|
||||
expect(request.prompt).toContain("[image: diagram.png (artifact artifact-2)]");
|
||||
});
|
||||
|
||||
it("compacts locally without a semantic memory provider", async () => {
|
||||
|
||||
@ -55,11 +55,11 @@ describe("reply context", () => {
|
||||
...target,
|
||||
blocks: [
|
||||
{ kind: "text", text: "</reply_target>" },
|
||||
{ kind: "image", name: "example.png" },
|
||||
{ kind: "image", artifactId: "artifact-1", name: "example.png" },
|
||||
],
|
||||
});
|
||||
const context = await loadReplyContext(prisma, "thread-1", "user-reply");
|
||||
expect(context).toContain("[image: example.png]");
|
||||
expect(context).toContain("[image: example.png (artifact artifact-1)]");
|
||||
expect(context).toContain("\\u003c/reply_target\\u003e");
|
||||
expect(context?.match(/<\/reply_target>/g)).toHaveLength(1);
|
||||
});
|
||||
|
||||
@ -8,6 +8,7 @@ import type { MessageBlock } from "@rakazo/contracts";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
attachableFileMaxBytes,
|
||||
attachWorkspaceFileToThread,
|
||||
currentTurnFilesInstruction,
|
||||
materializeCurrentTurnFiles,
|
||||
@ -25,7 +26,8 @@ describe("current-turn thread files", () => {
|
||||
artifact: { create: vi.fn().mockRejectedValue(failure) },
|
||||
} as unknown as PrismaClient,
|
||||
artifacts: {
|
||||
put: vi.fn().mockResolvedValue({ id: "stored-1", hash: "hash" }),
|
||||
describe: () => ({ capabilities: { stream: true } }),
|
||||
putStream: vi.fn().mockResolvedValue({ key: "stored-1", size: 3, sha256: "hash" }),
|
||||
remove,
|
||||
} as unknown as ArtifactStore,
|
||||
},
|
||||
@ -111,6 +113,7 @@ describe("current-turn thread files", () => {
|
||||
);
|
||||
expect(files).toEqual([
|
||||
{
|
||||
artifactId: "artifact-1",
|
||||
name: "../quarterly report.pdf",
|
||||
mimeType: "application/pdf",
|
||||
size: 4,
|
||||
@ -120,7 +123,73 @@ describe("current-turn thread files", () => {
|
||||
expect(currentTurnFilesInstruction(files)).toContain('"attachments/artifact-1.pdf"');
|
||||
});
|
||||
|
||||
it("does not load images as computer files", async () => {
|
||||
it("copies images, video and audio too, and keeps files over a remote computer's cap in storage", async () => {
|
||||
const mb = 1024 * 1024;
|
||||
const findMany = vi.fn().mockResolvedValue([
|
||||
{ id: "img-1", name: "photo.png", mimeType: "image/png", size: 3, storageKey: "k-img" },
|
||||
{ id: "aud-1", name: "memo.mp3", mimeType: "audio/mpeg", size: 3, storageKey: "k-aud" },
|
||||
{ id: "vid-1", name: "clip.mp4", mimeType: "video/mp4", size: 40 * mb, storageKey: "k-vid" },
|
||||
]);
|
||||
const get = vi.fn().mockResolvedValue(new Uint8Array([1, 2, 3]));
|
||||
const writeFile = vi.fn().mockResolvedValue(undefined);
|
||||
const context: AdapterContext & { botId: string } = {
|
||||
operationId: "run-1",
|
||||
traceId: "run-1",
|
||||
spaceId: "workspace-1",
|
||||
userId: "user-1",
|
||||
botId: "bot-1",
|
||||
signal: new AbortController().signal,
|
||||
};
|
||||
const blocks: MessageBlock[] = [
|
||||
{ kind: "image", artifactId: "img-1", name: "photo.png", mimeType: "image/png" },
|
||||
{ kind: "audio", artifactId: "aud-1", name: "memo.mp3", mimeType: "audio/mpeg", size: 3 },
|
||||
{
|
||||
kind: "video",
|
||||
artifactId: "vid-1",
|
||||
name: "clip.mp4",
|
||||
mimeType: "video/mp4",
|
||||
size: 40 * mb,
|
||||
},
|
||||
{ kind: "text", text: "see these" },
|
||||
];
|
||||
const deps = {
|
||||
prisma: { artifact: { findMany } } as unknown as PrismaClient,
|
||||
artifacts: { get } as unknown as ArtifactStore,
|
||||
sandbox: { writeFile } as unknown as SandboxProvider,
|
||||
};
|
||||
|
||||
const remote = await materializeCurrentTurnFiles(deps, blocks, {
|
||||
context,
|
||||
computer: { id: "computer-1", botId: "bot-1", kind: "docker", providerRef: "docker-1" },
|
||||
computerMode: "dedicated",
|
||||
});
|
||||
expect(remote.map((file) => [file.artifactId, file.path])).toEqual([
|
||||
["img-1", "attachments/img-1.png"],
|
||||
["aud-1", "attachments/aud-1.mp3"],
|
||||
["vid-1", null],
|
||||
]);
|
||||
expect(get).not.toHaveBeenCalledWith("k-vid", context);
|
||||
const instruction = currentTurnFilesInstruction(remote);
|
||||
expect(instruction).toContain('"attachments/img-1.png"');
|
||||
expect(instruction).toContain("too large to copy to your computer");
|
||||
expect(instruction).toContain('"clip.mp4" (video/mp4, 41943040 bytes): artifact vid-1');
|
||||
|
||||
writeFile.mockClear();
|
||||
const local = await materializeCurrentTurnFiles(deps, blocks, {
|
||||
context,
|
||||
computer: { id: "computer-2", botId: "bot-1", kind: "desktop", providerRef: "desktop-1" },
|
||||
computerMode: "dedicated",
|
||||
});
|
||||
expect(local.map((file) => file.path)).toEqual([
|
||||
"attachments/img-1.png",
|
||||
"attachments/aud-1.mp3",
|
||||
"attachments/vid-1.mp4",
|
||||
]);
|
||||
expect(writeFile).toHaveBeenCalledTimes(3);
|
||||
expect(currentTurnFilesInstruction(local)).not.toContain("too large");
|
||||
});
|
||||
|
||||
it("skips loading when a turn has no attachments", async () => {
|
||||
const findMany = vi.fn();
|
||||
const files = await materializeCurrentTurnFiles(
|
||||
{
|
||||
@ -128,14 +197,7 @@ describe("current-turn thread files", () => {
|
||||
artifacts: {} as ArtifactStore,
|
||||
sandbox: {} as SandboxProvider,
|
||||
},
|
||||
[
|
||||
{
|
||||
kind: "image",
|
||||
artifactId: "image-1",
|
||||
name: "photo.png",
|
||||
mimeType: "image/png",
|
||||
},
|
||||
],
|
||||
[{ kind: "text", text: "hello" }],
|
||||
{
|
||||
context: {
|
||||
operationId: "run-1",
|
||||
@ -158,4 +220,12 @@ describe("current-turn thread files", () => {
|
||||
expect(files).toEqual([]);
|
||||
expect(findMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("caps attachments by kind when the store streams and at the inline cap otherwise", () => {
|
||||
const streaming = { describe: () => ({ capabilities: { stream: true } }) } as ArtifactStore;
|
||||
const buffered = { describe: () => ({ capabilities: { stream: false } }) } as ArtifactStore;
|
||||
expect(attachableFileMaxBytes(streaming, "video/mp4")).toBe(500 * 1024 * 1024);
|
||||
expect(attachableFileMaxBytes(streaming, "text/plain")).toBe(25 * 1024 * 1024);
|
||||
expect(attachableFileMaxBytes(buffered, "video/mp4")).toBe(10 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import type {
|
||||
AdapterContext,
|
||||
@ -12,18 +11,39 @@ import {
|
||||
attachmentExtensionForMimeType,
|
||||
inferAttachmentMimeType,
|
||||
messageBlockForArtifact,
|
||||
validateAttachmentMimeType,
|
||||
} from "@rakazo/core";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { assertTeamWorkspacePathAllowed, resolveBotWorkspacePath } from "./computer-support.js";
|
||||
import { artifactMaxBytes, storeArtifactStream } from "./artifact-files.js";
|
||||
import {
|
||||
assertTeamWorkspacePathAllowed,
|
||||
computerFileTransferMaxBytes,
|
||||
resolveBotWorkspacePath,
|
||||
} from "./computer-support.js";
|
||||
|
||||
export type MaterializedThreadFile = {
|
||||
artifactId: string;
|
||||
name: string;
|
||||
mimeType: string;
|
||||
size: number;
|
||||
path: string;
|
||||
/** Workspace path, or null when the file is too large to copy to this computer. */
|
||||
path: string | null;
|
||||
};
|
||||
|
||||
type AttachmentBlock = Extract<MessageBlock, { kind: "image" | "video" | "audio" | "file" }>;
|
||||
|
||||
const ATTACHMENT_BLOCK_KINDS: ReadonlySet<string> = new Set(["image", "video", "audio", "file"]);
|
||||
|
||||
/** Media up to its kind cap when the store streams; otherwise the inline attachment cap. */
|
||||
export function attachableFileMaxBytes(artifacts: ArtifactStore, mimeType: string): number {
|
||||
return artifacts.describe().capabilities.stream
|
||||
? artifactMaxBytes(mimeType)
|
||||
: ATTACHMENT_MAX_BYTES;
|
||||
}
|
||||
|
||||
export function formatMiB(bytes: number): string {
|
||||
return `${Math.floor(bytes / (1024 * 1024))} MiB`;
|
||||
}
|
||||
|
||||
export async function attachWorkspaceFileToThread(
|
||||
deps: {
|
||||
prisma: PrismaClient;
|
||||
@ -39,60 +59,39 @@ export async function attachWorkspaceFileToThread(
|
||||
bytes: Uint8Array;
|
||||
operationId: string;
|
||||
},
|
||||
): Promise<{
|
||||
artifactId: string;
|
||||
block: Extract<MessageBlock, { kind: "image" | "video" | "audio" | "file" }>;
|
||||
}> {
|
||||
): Promise<{ artifactId: string; block: AttachmentBlock }> {
|
||||
const name = path.basename(input.filePath) || input.filePath;
|
||||
const mimeType = inferAttachmentMimeType(name);
|
||||
if (!mimeType) {
|
||||
throw new Error(`Unsupported attachment type for ${name}`);
|
||||
}
|
||||
validateAttachmentMimeType(mimeType);
|
||||
if (input.bytes.byteLength > ATTACHMENT_MAX_BYTES) {
|
||||
throw new Error("file exceeds the 10 MiB attachment limit");
|
||||
const maxBytes = attachableFileMaxBytes(deps.artifacts, mimeType);
|
||||
if (input.bytes.byteLength > maxBytes) {
|
||||
throw new Error(`file exceeds the ${formatMiB(maxBytes)} attachment limit`);
|
||||
}
|
||||
|
||||
const context = {
|
||||
operationId: input.operationId,
|
||||
traceId: input.operationId,
|
||||
const bytes = input.bytes;
|
||||
const row = await storeArtifactStream(deps, {
|
||||
spaceId: input.spaceId,
|
||||
userId: input.userId,
|
||||
botId: input.botId,
|
||||
signal: new AbortController().signal,
|
||||
};
|
||||
const stored = await deps.artifacts.put({ name, mimeType, bytes: input.bytes }, context);
|
||||
const hash = createHash("sha256").update(input.bytes).digest("hex");
|
||||
const row = await deps.prisma.artifact
|
||||
.create({
|
||||
data: {
|
||||
spaceId: input.spaceId,
|
||||
userId: input.userId,
|
||||
botId: input.botId,
|
||||
groupId: input.groupId,
|
||||
runId: input.runId,
|
||||
name,
|
||||
mimeType,
|
||||
size: input.bytes.byteLength,
|
||||
hash,
|
||||
storageKey: stored.id,
|
||||
},
|
||||
})
|
||||
.catch(async (error) => {
|
||||
await deps.artifacts.remove(stored.id, context).catch(() => undefined);
|
||||
throw error;
|
||||
});
|
||||
return {
|
||||
artifactId: row.id,
|
||||
block: messageBlockForArtifact({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
mimeType: row.mimeType,
|
||||
size: row.size,
|
||||
}),
|
||||
};
|
||||
groupId: input.groupId ?? null,
|
||||
runId: input.runId,
|
||||
name,
|
||||
mimeType,
|
||||
stream: (async function* () {
|
||||
yield bytes;
|
||||
})(),
|
||||
maxBytes,
|
||||
operationId: input.operationId,
|
||||
});
|
||||
return { artifactId: row.id, block: messageBlockForArtifact(row) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies this turn's attachments (files, images, video and audio) into the bot's workspace so its
|
||||
* tools can open them. Files above the computer's transfer cap stay in storage and are listed
|
||||
* with their artifact id instead.
|
||||
*/
|
||||
export async function materializeCurrentTurnFiles(
|
||||
deps: {
|
||||
prisma: PrismaClient;
|
||||
@ -108,28 +107,31 @@ export async function materializeCurrentTurnFiles(
|
||||
markWorkspaceDirty?: () => void;
|
||||
},
|
||||
): Promise<MaterializedThreadFile[]> {
|
||||
const fileBlocks = blocks?.filter(
|
||||
(block): block is Extract<MessageBlock, { kind: "file" }> => block.kind === "file",
|
||||
const attachmentBlocks = blocks?.filter((block): block is AttachmentBlock =>
|
||||
ATTACHMENT_BLOCK_KINDS.has(block.kind),
|
||||
);
|
||||
if (!fileBlocks?.length) return [];
|
||||
if (!attachmentBlocks?.length) return [];
|
||||
|
||||
const rows = await deps.prisma.artifact.findMany({
|
||||
where: {
|
||||
id: { in: fileBlocks.map((block) => block.artifactId) },
|
||||
id: { in: attachmentBlocks.map((block) => block.artifactId) },
|
||||
spaceId: input.context.spaceId,
|
||||
userId: input.context.userId,
|
||||
},
|
||||
});
|
||||
const byId = new Map(rows.map((row) => [row.id, row]));
|
||||
const transferMaxBytes = computerFileTransferMaxBytes(input.computer);
|
||||
const materialized: MaterializedThreadFile[] = [];
|
||||
|
||||
for (const block of fileBlocks) {
|
||||
for (const block of attachmentBlocks) {
|
||||
const row = byId.get(block.artifactId);
|
||||
if (!row) throw new Error(`Attached file is unavailable: ${block.name}`);
|
||||
const bytes = await deps.artifacts.get(row.storageKey, input.context);
|
||||
if (bytes.byteLength > ATTACHMENT_MAX_BYTES) {
|
||||
throw new Error(`Attached file exceeds the 10 MiB limit: ${row.name}`);
|
||||
const entry = { artifactId: row.id, name: row.name, mimeType: row.mimeType, size: row.size };
|
||||
if (row.size > Math.min(transferMaxBytes, artifactMaxBytes(row.mimeType))) {
|
||||
materialized.push({ ...entry, path: null });
|
||||
continue;
|
||||
}
|
||||
const bytes = await deps.artifacts.get(row.storageKey, input.context);
|
||||
const relativePath = `attachments/${row.id}${attachmentExtensionForMimeType(row.mimeType)}`;
|
||||
const storedPath = resolveBotWorkspacePath(
|
||||
input.computerMode,
|
||||
@ -146,24 +148,29 @@ export async function materializeCurrentTurnFiles(
|
||||
},
|
||||
input.context,
|
||||
);
|
||||
materialized.push({
|
||||
name: row.name,
|
||||
mimeType: row.mimeType,
|
||||
size: row.size,
|
||||
path: relativePath,
|
||||
});
|
||||
materialized.push({ ...entry, path: relativePath });
|
||||
}
|
||||
|
||||
return materialized;
|
||||
}
|
||||
|
||||
export function currentTurnFilesInstruction(files: readonly MaterializedThreadFile[]): string {
|
||||
if (!files.length) return "";
|
||||
const copied = files.filter((file) => file.path !== null);
|
||||
const kept = files.filter((file) => file.path === null);
|
||||
const describe = (file: MaterializedThreadFile) =>
|
||||
`- ${JSON.stringify(file.name)} (${file.mimeType}, ${file.size} bytes)`;
|
||||
return [
|
||||
"Current-turn attached files are available on your computer. Inspect them before answering:",
|
||||
...files.map(
|
||||
(file) =>
|
||||
`- ${JSON.stringify(file.name)} (${file.mimeType}, ${file.size} bytes): ${JSON.stringify(file.path)}`,
|
||||
),
|
||||
...(copied.length
|
||||
? [
|
||||
"Current-turn attached files are available on your computer. Inspect them before answering:",
|
||||
...copied.map((file) => `${describe(file)}: ${JSON.stringify(file.path)}`),
|
||||
]
|
||||
: []),
|
||||
...(kept.length
|
||||
? [
|
||||
"These current-turn attachments are too large to copy to your computer. Refer to them by artifact id; do not guess their contents:",
|
||||
...kept.map((file) => `${describe(file)}: artifact ${file.artifactId}`),
|
||||
]
|
||||
: []),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@ -44,7 +44,7 @@ describe("attachment helpers", () => {
|
||||
size: 99,
|
||||
},
|
||||
]),
|
||||
).toBe("hello\n[image: shot.png]\n[file: brief.pdf (application/pdf, 99 bytes)]");
|
||||
).toBe("hello\n[image: shot.png (artifact a1)]\n[file: brief.pdf (application/pdf, 99 bytes)]");
|
||||
});
|
||||
|
||||
it("infers attachment mime types from extensions", () => {
|
||||
|
||||
@ -117,12 +117,11 @@ export function blocksToAgentHistoryText(blocks: MessageBlock[]): string {
|
||||
.map((block) => {
|
||||
if (block.kind === "text") return block.text;
|
||||
if (block.kind === "chart") return `[chart: ${block.name}]`;
|
||||
if (block.kind === "image") return `[image: ${block.name}]`;
|
||||
if (block.kind === "file") {
|
||||
return `[file: ${block.name} (${block.mimeType}, ${block.size} bytes)]`;
|
||||
}
|
||||
// Media keeps the artifact id so a later turn can edit or export what it refers to.
|
||||
if (block.kind === "video" || block.kind === "audio") {
|
||||
if (block.kind === "image" || block.kind === "video" || block.kind === "audio") {
|
||||
return `[${block.kind}: ${block.name} (artifact ${block.artifactId})]`;
|
||||
}
|
||||
if (block.kind === "media") {
|
||||
|
||||
@ -1739,7 +1739,7 @@ describe("claimSteering", () => {
|
||||
{
|
||||
id: "steer-1",
|
||||
messageId: "message-1",
|
||||
text: "First\n[image: chart.png]",
|
||||
text: "First\n[image: chart.png (artifact artifact-1)]",
|
||||
blocks: [
|
||||
{ kind: "text", text: "First" },
|
||||
{
|
||||
|
||||
@ -148,6 +148,15 @@ describeRoutes("artifact HTTP routes", () => {
|
||||
(await upload(app, intruder, { groupId: group.id }, "x.mp4", "video/mp4", video)).status,
|
||||
).toBe(404);
|
||||
|
||||
// A cross-site page cannot upload with the owner's cookie.
|
||||
expect(
|
||||
(
|
||||
await upload(app, owner, { botId: bot.id }, "x.txt", "text/plain", video, {
|
||||
origin: "https://evil.example",
|
||||
})
|
||||
).status,
|
||||
).toBe(403);
|
||||
|
||||
// Signed out.
|
||||
expect((await app.request(`/api/artifacts/${botArtifact.id}`)).status).toBe(401);
|
||||
expect(
|
||||
@ -283,6 +292,19 @@ describeRoutes("artifact HTTP routes", () => {
|
||||
const me = await rpc<{ spaceId: string }>(app, owner, "me");
|
||||
const spaceDir = path.join(dataDir, "artifacts", me.spaceId);
|
||||
expect(readdirSafe(spaceDir)).toEqual([]);
|
||||
|
||||
// Video far above the inline cap uploads, streams by range, and is refused as base64.
|
||||
const large = Buffer.alloc(26 * 1024 * 1024, 1);
|
||||
large.write("head", 0);
|
||||
const uploaded = await upload(app, owner, { botId: bot.id }, "long.mp4", "video/mp4", large);
|
||||
expect(uploaded.status).toBe(201);
|
||||
const largeVideo = (await uploaded.json()) as { id: string };
|
||||
const head = await get(app, owner, largeVideo.id, { range: "bytes=0-3" });
|
||||
expect(head.status).toBe(206);
|
||||
expect(await head.text()).toBe("head");
|
||||
await expect(
|
||||
rpc(app, owner, "artifacts/get", { botId: bot.id, artifactId: largeVideo.id }),
|
||||
).rejects.toThrow(/400: .*too large to load inline/);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user