feat(web): play video and audio inline and stream uploads and downloads

Media sent in a conversation had nowhere to go: the composer capped every
file at 10 MiB and pushed it through a base64 RPC, video and audio blocks
did not render, and downloads decoded whole files in the page.

- Video and audio blocks play inline through MediaPlayer, which loads
  nothing until the player is near the viewport, sizes the frame from the
  block's dimensions (vertical clips stay within the column) and uses the
  poster when one exists. Files the browser cannot play fall back to their
  name with a download action.
- Images load straight from the artifact route; the lightbox gains a
  download action.
- File cards, the Markdown preview and downloads read from the artifact
  route instead of decoding base64.
- The composer accepts each kind up to its cap and names the computed
  limit when it skips a file; video, audio and anything above the inline
  cap upload as a raw stream, small files keep the inline RPC.
- MediaPlayer and artifactUrl are what the media card builds on.

Tests: MediaPlayer laziness, sizing and fallback, the upload helper, and an
E2E that uploads a tiny MP4 fixture (VP9, since Playwright's Chromium has
no H.264 decoder), plays it from the artifact route and captures
media-video-block, plus the image lightbox download.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude 2026-10-03 13:50:02 +00:00
parent 62e9bc67f1
commit 979034a8a6
No known key found for this signature in database
12 changed files with 562 additions and 245 deletions

Binary file not shown.

View File

@ -0,0 +1,79 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import { expect, test } from "@playwright/test";
import { captureScreenshot, completeOnboarding, signup } from "./helpers";
test.describe.configure({ mode: "serial" });
// VP9 in MP4: Playwright's Chromium has no H.264 decoder, real browsers play both.
const clip = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "media-clip.mp4");
const pixel = Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
"base64",
);
test("an uploaded video streams from the artifact route and plays inline", async ({
page,
}, testInfo) => {
const stamp = Date.now();
await signup(page, `media-video-${stamp}@rakazo.test`, "password12", "Media Video");
await completeOnboarding(page);
const upload = page.waitForResponse(
(response) =>
response.request().method() === "POST" && /\/api\/artifacts\?/.test(response.url()),
);
await page.locator('input[type="file"]').setInputFiles(clip);
await expect(page.getByRole("button", { name: "Remove media-clip.mp4" })).toBeVisible();
await page.getByRole("combobox", { name: /Message/ }).fill("here is the clip");
await page.keyboard.press("Enter");
expect((await upload).status()).toBe(201);
const video = page.getByTestId("media-video").locator("video");
await expect(video).toBeVisible({ timeout: 30_000 });
await expect(video).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/);
await expect
.poll(() => video.evaluate((element: HTMLVideoElement) => element.readyState), {
timeout: 15_000,
})
.toBeGreaterThanOrEqual(1);
expect(await video.evaluate((element: HTMLVideoElement) => element.duration)).toBeCloseTo(1, 0);
await video.evaluate(async (element: HTMLVideoElement) => {
element.muted = true;
await element.play();
});
await expect
.poll(() => video.evaluate((element: HTMLVideoElement) => element.currentTime))
.toBeGreaterThan(0);
await video.evaluate((element: HTMLVideoElement) => {
element.pause();
element.currentTime = 0.5;
element.scrollIntoView({ block: "center" });
});
await captureScreenshot(page, testInfo, "media-video-block");
});
test("the image lightbox downloads the original from the artifact route", async ({ page }) => {
const stamp = Date.now();
await signup(page, `media-image-${stamp}@rakazo.test`, "password12", "Media Image");
await completeOnboarding(page);
await page
.locator('input[type="file"]')
.setInputFiles({ name: "pixel.png", mimeType: "image/png", buffer: pixel });
await page.getByRole("combobox", { name: /Message/ }).fill("a pixel");
await page.keyboard.press("Enter");
const thumbnail = page.getByRole("img", { name: "pixel.png" });
await expect(thumbnail).toBeVisible({ timeout: 30_000 });
await expect(thumbnail).toHaveAttribute("src", /^\/api\/artifacts\/[^/?]+$/);
await thumbnail.click();
const dialog = page.getByRole("dialog", { name: "pixel.png" });
const download = dialog.getByRole("link", { name: "Download pixel.png" });
await expect(download).toHaveAttribute("href", /\/api\/artifacts\/[^/?]+\?download=1$/);
const saved = page.waitForEvent("download");
await download.click();
expect((await saved).suggestedFilename()).toBe("pixel.png");
});

View File

@ -1,19 +1,12 @@
import { i18n } from "@lingui/core";
import { t } from "@lingui/core/macro";
import { Trans, useLingui } from "@lingui/react/macro";
import { ChatMarkdown } from "@rakazo/chat-ui/web";
import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web";
import { Download, FileText, X } from "lucide-react";
import { type RefObject, useEffect, useRef, useState } from "react";
import {
type ArtifactTarget,
downloadArtifact,
downloadArtifactBytes,
fetchArtifactBytes,
} from "../lib/artifact-open";
import { downloadArtifact, fetchArtifactBytes } from "../lib/artifact-open";
import { formatBytes } from "../lib/format-bytes";
type ArtifactFileCardProps = {
target: ArtifactTarget;
artifactId: string;
name: string;
mimeType: string;
@ -26,23 +19,14 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
const previewButton = useRef<HTMLButtonElement>(null);
const closeButton = useRef<HTMLButtonElement>(null);
const [previewOpen, setPreviewOpen] = useState(false);
const [downloadError, setDownloadError] = useState<string | null>(null);
async function startDownload() {
setDownloadError(null);
try {
await downloadArtifact(props.target, props.artifactId, props.name, props.mimeType);
} catch {
setDownloadError(t`Could not download ${props.name}. Try again.`);
}
}
const startDownload = () => downloadArtifact(props.artifactId, props.name);
if (!markdown) {
return (
<div>
<button
type="button"
onClick={() => void startDownload()}
onClick={startDownload}
className="rounded-2xl border border-border bg-card px-4 py-3 text-left text-ui-body text-foreground hover:bg-accent"
>
<div className="font-medium">{props.name}</div>
@ -50,7 +34,6 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
{props.mimeType} · {formatBytes(props.size)}
</div>
</button>
{downloadError ? <DownloadError message={downloadError} /> : null}
</div>
);
}
@ -80,13 +63,12 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
type="button"
aria-label={t`Download ${props.name}`}
title={t`Download ${props.name}`}
onClick={() => void startDownload()}
onClick={startDownload}
className="grid w-14 shrink-0 place-items-center border-l border-border text-muted-foreground hover:bg-accent hover:text-foreground"
>
<Download strokeWidth={1.8} className="size-icon-lg" />
</button>
</div>
{downloadError ? <DownloadError message={downloadError} /> : null}
</div>
<Dialog open={previewOpen} onOpenChange={setPreviewOpen}>
<DialogContent
@ -103,32 +85,25 @@ export function ArtifactFileCard(props: ArtifactFileCardProps) {
}
function MarkdownPreview({
target,
artifactId,
name,
mimeType,
closeButtonRef,
}: ArtifactFileCardProps & { closeButtonRef: RefObject<HTMLButtonElement | null> }) {
const { t } = useLingui();
const [downloadError, setDownloadError] = useState<string | null>(null);
const [state, setState] = useState<
| { status: "loading" }
| { status: "ready"; bytes: Uint8Array; markdown: string }
| { status: "ready"; markdown: string }
| { status: "error"; message: string }
>({ status: "loading" });
const targetBotId = "botId" in target ? target.botId : undefined;
const targetGroupId = "groupId" in target ? target.groupId : undefined;
useEffect(() => {
let cancelled = false;
const artifactTarget: ArtifactTarget =
targetBotId !== undefined ? { botId: targetBotId } : { groupId: targetGroupId! };
void fetchArtifactBytes(artifactTarget, artifactId)
void fetchArtifactBytes(artifactId)
.then((bytes) => {
if (cancelled) return;
try {
const markdown = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
setState({ status: "ready", bytes, markdown });
setState({ status: "ready", markdown });
} catch {
setState({ status: "error", message: t`This file is not valid UTF-8 Markdown.` });
}
@ -143,7 +118,7 @@ function MarkdownPreview({
return () => {
cancelled = true;
};
}, [artifactId, targetBotId, targetGroupId, t]);
}, [artifactId, t]);
return (
<>
@ -157,17 +132,7 @@ function MarkdownPreview({
className="rounded-full text-muted-foreground"
aria-label={t`Download ${name}`}
title={t`Download ${name}`}
onClick={() =>
void (async () => {
setDownloadError(null);
try {
if (state.status === "ready") downloadArtifactBytes(name, mimeType, state.bytes);
else await downloadArtifact(target, artifactId, name, mimeType);
} catch {
setDownloadError(t`Could not download ${name}. Try again.`);
}
})()
}
onClick={() => downloadArtifact(artifactId, name)}
>
<Download />
</Button>
@ -181,11 +146,6 @@ function MarkdownPreview({
<X />
</DialogClose>
</header>
{downloadError ? (
<div className="shrink-0 px-5 pt-4">
<DownloadError message={downloadError} />
</div>
) : null}
<div className="min-h-0 flex-1 overflow-y-auto">
<article className="mx-auto w-full max-w-[760px] px-8 py-10 text-ui-body leading-7 text-foreground sm:px-12 sm:py-12">
{state.status === "loading" ? (
@ -204,23 +164,3 @@ function MarkdownPreview({
</>
);
}
function DownloadError({ message }: { message: string }) {
return (
<div role="alert" className="mt-2 text-left text-ui-body-sm text-destructive">
{message}
</div>
);
}
function formatBytes(size: number) {
const locale = i18n.locale || "en";
if (size < 1024) return t`${size} B`;
const format = (value: number) =>
new Intl.NumberFormat(locale, {
minimumFractionDigits: 1,
maximumFractionDigits: 1,
}).format(value);
if (size < 1024 * 1024) return t`${format(size / 1024)} KB`;
return t`${format(size / (1024 * 1024))} MB`;
}

View File

@ -0,0 +1,84 @@
// @vitest-environment jsdom
import { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { renderToStaticMarkup } from "react-dom/server";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("@lingui/react/macro", () => {
const t = (parts: TemplateStringsArray, ...values: unknown[]) =>
parts.reduce(
(text, part, index) => text + part + (index < values.length ? values[index] : ""),
"",
);
return { useLingui: () => ({ t }) };
});
import { MediaPlayer } from "./MediaPlayer";
let container: HTMLDivElement;
let root: Root;
beforeEach(() => {
vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
container = document.createElement("div");
document.body.append(container);
root = createRoot(container);
});
afterEach(async () => {
await act(async () => root.unmount());
container.remove();
vi.unstubAllGlobals();
});
describe("MediaPlayer", () => {
it("loads nothing before it is near the viewport and fits vertical video to the column", () => {
const html = renderToStaticMarkup(
<MediaPlayer artifactId="art-1" kind="video" name="clip.mp4" width={1080} height={1920} />,
);
expect(html).toContain('aria-label="clip.mp4"');
expect(html).not.toContain("src=");
expect(html).toContain("width:270px");
expect(html).toContain("aspect-ratio:270 / 480");
});
it("streams video with its poster from the artifact route once visible", async () => {
vi.stubGlobal(
"IntersectionObserver",
class {
constructor(private readonly callback: IntersectionObserverCallback) {}
observe() {
this.callback([{ isIntersecting: true } as IntersectionObserverEntry], this as never);
}
disconnect() {}
},
);
await act(async () =>
root.render(
<MediaPlayer artifactId="art-1" kind="video" name="clip.mp4" posterArtifactId="art-2" />,
),
);
const video = container.querySelector("video")!;
expect(video.getAttribute("src")).toBe("/api/artifacts/art-1");
expect(video.getAttribute("poster")).toBe("/api/artifacts/art-2");
expect(video.getAttribute("preload")).toBe("metadata");
expect(video.hasAttribute("controls")).toBe(true);
});
it("falls back to a download link when the browser cannot play the file", async () => {
await act(async () =>
root.render(<MediaPlayer artifactId="art-3" kind="audio" name="memo.ogg" />),
);
const audio = container.querySelector("audio")!;
expect(audio.getAttribute("src")).toBe("/api/artifacts/art-3");
await act(async () => {
audio.dispatchEvent(new Event("error"));
});
expect(container.querySelector("audio")).toBeNull();
const link = container.querySelector("a")!;
expect(link.getAttribute("href")).toBe("/api/artifacts/art-3?download=1");
expect(link.getAttribute("download")).toBe("memo.ogg");
expect(link.getAttribute("aria-label")).toBe("Download memo.ogg");
});
});

View File

@ -0,0 +1,121 @@
import { useLingui } from "@lingui/react/macro";
import { buttonVariants } from "@rakazo/ui-web";
import { Download } from "lucide-react";
import { type RefObject, useEffect, useRef, useState } from "react";
import { artifactUrl } from "../../lib/artifact-url";
export type MediaPlayerProps = {
artifactId: string;
kind: "video" | "audio";
name: string;
posterArtifactId?: string;
width?: number;
height?: number;
};
const MAX_VIDEO_WIDTH = 420;
const MAX_VIDEO_HEIGHT = 480;
/** Plays a video or audio artifact in place; loads nothing until it is near the viewport. */
export function MediaPlayer({
artifactId,
kind,
name,
posterArtifactId,
width,
height,
}: MediaPlayerProps) {
const container = useRef<HTMLDivElement>(null);
const near = useNearViewport(container);
const [failed, setFailed] = useState(false);
const src = near ? artifactUrl(artifactId) : undefined;
if (failed) return <MediaFallback artifactId={artifactId} name={name} />;
if (kind === "audio") {
return (
<div ref={container} className="w-72 max-w-full" data-testid="media-audio">
{/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */}
<audio
src={src}
controls
preload="metadata"
aria-label={name}
className="w-full"
onError={() => setFailed(true)}
/>
</div>
);
}
const box = videoBox(width, height);
return (
<div
ref={container}
className="max-w-full overflow-hidden rounded-[20px] bg-muted"
style={{ width: box.width, aspectRatio: `${box.width} / ${box.height}` }}
data-testid="media-video"
>
{/* biome-ignore lint/a11y/useMediaCaption: user media has no caption track. */}
<video
src={src}
poster={near && posterArtifactId ? artifactUrl(posterArtifactId) : undefined}
controls
playsInline
preload="metadata"
aria-label={name}
className="block size-full object-contain"
onError={() => setFailed(true)}
/>
</div>
);
}
/** Fits the frame into the chat column, defaulting to 16:9 until the size is known. */
function videoBox(width?: number, height?: number) {
const w = width && width > 0 ? width : 16;
const h = height && height > 0 ? height : 9;
const scale = Math.min(MAX_VIDEO_WIDTH / w, MAX_VIDEO_HEIGHT / h);
return { width: Math.round(w * scale), height: Math.round(h * scale) };
}
function MediaFallback({ artifactId, name }: { artifactId: string; name: string }) {
const { t } = useLingui();
return (
<div className="flex max-w-full items-center gap-2 rounded-[20px] border border-border bg-muted py-2 ps-4 pe-2 text-ui-body text-muted-foreground">
<span className="min-w-0 truncate">{name}</span>
<a
href={artifactUrl(artifactId, { download: true })}
download={name}
aria-label={t`Download ${name}`}
title={t`Download ${name}`}
className={buttonVariants({ variant: "ghost", size: "icon-sm" })}
>
<Download />
</a>
</div>
);
}
function useNearViewport(ref: RefObject<HTMLElement | null>) {
const [near, setNear] = useState(false);
useEffect(() => {
const element = ref.current;
if (!element || typeof IntersectionObserver === "undefined") {
setNear(true);
return;
}
const observer = new IntersectionObserver(
(entries) => {
if (entries.some((entry) => entry.isIntersecting)) {
setNear(true);
observer.disconnect();
}
},
{ rootMargin: "320px" },
);
observer.observe(element);
return () => observer.disconnect();
}, [ref]);
return near;
}

View File

@ -1,40 +1,29 @@
import { rpc } from "./rpc";
import { t } from "@lingui/core/macro";
import { artifactUrl } from "./artifact-url";
export type ArtifactTarget = { botId: string } | { groupId: string };
export function decodeArtifactBase64(contentBase64: string): Uint8Array {
const binary = atob(contentBase64);
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) {
bytes[index] = binary.charCodeAt(index);
}
return bytes;
}
export async function fetchArtifactBytes(
target: ArtifactTarget,
artifactId: string,
): Promise<Uint8Array> {
const artifact = await rpc.artifacts.get({ ...target, artifactId });
return decodeArtifactBase64(artifact.contentBase64);
export async function fetchArtifactBytes(artifactId: string): Promise<Uint8Array> {
const response = await fetch(artifactUrl(artifactId), { credentials: "include" });
if (!response.ok) throw new Error(t`Could not load this file.`);
return new Uint8Array(await response.arrayBuffer());
}
export function downloadArtifactBytes(name: string, mimeType: string, bytes: Uint8Array): void {
const blob = new Blob([new Uint8Array(bytes)], { type: mimeType });
const url = URL.createObjectURL(blob);
saveUrl(url, name);
window.setTimeout(() => URL.revokeObjectURL(url), 60_000);
}
/** Lets the browser stream the download from the artifact route instead of buffering it. */
export function downloadArtifact(artifactId: string, name: string): void {
saveUrl(artifactUrl(artifactId, { download: true }), name);
}
function saveUrl(url: string, name: string) {
const anchor = document.createElement("a");
anchor.href = url;
anchor.download = name;
anchor.click();
window.setTimeout(() => URL.revokeObjectURL(url), 60_000);
}
export async function downloadArtifact(
target: ArtifactTarget,
artifactId: string,
name: string,
mimeType: string,
): Promise<void> {
const bytes = await fetchArtifactBytes(target, artifactId);
downloadArtifactBytes(name, mimeType, bytes);
}

View File

@ -0,0 +1,83 @@
import { ATTACHMENT_MAX_BYTES } from "@rakazo/contracts";
import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("@lingui/core/macro", () => ({
t: (parts: TemplateStringsArray, ...values: unknown[]) =>
parts.reduce(
(text, part, index) => text + part + (index < values.length ? values[index] : ""),
"",
),
}));
const create = vi.hoisted(() => vi.fn());
vi.mock("./rpc", () => ({
rpc: { artifacts: { create } },
withSpaceHeaders: (init: HeadersInit) => new Headers(init),
}));
import {
attachmentMaxBytes,
uploadArtifactFile,
uploadAttachment,
usesStreamingUpload,
} from "./artifact-upload";
const artifact = {
id: "art-1",
botId: "bot-1",
groupId: null,
runId: null,
name: "clip.mp4",
mimeType: "video/mp4",
size: 4,
createdAt: "2026-10-03T12:00:00.000Z",
};
afterEach(() => {
vi.unstubAllGlobals();
create.mockReset();
});
describe("artifact uploads", () => {
it("streams media and large files and keeps small files on the inline RPC", () => {
expect(usesStreamingUpload(10, "video/mp4")).toBe(true);
expect(usesStreamingUpload(10, "audio/mpeg")).toBe(true);
expect(usesStreamingUpload(10, "image/png")).toBe(false);
expect(usesStreamingUpload(ATTACHMENT_MAX_BYTES + 1, "image/png")).toBe(true);
expect(attachmentMaxBytes("video/mp4")).toBe(500 * 1024 * 1024);
expect(attachmentMaxBytes("text/plain")).toBe(25 * 1024 * 1024);
});
it("posts the raw file with its target, name and type", async () => {
const fetch = vi.fn(async () => Response.json(artifact, { status: 201 }));
vi.stubGlobal("fetch", fetch);
const file = new File(["data"], "clip.mp4", { type: "video/mp4" });
await expect(uploadAttachment({ groupId: "group-1" }, file, "video/mp4")).resolves.toEqual(
artifact,
);
const [url, init] = fetch.mock.calls[0] as unknown as [string, RequestInit];
expect(url).toBe("/api/artifacts?groupId=group-1&name=clip.mp4");
expect(init.method).toBe("POST");
expect(init.credentials).toBe("include");
expect(new Headers(init.headers).get("content-type")).toBe("video/mp4");
expect(init.body).toBe(file);
expect(create).not.toHaveBeenCalled();
});
it("surfaces the server's reason, or a generic one", async () => {
vi.stubGlobal(
"fetch",
vi.fn(async () => Response.json({ error: "File is too large." }, { status: 413 })),
);
await expect(
uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"),
).rejects.toThrow("File is too large.");
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response("bad gateway", { status: 502 })),
);
await expect(
uploadArtifactFile({ botId: "bot-1" }, new Blob(["x"]), "big.mp4", "video/mp4"),
).rejects.toThrow("Could not upload big.mp4. Try again.");
});
});

View File

@ -0,0 +1,68 @@
import { t } from "@lingui/core/macro";
import {
ARTIFACT_MAX_BYTES_BY_KIND,
ArtifactSchema,
ATTACHMENT_MAX_BYTES,
attachmentKindOf,
} from "@rakazo/contracts";
import type { ArtifactTarget } from "./artifact-open";
import { rpc, withSpaceHeaders } from "./rpc";
type UploadedArtifact = ReturnType<typeof ArtifactSchema.parse>;
/** Largest file of this type the composer accepts. */
export function attachmentMaxBytes(mimeType: string): number {
return ARTIFACT_MAX_BYTES_BY_KIND[attachmentKindOf(mimeType)];
}
/** Media and anything above the inline RPC cap upload as a raw stream instead of base64. */
export function usesStreamingUpload(size: number, mimeType: string): boolean {
const kind = attachmentKindOf(mimeType);
return size > ATTACHMENT_MAX_BYTES || kind === "video" || kind === "audio";
}
export async function uploadAttachment(
target: ArtifactTarget,
file: File,
mimeType: string,
): Promise<UploadedArtifact> {
if (usesStreamingUpload(file.size, mimeType)) {
return uploadArtifactFile(target, file, file.name, mimeType);
}
const contentBase64 = await readFileAsBase64(file);
return rpc.artifacts.create({ ...target, name: file.name, mimeType, contentBase64 });
}
/** Uploads a file as a raw streamed body to the artifact route. */
export async function uploadArtifactFile(
target: ArtifactTarget,
file: Blob,
name: string,
mimeType: string,
): Promise<UploadedArtifact> {
const query = new URLSearchParams({ ...target, name });
const response = await fetch(`/api/artifacts?${query}`, {
method: "POST",
headers: withSpaceHeaders({ "content-type": mimeType }),
credentials: "include",
body: file,
});
const body: unknown = await response.json().catch(() => null);
const artifact = ArtifactSchema.safeParse(body);
if (response.ok && artifact.success) return artifact.data;
const error = (body as { error?: unknown } | null)?.error;
throw new Error(typeof error === "string" ? error : t`Could not upload ${name}. Try again.`);
}
function readFileAsBase64(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const result = typeof reader.result === "string" ? reader.result : "";
const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result;
resolve(base64);
};
reader.onerror = () => reject(reader.error ?? new Error("Failed to read file"));
reader.readAsDataURL(file);
});
}

View File

@ -1,38 +1,8 @@
import { t } from "@lingui/core/macro";
import { ArtifactSchema } from "@rakazo/contracts";
import type { ArtifactTarget } from "./artifact-open";
import { withSpaceHeaders } from "./rpc";
const ARTIFACT_PATH = "/api/artifacts";
/**
* Same-origin URL that streams an artifact with Range support, for `<video>`, `<audio>`, `<img>`
* and downloads. The session cookie authorizes it; `download` forces a save instead of inline.
*/
export function artifactUrl(artifactId: string, options: { download?: boolean } = {}): string {
const path = `${ARTIFACT_PATH}/${encodeURIComponent(artifactId)}`;
const path = `/api/artifacts/${encodeURIComponent(artifactId)}`;
return options.download ? `${path}?download=1` : path;
}
/** Uploads a file as a raw streamed body, for media and files above the inline RPC cap. */
export async function uploadArtifactFile(
target: ArtifactTarget,
file: Blob,
name: string,
mimeType: string,
): Promise<ReturnType<typeof ArtifactSchema.parse>> {
const query = new URLSearchParams(
"botId" in target ? { botId: target.botId, name } : { groupId: target.groupId, name },
);
const response = await fetch(`${ARTIFACT_PATH}?${query}`, {
method: "POST",
headers: withSpaceHeaders({ "content-type": mimeType }),
credentials: "include",
body: file,
});
const body: unknown = await response.json().catch(() => null);
const artifact = ArtifactSchema.safeParse(body);
if (response.ok && artifact.success) return artifact.data;
const error = (body as { error?: unknown } | null)?.error;
throw new Error(typeof error === "string" ? error : t`Could not upload ${name}. Try again.`);
}

View File

@ -0,0 +1,12 @@
import { i18n } from "@lingui/core";
import { t } from "@lingui/core/macro";
/** Human file size in the UI locale: B, KB or MB with at most one decimal. */
export function formatBytes(size: number): string {
const locale = i18n.locale || "en";
if (size < 1024) return t`${size} B`;
const format = (value: number) =>
new Intl.NumberFormat(locale, { maximumFractionDigits: 1 }).format(value);
if (size < 1024 * 1024) return t`${format(size / 1024)} KB`;
return t`${format(size / (1024 * 1024))} MB`;
}

View File

@ -41,7 +41,6 @@ import type {
} from "@rakazo/contracts";
import {
ATTACHMENT_ALLOWED_MIME_TYPES,
ATTACHMENT_MAX_BYTES,
ATTACHMENT_MAX_COUNT,
canReactToThreadMessage,
MESSAGE_REACTIONS,
@ -169,6 +168,7 @@ import { ComputerUpdateProgress } from "../components/ComputerUpdateProgress";
import { DataTrustPanel } from "../components/DataTrustPanel";
import { MainInbox } from "../components/MainInbox";
import { MessageHoverMetadata } from "../components/MessageHoverMetadata";
import { MediaPlayer } from "../components/media/MediaPlayer";
import { OoGroupStage, OoStage } from "../components/stage/OoStage";
import { TradingModeIndicator } from "../components/TradingModeIndicator";
import { SkillDraftCard } from "../components/teach/SkillDraftCard";
@ -180,6 +180,7 @@ import { useSidebarResize } from "../hooks/useSidebarResize";
import { readActivityMode, writeActivityMode } from "../lib/activity-mode";
import { needsYouChipLabel, rosterActivitySubtitle, rosterStatusDot } from "../lib/agent-activity";
import type { ArtifactTarget } from "../lib/artifact-open";
import { attachmentMaxBytes, uploadAttachment } from "../lib/artifact-upload";
import { authClient } from "../lib/auth";
import { takeInitialBootstrap } from "../lib/bootstrap";
import {
@ -197,6 +198,7 @@ import { browserPreviewActivity } from "../lib/browser-preview-context";
import { loadComputerScreen } from "../lib/computer-screen";
import { desktopBridge } from "../lib/desktop";
import { scheduleFocusPrompt } from "../lib/focus-prompt";
import { formatBytes } from "../lib/format-bytes";
import {
isWorkingRunStatus,
resolveActiveGroupSpeaker,
@ -2711,15 +2713,16 @@ export function ShellPage() {
skipped.push(t`${file.name} (max ${ATTACHMENT_MAX_COUNT} attachments)`);
continue;
}
if (file.size > ATTACHMENT_MAX_BYTES) {
skipped.push(t`${file.name} (over 10 MiB)`);
continue;
}
const mimeType = inferAttachmentMimeType(file.name, file.type);
if (!mimeType) {
skipped.push(file.name);
continue;
}
const maxBytes = attachmentMaxBytes(mimeType);
if (file.size > maxBytes) {
skipped.push(t`${file.name} (over ${formatBytes(maxBytes)})`);
continue;
}
next.push({
id: `${file.name}-${file.size}-${file.lastModified}-${next.length}`,
threadKey,
@ -2859,11 +2862,10 @@ export function ShellPage() {
if (!mimeType) {
throw new Error(t`Unsupported file type: ${pending.file.name}`);
}
const contentBase64 = await readFileAsBase64(pending.file);
const artifact = await rpc.artifacts.create(
groupTarget
? { groupId: groupTarget, name: pending.file.name, mimeType, contentBase64 }
: { botId: botTarget!, name: pending.file.name, mimeType, contentBase64 },
const artifact = await uploadAttachment(
groupTarget ? { groupId: groupTarget } : { botId: botTarget! },
pending.file,
mimeType,
);
artifactIds.push(artifact.id);
}
@ -7186,6 +7188,8 @@ function MentionChipIcon({ mention }: { mention: ComposerMention }) {
);
}
const ATTACHMENT_BLOCK_KINDS = new Set(["image", "file", "video", "audio", "media"]);
function previewMessageText(message: ThreadMessage): string {
const text = message.blocks
.map((block) => (block.kind === "text" || block.kind === "channel_message" ? block.text : ""))
@ -7193,7 +7197,7 @@ function previewMessageText(message: ThreadMessage): string {
.join(" ")
.trim();
if (text) return text;
if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) {
if (message.blocks.some((block) => ATTACHMENT_BLOCK_KINDS.has(block.kind))) {
return t`Attachment`;
}
return t`Message`;
@ -7841,10 +7845,23 @@ const MessageView = memo(function MessageView({
key={i}
className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`}
>
<ArtifactImage
target={artifactTarget}
<ArtifactImage artifactId={block.artifactId} name={block.name} />
</div>
);
}
if (block.kind === "video" || block.kind === "audio") {
return (
<div
key={i}
className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`}
>
<MediaPlayer
artifactId={block.artifactId}
kind={block.kind}
name={block.name}
posterArtifactId={block.kind === "video" ? block.posterArtifactId : undefined}
width={block.kind === "video" ? block.width : undefined}
height={block.kind === "video" ? block.height : undefined}
/>
</div>
);
@ -7856,7 +7873,6 @@ const MessageView = memo(function MessageView({
className={`flex ${message.role === "user" ? "justify-end" : "justify-start"}`}
>
<ArtifactFileCard
target={artifactTarget}
artifactId={block.artifactId}
name={block.name}
mimeType={block.mimeType}
@ -8059,16 +8075,3 @@ function newClientNonce(): string {
}
return `m-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
}
function readFileAsBase64(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const result = typeof reader.result === "string" ? reader.result : "";
const base64 = result.includes(",") ? (result.split(",")[1] ?? "") : result;
resolve(base64);
};
reader.onerror = () => reject(reader.error ?? new Error("Failed to read file"));
reader.readAsDataURL(file);
});
}

View File

@ -1,11 +1,18 @@
import { Trans, useLingui } from "@lingui/react/macro";
import type { MessageBlock } from "@rakazo/contracts";
import { abortableDelay } from "@rakazo/core";
import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web";
import { X } from "lucide-react";
import {
Button,
buttonVariants,
Dialog,
DialogClose,
DialogContent,
DialogTitle,
} from "@rakazo/ui-web";
import { Download, X } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { BuiCard, SuccessPop } from "../../components/ai/primitives";
import { type ArtifactTarget, decodeArtifactBase64 } from "../../lib/artifact-open";
import { artifactUrl } from "../../lib/artifact-url";
import { chartViewport } from "../../lib/chart-viewport";
import { connectMcpOauth } from "../../lib/mcp-connect";
import { rpc } from "../../lib/rpc";
@ -461,104 +468,65 @@ export function ChartBlockView({
);
}
export function ArtifactImage({
target,
artifactId,
name,
}: {
target: ArtifactTarget;
artifactId: string;
name: string;
}) {
export function ArtifactImage({ artifactId, name }: { artifactId: string; name: string }) {
const { t } = useLingui();
const [src, setSrc] = useState<string | null>(null);
const [open, setOpen] = useState(false);
const [visible, setVisible] = useState(false);
const container = useRef<HTMLDivElement>(null);
const targetBotId = "botId" in target ? target.botId : undefined;
const targetGroupId = "groupId" in target ? target.groupId : undefined;
const [failed, setFailed] = useState(false);
const src = artifactUrl(artifactId);
useEffect(() => {
const element = container.current;
if (!element || typeof IntersectionObserver === "undefined") {
setVisible(true);
return;
}
const observer = new IntersectionObserver(
(entries) => {
if (entries.some((entry) => entry.isIntersecting)) {
setVisible(true);
observer.disconnect();
}
},
{ rootMargin: "320px" },
if (failed) {
return (
<div className="rounded-[20px] border border-border bg-muted px-4 py-3 text-ui-body text-muted-foreground">
{name}
</div>
);
observer.observe(element);
return () => observer.disconnect();
}, []);
useEffect(() => {
if (!visible) return;
let cancelled = false;
let objectUrl: string | null = null;
setSrc(null);
void rpc.artifacts
.get(
targetBotId
? { botId: targetBotId, artifactId }
: { groupId: targetGroupId ?? "", artifactId },
)
.then((artifact) => {
const bytes = decodeArtifactBase64(artifact.contentBase64);
objectUrl = URL.createObjectURL(
new Blob([new Uint8Array(bytes)], { type: artifact.mimeType }),
);
if (cancelled) URL.revokeObjectURL(objectUrl);
else setSrc(objectUrl);
})
.catch(() => undefined);
return () => {
cancelled = true;
if (objectUrl) URL.revokeObjectURL(objectUrl);
};
}, [artifactId, targetBotId, targetGroupId, visible]);
}
return (
<div ref={container}>
{src ? (
<button
type="button"
onClick={() => setOpen(true)}
className="max-w-[240px] overflow-hidden rounded-[20px]"
<div>
<button
type="button"
onClick={() => setOpen(true)}
className="max-w-[240px] overflow-hidden rounded-[20px]"
>
<img
src={src}
alt={name}
loading="lazy"
decoding="async"
onError={() => setFailed(true)}
className="max-h-48 w-full object-cover"
/>
</button>
<Dialog open={open} onOpenChange={setOpen}>
<DialogContent
showCloseButton={false}
className="w-auto max-w-none bg-transparent p-0 ring-0 sm:max-w-none"
>
<img src={src} alt={name} className="max-h-48 w-full object-cover" />
</button>
) : (
<div className="rounded-[20px] border border-border bg-muted px-4 py-3 text-ui-body text-muted-foreground">
{name}
</div>
)}
{src ? (
<Dialog open={open} onOpenChange={setOpen}>
<DialogContent
showCloseButton={false}
className="w-auto max-w-none bg-transparent p-0 ring-0 sm:max-w-none"
>
<DialogTitle className="sr-only">{name}</DialogTitle>
<DialogTitle className="sr-only">{name}</DialogTitle>
<div className="absolute end-2 top-2 flex gap-1">
<a
href={artifactUrl(artifactId, { download: true })}
download={name}
aria-label={t`Download ${name}`}
title={t`Download ${name}`}
className={buttonVariants({ variant: "ghost", size: "icon-sm" })}
>
<Download />
</a>
<DialogClose
aria-label={t`Close image preview`}
render={<Button variant="ghost" size="icon-sm" className="absolute end-2 top-2" />}
render={<Button variant="ghost" size="icon-sm" />}
>
<X />
</DialogClose>
<img
src={src}
alt={name}
className="max-h-[85vh] max-w-[90vw] rounded-xl object-contain"
/>
</DialogContent>
</Dialog>
) : null}
</div>
<img
src={src}
alt={name}
className="max-h-[85vh] max-w-[90vw] rounded-xl object-contain"
/>
</DialogContent>
</Dialog>
</div>
);
}