From c008112ab4875029ae434d60cb4dcb51d95cb818 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:53:55 +0000 Subject: [PATCH] fix(mobile): show video, audio and media cards as plain rows Mobile has no media UI yet, so video, audio and media-card blocks rendered as empty bubbles or fell out of previews. - Thread text and reply previews describe video and audio files and the media card's title and status. - The bubble lists video, audio and a finished card's outputs as tappable rows that open the file like other attachments. - Opening video or audio downloads it from the artifact route to the cache instead of a base64 RPC, so long recordings open without the server holding them in memory. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- apps/mobile/app/thread.tsx | 15 +++++++---- apps/mobile/lib/api.test.ts | 46 ++++++++++++++++++++++++++++++++ apps/mobile/lib/api.ts | 22 +++++++++++++++ apps/mobile/lib/artifact-open.ts | 13 ++++++++- 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/apps/mobile/app/thread.tsx b/apps/mobile/app/thread.tsx index c2eb734..c8ae7d4 100644 --- a/apps/mobile/app/thread.tsx +++ b/apps/mobile/app/thread.tsx @@ -77,6 +77,7 @@ import { type MobileMessagePage, type MobileSnapshot, mergeMobileSnapshot, + messageAttachments, messagingProviderLabel, prependMobileMessagePage, rpc, @@ -2243,7 +2244,11 @@ function previewMessageText(message: MobileMessage): string { .join(" ") .trim(); if (text) return text; - if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) { + if ( + message.blocks.some((block) => + ["image", "file", "video", "audio", "media"].includes(block.kind), + ) + ) { return t("Attachment"); } return t("Message"); @@ -2676,9 +2681,7 @@ const MessageBubble = memo(function MessageBubble({ ); } - const attachments = message.blocks.filter( - (block) => block.kind === "image" || block.kind === "file", - ); + const attachments = messageAttachments(message.blocks); const caption = message.blocks .flatMap((block) => { if (block.kind === "channel_message" && block.text) { @@ -2686,6 +2689,7 @@ const MessageBubble = memo(function MessageBubble({ `${messagingProviderLabel(block.provider, block.transport)} ยท ${block.fromLabel}: ${block.text}`, ]; } + if (block.kind === "media") return [block.title]; return block.kind === "text" && block.text ? [block.text] : []; }) .join("\n"); @@ -2797,7 +2801,8 @@ const MessageBubble = memo(function MessageBubble({ fontSize: 15, }} > - ๐Ÿ“Ž {attachment.name ?? t("File")} + {attachment.kind === "video" ? "๐ŸŽฌ" : attachment.kind === "audio" ? "๐ŸŽต" : "๐Ÿ“Ž"}{" "} + {attachment.name ?? t("File")} {attachment.size ? ( { ).toBe("Slack ยท Alex: Hello from the group"); }); + it("falls back to plain text and file rows for video, audio and media cards", () => { + const blocks = [ + { + kind: "video" as const, + artifactId: "vid-1", + mimeType: "video/mp4", + name: "clip.mp4", + size: 10, + width: 1080, + height: 1920, + }, + { + kind: "audio" as const, + artifactId: "aud-1", + mimeType: "audio/mpeg", + name: "memo.mp3", + size: 5, + }, + { + kind: "media" as const, + jobId: "job-1", + operation: "export_video" as const, + status: "done" as const, + title: "Vertical cut", + outputs: [ + { + artifactId: "out-1", + kind: "video" as const, + mimeType: "video/mp4", + name: "cut.mp4", + size: 7, + }, + ], + }, + ]; + expect(blockText(mobileMessage("media-1", blocks))).toBe( + "[video: clip.mp4]\n[audio: memo.mp3]\nVertical cut: done", + ); + expect(messageAttachments(blocks).map((item) => [item.kind, item.artifactId])).toEqual([ + ["video", "vid-1"], + ["audio", "aud-1"], + ["video", "out-1"], + ]); + }); + it("deduplicates durable messages and replaces matching transient subagent state", () => { const initial = snapshot([ mobileMessage("message-1", [{ kind: "text", text: "old" }]), diff --git a/apps/mobile/lib/api.ts b/apps/mobile/lib/api.ts index f328abc..ac39cf1 100644 --- a/apps/mobile/lib/api.ts +++ b/apps/mobile/lib/api.ts @@ -860,6 +860,24 @@ export function copyableMobileMessageText(message: MobileMessage): string { .trim(); } +export type MobileAttachment = { + kind: "image" | "video" | "audio" | "file"; + artifactId: string; + name: string; + mimeType: string; + size?: number; +}; + +/** Files a message carries, including the outputs of a finished media card, as tappable rows. */ +export function messageAttachments(blocks: readonly MessageBlock[]): MobileAttachment[] { + return blocks.flatMap((block): MobileAttachment[] => { + if (block.kind === "image") return [{ ...block, kind: "image" }]; + if (block.kind === "video" || block.kind === "audio" || block.kind === "file") return [block]; + if (block.kind === "media") return block.outputs; + return []; + }); +} + export function blockText(message: MobileMessage) { return message.blocks .map((block) => { @@ -879,6 +897,10 @@ export function blockText(message: MobileMessage) { if (block.kind === "file") { return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`; } + if (block.kind === "video" || block.kind === "audio") { + return `[${block.kind}: ${block.name ?? "attachment"}]`; + } + if (block.kind === "media") return `${block.title}: ${block.status}`; if (block.kind === "steps") { return (block.steps ?? []) .map((step) => `${step.label}${step.count > 1 ? ` ร—${step.count}` : ""}`) diff --git a/apps/mobile/lib/artifact-open.ts b/apps/mobile/lib/artifact-open.ts index 17f0e00..ab18d9a 100644 --- a/apps/mobile/lib/artifact-open.ts +++ b/apps/mobile/lib/artifact-open.ts @@ -1,6 +1,7 @@ +import { attachmentKindOf } from "@rakazo/contracts"; import { File, Paths } from "expo-file-system"; import * as Sharing from "expo-sharing"; -import { rpc } from "./api"; +import { captureApiRequestContext, rpc } from "./api"; import { artifactCacheFileName } from "./artifact-file"; import { t } from "./i18n"; @@ -11,6 +12,16 @@ async function cacheMobileArtifact( artifactId: string, mimeType: string, ): Promise { + const kind = attachmentKindOf(mimeType); + if (kind === "video" || kind === "audio") { + // Media can be hundreds of megabytes: stream it to disk instead of a base64 RPC payload. + const { apiBase, headers } = await captureApiRequestContext(); + return File.downloadFileAsync( + `${apiBase}/api/artifacts/${encodeURIComponent(artifactId)}`, + new File(Paths.cache, artifactCacheFileName(artifactId, mimeType)), + { headers, idempotent: true }, + ); + } const artifact = await rpc<{ contentBase64: string }>("artifacts/get", { ...target, artifactId,