fix(mobile): show video, audio and media cards as plain rows

Mobile has no media UI yet, so video, audio and media-card blocks rendered
as empty bubbles or fell out of previews.

- Thread text and reply previews describe video and audio files and the
  media card's title and status.
- The bubble lists video, audio and a finished card's outputs as tappable
  rows that open the file like other attachments.
- Opening video or audio downloads it from the artifact route to the cache
  instead of a base64 RPC, so long recordings open without the server
  holding them in memory.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude 2026-10-03 13:53:55 +00:00
parent 4d75144cb8
commit c008112ab4
No known key found for this signature in database
4 changed files with 90 additions and 6 deletions

View File

@ -77,6 +77,7 @@ import {
type MobileMessagePage, type MobileMessagePage,
type MobileSnapshot, type MobileSnapshot,
mergeMobileSnapshot, mergeMobileSnapshot,
messageAttachments,
messagingProviderLabel, messagingProviderLabel,
prependMobileMessagePage, prependMobileMessagePage,
rpc, rpc,
@ -2243,7 +2244,11 @@ function previewMessageText(message: MobileMessage): string {
.join(" ") .join(" ")
.trim(); .trim();
if (text) return text; if (text) return text;
if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) { if (
message.blocks.some((block) =>
["image", "file", "video", "audio", "media"].includes(block.kind),
)
) {
return t("Attachment"); return t("Attachment");
} }
return t("Message"); return t("Message");
@ -2676,9 +2681,7 @@ const MessageBubble = memo(function MessageBubble({
</View> </View>
); );
} }
const attachments = message.blocks.filter( const attachments = messageAttachments(message.blocks);
(block) => block.kind === "image" || block.kind === "file",
);
const caption = message.blocks const caption = message.blocks
.flatMap((block) => { .flatMap((block) => {
if (block.kind === "channel_message" && block.text) { if (block.kind === "channel_message" && block.text) {
@ -2686,6 +2689,7 @@ const MessageBubble = memo(function MessageBubble({
`${messagingProviderLabel(block.provider, block.transport)} · ${block.fromLabel}: ${block.text}`, `${messagingProviderLabel(block.provider, block.transport)} · ${block.fromLabel}: ${block.text}`,
]; ];
} }
if (block.kind === "media") return [block.title];
return block.kind === "text" && block.text ? [block.text] : []; return block.kind === "text" && block.text ? [block.text] : [];
}) })
.join("\n"); .join("\n");
@ -2797,7 +2801,8 @@ const MessageBubble = memo(function MessageBubble({
fontSize: 15, fontSize: 15,
}} }}
> >
📎 {attachment.name ?? t("File")} {attachment.kind === "video" ? "🎬" : attachment.kind === "audio" ? "🎵" : "📎"}{" "}
{attachment.name ?? t("File")}
</Text> </Text>
{attachment.size ? ( {attachment.size ? (
<Text <Text

View File

@ -16,6 +16,7 @@ import {
MAX_MOBILE_AUTH_RESPONSE_BYTES, MAX_MOBILE_AUTH_RESPONSE_BYTES,
MAX_MOBILE_RPC_RESPONSE_BYTES, MAX_MOBILE_RPC_RESPONSE_BYTES,
mergeMobileSnapshot, mergeMobileSnapshot,
messageAttachments,
passwordResetCapabilities, passwordResetCapabilities,
prependMobileMessagePage, prependMobileMessagePage,
requestPasswordReset, requestPasswordReset,
@ -1857,6 +1858,51 @@ describe("mobile thread event reduction", () => {
).toBe("Slack · Alex: Hello from the group"); ).toBe("Slack · Alex: Hello from the group");
}); });
it("falls back to plain text and file rows for video, audio and media cards", () => {
const blocks = [
{
kind: "video" as const,
artifactId: "vid-1",
mimeType: "video/mp4",
name: "clip.mp4",
size: 10,
width: 1080,
height: 1920,
},
{
kind: "audio" as const,
artifactId: "aud-1",
mimeType: "audio/mpeg",
name: "memo.mp3",
size: 5,
},
{
kind: "media" as const,
jobId: "job-1",
operation: "export_video" as const,
status: "done" as const,
title: "Vertical cut",
outputs: [
{
artifactId: "out-1",
kind: "video" as const,
mimeType: "video/mp4",
name: "cut.mp4",
size: 7,
},
],
},
];
expect(blockText(mobileMessage("media-1", blocks))).toBe(
"[video: clip.mp4]\n[audio: memo.mp3]\nVertical cut: done",
);
expect(messageAttachments(blocks).map((item) => [item.kind, item.artifactId])).toEqual([
["video", "vid-1"],
["audio", "aud-1"],
["video", "out-1"],
]);
});
it("deduplicates durable messages and replaces matching transient subagent state", () => { it("deduplicates durable messages and replaces matching transient subagent state", () => {
const initial = snapshot([ const initial = snapshot([
mobileMessage("message-1", [{ kind: "text", text: "old" }]), mobileMessage("message-1", [{ kind: "text", text: "old" }]),

View File

@ -860,6 +860,24 @@ export function copyableMobileMessageText(message: MobileMessage): string {
.trim(); .trim();
} }
export type MobileAttachment = {
kind: "image" | "video" | "audio" | "file";
artifactId: string;
name: string;
mimeType: string;
size?: number;
};
/** Files a message carries, including the outputs of a finished media card, as tappable rows. */
export function messageAttachments(blocks: readonly MessageBlock[]): MobileAttachment[] {
return blocks.flatMap((block): MobileAttachment[] => {
if (block.kind === "image") return [{ ...block, kind: "image" }];
if (block.kind === "video" || block.kind === "audio" || block.kind === "file") return [block];
if (block.kind === "media") return block.outputs;
return [];
});
}
export function blockText(message: MobileMessage) { export function blockText(message: MobileMessage) {
return message.blocks return message.blocks
.map((block) => { .map((block) => {
@ -879,6 +897,10 @@ export function blockText(message: MobileMessage) {
if (block.kind === "file") { if (block.kind === "file") {
return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`; return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`;
} }
if (block.kind === "video" || block.kind === "audio") {
return `[${block.kind}: ${block.name ?? "attachment"}]`;
}
if (block.kind === "media") return `${block.title}: ${block.status}`;
if (block.kind === "steps") { if (block.kind === "steps") {
return (block.steps ?? []) return (block.steps ?? [])
.map((step) => `${step.label}${step.count > 1 ? ` ×${step.count}` : ""}`) .map((step) => `${step.label}${step.count > 1 ? ` ×${step.count}` : ""}`)

View File

@ -1,6 +1,7 @@
import { attachmentKindOf } from "@rakazo/contracts";
import { File, Paths } from "expo-file-system"; import { File, Paths } from "expo-file-system";
import * as Sharing from "expo-sharing"; import * as Sharing from "expo-sharing";
import { rpc } from "./api"; import { captureApiRequestContext, rpc } from "./api";
import { artifactCacheFileName } from "./artifact-file"; import { artifactCacheFileName } from "./artifact-file";
import { t } from "./i18n"; import { t } from "./i18n";
@ -11,6 +12,16 @@ async function cacheMobileArtifact(
artifactId: string, artifactId: string,
mimeType: string, mimeType: string,
): Promise<File> { ): Promise<File> {
const kind = attachmentKindOf(mimeType);
if (kind === "video" || kind === "audio") {
// Media can be hundreds of megabytes: stream it to disk instead of a base64 RPC payload.
const { apiBase, headers } = await captureApiRequestContext();
return File.downloadFileAsync(
`${apiBase}/api/artifacts/${encodeURIComponent(artifactId)}`,
new File(Paths.cache, artifactCacheFileName(artifactId, mimeType)),
{ headers, idempotent: true },
);
}
const artifact = await rpc<{ contentBase64: string }>("artifacts/get", { const artifact = await rpc<{ contentBase64: string }>("artifacts/get", {
...target, ...target,
artifactId, artifactId,