fix(mobile): show video, audio and media cards as plain rows
Mobile has no media UI yet, so video, audio and media-card blocks rendered as empty bubbles or fell out of previews. - Thread text and reply previews describe video and audio files and the media card's title and status. - The bubble lists video, audio and a finished card's outputs as tappable rows that open the file like other attachments. - Opening video or audio downloads it from the artifact route to the cache instead of a base64 RPC, so long recordings open without the server holding them in memory. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
parent
4d75144cb8
commit
c008112ab4
@ -77,6 +77,7 @@ import {
|
|||||||
type MobileMessagePage,
|
type MobileMessagePage,
|
||||||
type MobileSnapshot,
|
type MobileSnapshot,
|
||||||
mergeMobileSnapshot,
|
mergeMobileSnapshot,
|
||||||
|
messageAttachments,
|
||||||
messagingProviderLabel,
|
messagingProviderLabel,
|
||||||
prependMobileMessagePage,
|
prependMobileMessagePage,
|
||||||
rpc,
|
rpc,
|
||||||
@ -2243,7 +2244,11 @@ function previewMessageText(message: MobileMessage): string {
|
|||||||
.join(" ")
|
.join(" ")
|
||||||
.trim();
|
.trim();
|
||||||
if (text) return text;
|
if (text) return text;
|
||||||
if (message.blocks.some((block) => block.kind === "image" || block.kind === "file")) {
|
if (
|
||||||
|
message.blocks.some((block) =>
|
||||||
|
["image", "file", "video", "audio", "media"].includes(block.kind),
|
||||||
|
)
|
||||||
|
) {
|
||||||
return t("Attachment");
|
return t("Attachment");
|
||||||
}
|
}
|
||||||
return t("Message");
|
return t("Message");
|
||||||
@ -2676,9 +2681,7 @@ const MessageBubble = memo(function MessageBubble({
|
|||||||
</View>
|
</View>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const attachments = message.blocks.filter(
|
const attachments = messageAttachments(message.blocks);
|
||||||
(block) => block.kind === "image" || block.kind === "file",
|
|
||||||
);
|
|
||||||
const caption = message.blocks
|
const caption = message.blocks
|
||||||
.flatMap((block) => {
|
.flatMap((block) => {
|
||||||
if (block.kind === "channel_message" && block.text) {
|
if (block.kind === "channel_message" && block.text) {
|
||||||
@ -2686,6 +2689,7 @@ const MessageBubble = memo(function MessageBubble({
|
|||||||
`${messagingProviderLabel(block.provider, block.transport)} · ${block.fromLabel}: ${block.text}`,
|
`${messagingProviderLabel(block.provider, block.transport)} · ${block.fromLabel}: ${block.text}`,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
if (block.kind === "media") return [block.title];
|
||||||
return block.kind === "text" && block.text ? [block.text] : [];
|
return block.kind === "text" && block.text ? [block.text] : [];
|
||||||
})
|
})
|
||||||
.join("\n");
|
.join("\n");
|
||||||
@ -2797,7 +2801,8 @@ const MessageBubble = memo(function MessageBubble({
|
|||||||
fontSize: 15,
|
fontSize: 15,
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
📎 {attachment.name ?? t("File")}
|
{attachment.kind === "video" ? "🎬" : attachment.kind === "audio" ? "🎵" : "📎"}{" "}
|
||||||
|
{attachment.name ?? t("File")}
|
||||||
</Text>
|
</Text>
|
||||||
{attachment.size ? (
|
{attachment.size ? (
|
||||||
<Text
|
<Text
|
||||||
|
|||||||
@ -16,6 +16,7 @@ import {
|
|||||||
MAX_MOBILE_AUTH_RESPONSE_BYTES,
|
MAX_MOBILE_AUTH_RESPONSE_BYTES,
|
||||||
MAX_MOBILE_RPC_RESPONSE_BYTES,
|
MAX_MOBILE_RPC_RESPONSE_BYTES,
|
||||||
mergeMobileSnapshot,
|
mergeMobileSnapshot,
|
||||||
|
messageAttachments,
|
||||||
passwordResetCapabilities,
|
passwordResetCapabilities,
|
||||||
prependMobileMessagePage,
|
prependMobileMessagePage,
|
||||||
requestPasswordReset,
|
requestPasswordReset,
|
||||||
@ -1857,6 +1858,51 @@ describe("mobile thread event reduction", () => {
|
|||||||
).toBe("Slack · Alex: Hello from the group");
|
).toBe("Slack · Alex: Hello from the group");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("falls back to plain text and file rows for video, audio and media cards", () => {
|
||||||
|
const blocks = [
|
||||||
|
{
|
||||||
|
kind: "video" as const,
|
||||||
|
artifactId: "vid-1",
|
||||||
|
mimeType: "video/mp4",
|
||||||
|
name: "clip.mp4",
|
||||||
|
size: 10,
|
||||||
|
width: 1080,
|
||||||
|
height: 1920,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
kind: "audio" as const,
|
||||||
|
artifactId: "aud-1",
|
||||||
|
mimeType: "audio/mpeg",
|
||||||
|
name: "memo.mp3",
|
||||||
|
size: 5,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
kind: "media" as const,
|
||||||
|
jobId: "job-1",
|
||||||
|
operation: "export_video" as const,
|
||||||
|
status: "done" as const,
|
||||||
|
title: "Vertical cut",
|
||||||
|
outputs: [
|
||||||
|
{
|
||||||
|
artifactId: "out-1",
|
||||||
|
kind: "video" as const,
|
||||||
|
mimeType: "video/mp4",
|
||||||
|
name: "cut.mp4",
|
||||||
|
size: 7,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
expect(blockText(mobileMessage("media-1", blocks))).toBe(
|
||||||
|
"[video: clip.mp4]\n[audio: memo.mp3]\nVertical cut: done",
|
||||||
|
);
|
||||||
|
expect(messageAttachments(blocks).map((item) => [item.kind, item.artifactId])).toEqual([
|
||||||
|
["video", "vid-1"],
|
||||||
|
["audio", "aud-1"],
|
||||||
|
["video", "out-1"],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("deduplicates durable messages and replaces matching transient subagent state", () => {
|
it("deduplicates durable messages and replaces matching transient subagent state", () => {
|
||||||
const initial = snapshot([
|
const initial = snapshot([
|
||||||
mobileMessage("message-1", [{ kind: "text", text: "old" }]),
|
mobileMessage("message-1", [{ kind: "text", text: "old" }]),
|
||||||
|
|||||||
@ -860,6 +860,24 @@ export function copyableMobileMessageText(message: MobileMessage): string {
|
|||||||
.trim();
|
.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type MobileAttachment = {
|
||||||
|
kind: "image" | "video" | "audio" | "file";
|
||||||
|
artifactId: string;
|
||||||
|
name: string;
|
||||||
|
mimeType: string;
|
||||||
|
size?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Files a message carries, including the outputs of a finished media card, as tappable rows. */
|
||||||
|
export function messageAttachments(blocks: readonly MessageBlock[]): MobileAttachment[] {
|
||||||
|
return blocks.flatMap((block): MobileAttachment[] => {
|
||||||
|
if (block.kind === "image") return [{ ...block, kind: "image" }];
|
||||||
|
if (block.kind === "video" || block.kind === "audio" || block.kind === "file") return [block];
|
||||||
|
if (block.kind === "media") return block.outputs;
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function blockText(message: MobileMessage) {
|
export function blockText(message: MobileMessage) {
|
||||||
return message.blocks
|
return message.blocks
|
||||||
.map((block) => {
|
.map((block) => {
|
||||||
@ -879,6 +897,10 @@ export function blockText(message: MobileMessage) {
|
|||||||
if (block.kind === "file") {
|
if (block.kind === "file") {
|
||||||
return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`;
|
return `[file: ${block.name ?? "attachment"}${block.size ? ` (${block.size} bytes)` : ""}]`;
|
||||||
}
|
}
|
||||||
|
if (block.kind === "video" || block.kind === "audio") {
|
||||||
|
return `[${block.kind}: ${block.name ?? "attachment"}]`;
|
||||||
|
}
|
||||||
|
if (block.kind === "media") return `${block.title}: ${block.status}`;
|
||||||
if (block.kind === "steps") {
|
if (block.kind === "steps") {
|
||||||
return (block.steps ?? [])
|
return (block.steps ?? [])
|
||||||
.map((step) => `${step.label}${step.count > 1 ? ` ×${step.count}` : ""}`)
|
.map((step) => `${step.label}${step.count > 1 ? ` ×${step.count}` : ""}`)
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
|
import { attachmentKindOf } from "@rakazo/contracts";
|
||||||
import { File, Paths } from "expo-file-system";
|
import { File, Paths } from "expo-file-system";
|
||||||
import * as Sharing from "expo-sharing";
|
import * as Sharing from "expo-sharing";
|
||||||
import { rpc } from "./api";
|
import { captureApiRequestContext, rpc } from "./api";
|
||||||
import { artifactCacheFileName } from "./artifact-file";
|
import { artifactCacheFileName } from "./artifact-file";
|
||||||
import { t } from "./i18n";
|
import { t } from "./i18n";
|
||||||
|
|
||||||
@ -11,6 +12,16 @@ async function cacheMobileArtifact(
|
|||||||
artifactId: string,
|
artifactId: string,
|
||||||
mimeType: string,
|
mimeType: string,
|
||||||
): Promise<File> {
|
): Promise<File> {
|
||||||
|
const kind = attachmentKindOf(mimeType);
|
||||||
|
if (kind === "video" || kind === "audio") {
|
||||||
|
// Media can be hundreds of megabytes: stream it to disk instead of a base64 RPC payload.
|
||||||
|
const { apiBase, headers } = await captureApiRequestContext();
|
||||||
|
return File.downloadFileAsync(
|
||||||
|
`${apiBase}/api/artifacts/${encodeURIComponent(artifactId)}`,
|
||||||
|
new File(Paths.cache, artifactCacheFileName(artifactId, mimeType)),
|
||||||
|
{ headers, idempotent: true },
|
||||||
|
);
|
||||||
|
}
|
||||||
const artifact = await rpc<{ contentBase64: string }>("artifacts/get", {
|
const artifact = await rpc<{ contentBase64: string }>("artifacts/get", {
|
||||||
...target,
|
...target,
|
||||||
artifactId,
|
artifactId,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user