feat(media): title transcription cards with what they transcribe

A card that only says "Transcript" does not tell two transcriptions apart,
and it is English whatever the user's language.

- transcribe takes an optional title like the video operations, so a bot
  names the card in the user's language; "Transcript" stays the default.
- Transcribe on a media card passes the source file name as the title.
- The E2E seeds the card through bootstrap as well as threads/get, since the
  bot thread's first snapshot arrives with bootstrap.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude 2026-10-03 14:18:56 +00:00
parent a61aa02a8c
commit f9ffa5a961
No known key found for this signature in database
5 changed files with 47 additions and 33 deletions

View File

@ -11,6 +11,7 @@ import {
type MediaBlock = Record<string, unknown> & { kind: "media" }; type MediaBlock = Record<string, unknown> & { kind: "media" };
type Snapshot = { type Snapshot = {
botId?: string;
threadId: string; threadId: string;
messages: Array<{ seq: number } & Record<string, unknown>>; messages: Array<{ seq: number } & Record<string, unknown>>;
}; };
@ -59,31 +60,36 @@ async function stillPngBase64(page: Page): Promise<string> {
/** /**
* Media jobs run on a GPU server or ffmpeg that CI does not have, so the card is seeded: real * Media jobs run on a GPU server or ffmpeg that CI does not have, so the card is seeded: real
* artifacts are uploaded, and the thread snapshot carries a bot message with the job's card. * artifacts are uploaded, and the bot thread's snapshot (from bootstrap on load, threads/get on
* refresh) carries a bot message with the job's card.
*/ */
async function seedMediaMessage(page: Page, botId: string, block: () => MediaBlock) { async function seedMediaMessage(page: Page, botId: string, block: () => MediaBlock) {
await page.route("**/rpc/threads/get", async (route) => { const seed = (snapshot: Snapshot | undefined) => {
const fetched = await fetchRpcEnvelope(route); if (snapshot?.botId !== botId) return;
const snapshot = fetched.body.json as (Snapshot & { botId?: string }) | undefined; const seq = Math.max(0, ...snapshot.messages.map((message) => message.seq)) + 1;
if (snapshot?.botId === botId) { snapshot.messages.push({
const seq = Math.max(0, ...snapshot.messages.map((message) => message.seq)) + 1; id: `media-message-${seq}`,
snapshot.messages.push({ threadId: snapshot.threadId,
id: `media-message-${seq}`, seq,
threadId: snapshot.threadId, role: "bot",
seq, botId,
role: "bot", blocks: [block()],
botId, createdAt: new Date().toISOString(),
blocks: [block()],
createdAt: new Date().toISOString(),
});
}
// Fresh headers: the original content length no longer matches the patched body.
await route.fulfill({
status: fetched.status,
contentType: "application/json",
body: JSON.stringify(fetched.body),
}); });
}); };
for (const procedure of ["bootstrap", "threads/get"]) {
await page.route(`**/rpc/${procedure}`, async (route) => {
const fetched = await fetchRpcEnvelope(route);
const json = fetched.body.json as (Snapshot & { thread?: Snapshot }) | undefined;
seed(procedure === "bootstrap" ? json?.thread : json);
// Fresh headers: the original content length no longer matches the patched body.
await route.fulfill({
status: fetched.status,
contentType: "application/json",
body: JSON.stringify(fetched.body),
});
});
}
} }
test("renders a media job card while it runs and when it is done", async ({ page }, testInfo) => { test("renders a media job card while it runs and when it is done", async ({ page }, testInfo) => {
@ -168,13 +174,7 @@ test("renders a media job card while it runs and when it is done", async ({ page
body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }), body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }),
}); });
}); });
const seeded = page.waitForResponse(
async (response) =>
response.url().includes("/rpc/threads/get") &&
(await response.text()).includes("media-job-1"),
);
await page.reload(); await page.reload();
await seeded;
const card = page.getByTestId("media-card"); const card = page.getByTestId("media-card");
await expect(card).toBeVisible({ timeout: 30_000 }); await expect(card).toBeVisible({ timeout: 30_000 });
@ -206,6 +206,6 @@ test("renders a media job card while it runs and when it is done", async ({ page
botId, botId,
messageId: expect.stringMatching(/^media-message-\d+$/), messageId: expect.stringMatching(/^media-message-\d+$/),
operation: "transcribe", operation: "transcribe",
input: { artifactId: voice.id }, input: { artifactId: voice.id, title: "voice-note.wav" },
}); });
}); });

View File

@ -32,7 +32,7 @@ const CARD_CLASS =
export type MediaCardAction = export type MediaCardAction =
| { kind: "export"; artifactId: string; preset: MediaPresetId } | { kind: "export"; artifactId: string; preset: MediaPresetId }
| { kind: "transcribe"; artifactId: string } | { kind: "transcribe"; artifactId: string; name: string }
| { kind: "cancel" }; | { kind: "cancel" };
/** Which follow-up jobs the space can run; actions it cannot run are not offered. */ /** Which follow-up jobs the space can run; actions it cannot run are not offered. */
@ -156,7 +156,9 @@ function OutputActions({
variant="outline" variant="outline"
size="sm" size="sm"
disabled={busy} disabled={busy}
onClick={() => onAction({ kind: "transcribe", artifactId: output.artifactId })} onClick={() =>
onAction({ kind: "transcribe", artifactId: output.artifactId, name: output.name })
}
> >
<Captions /> <Captions />
{t`Transcribe`} {t`Transcribe`}
@ -378,7 +380,11 @@ export function MediaCard({
operation: "export_video", operation: "export_video",
input: { artifact_id: action.artifactId, presets: [action.preset] }, input: { artifact_id: action.artifactId, presets: [action.preset] },
} }
: { operation: "transcribe", input: { artifactId: action.artifactId } }), : // The new card is titled with the file it transcribes.
{
operation: "transcribe",
input: { artifactId: action.artifactId, title: action.name },
}),
}); });
}; };
return <MediaCardView block={block} target={target} available={available} onAction={onAction} />; return <MediaCardView block={block} target={target} available={available} onAction={onAction} />;

View File

@ -201,6 +201,7 @@ describe("transcribe operation", () => {
const input = registration.inputSchema.parse({ artifactId: "audio-1", language: "de" }); const input = registration.inputSchema.parse({ artifactId: "audio-1", language: "de" });
expect(registration.artifactIds?.(input)).toEqual(["audio-1"]); expect(registration.artifactIds?.(input)).toEqual(["audio-1"]);
expect(registration.title(input)).toBe("Transcript"); expect(registration.title(input)).toBe("Transcript");
expect(registration.title({ ...input, title: "voice-note.wav" })).toBe("voice-note.wav");
expect( expect(
registration.inputSchema.safeParse({ artifactId: "audio-1", language: "German!" }).success, registration.inputSchema.safeParse({ artifactId: "audio-1", language: "German!" }).success,
).toBe(false); ).toBe(false);

View File

@ -146,7 +146,7 @@ export function createTranscriptionOperations(
{ {
operation: "transcribe", operation: "transcribe",
inputSchema: TranscribeInputSchema, inputSchema: TranscribeInputSchema,
title: () => "Transcript", title: (input) => input.title ?? "Transcript",
available: async () => Boolean((await deps.loadSettings()).transcription), available: async () => Boolean((await deps.loadSettings()).transcription),
artifactIds: (input) => [input.artifactId], artifactIds: (input) => [input.artifactId],
handler: { handler: {

View File

@ -9,5 +9,12 @@ export const TranscribeInputSchema = z.object({
.regex(/^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/, "Use a language code such as en or de") .regex(/^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/, "Use a language code such as en or de")
.optional() .optional()
.describe("Spoken language as a code such as en or de; detected when omitted."), .describe("Spoken language as a code such as en or de; detected when omitted."),
title: z
.string()
.trim()
.min(1)
.max(120)
.optional()
.describe("A few words in the user's language for the media card."),
}); });
export type TranscribeInput = z.infer<typeof TranscribeInputSchema>; export type TranscribeInput = z.infer<typeof TranscribeInputSchema>;