feat(media): title transcription cards with what they transcribe
A card that only says "Transcript" does not tell two transcriptions apart, and it is English whatever the user's language. - transcribe takes an optional title like the video operations, so a bot names the card in the user's language; "Transcript" stays the default. - Transcribe on a media card passes the source file name as the title. - The E2E seeds the card through bootstrap as well as threads/get, since the bot thread's first snapshot arrives with bootstrap. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
parent
a61aa02a8c
commit
f9ffa5a961
@ -11,6 +11,7 @@ import {
|
|||||||
|
|
||||||
type MediaBlock = Record<string, unknown> & { kind: "media" };
|
type MediaBlock = Record<string, unknown> & { kind: "media" };
|
||||||
type Snapshot = {
|
type Snapshot = {
|
||||||
|
botId?: string;
|
||||||
threadId: string;
|
threadId: string;
|
||||||
messages: Array<{ seq: number } & Record<string, unknown>>;
|
messages: Array<{ seq: number } & Record<string, unknown>>;
|
||||||
};
|
};
|
||||||
@ -59,31 +60,36 @@ async function stillPngBase64(page: Page): Promise<string> {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Media jobs run on a GPU server or ffmpeg that CI does not have, so the card is seeded: real
|
* Media jobs run on a GPU server or ffmpeg that CI does not have, so the card is seeded: real
|
||||||
* artifacts are uploaded, and the thread snapshot carries a bot message with the job's card.
|
* artifacts are uploaded, and the bot thread's snapshot (from bootstrap on load, threads/get on
|
||||||
|
* refresh) carries a bot message with the job's card.
|
||||||
*/
|
*/
|
||||||
async function seedMediaMessage(page: Page, botId: string, block: () => MediaBlock) {
|
async function seedMediaMessage(page: Page, botId: string, block: () => MediaBlock) {
|
||||||
await page.route("**/rpc/threads/get", async (route) => {
|
const seed = (snapshot: Snapshot | undefined) => {
|
||||||
const fetched = await fetchRpcEnvelope(route);
|
if (snapshot?.botId !== botId) return;
|
||||||
const snapshot = fetched.body.json as (Snapshot & { botId?: string }) | undefined;
|
const seq = Math.max(0, ...snapshot.messages.map((message) => message.seq)) + 1;
|
||||||
if (snapshot?.botId === botId) {
|
snapshot.messages.push({
|
||||||
const seq = Math.max(0, ...snapshot.messages.map((message) => message.seq)) + 1;
|
id: `media-message-${seq}`,
|
||||||
snapshot.messages.push({
|
threadId: snapshot.threadId,
|
||||||
id: `media-message-${seq}`,
|
seq,
|
||||||
threadId: snapshot.threadId,
|
role: "bot",
|
||||||
seq,
|
botId,
|
||||||
role: "bot",
|
blocks: [block()],
|
||||||
botId,
|
createdAt: new Date().toISOString(),
|
||||||
blocks: [block()],
|
|
||||||
createdAt: new Date().toISOString(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
// Fresh headers: the original content length no longer matches the patched body.
|
|
||||||
await route.fulfill({
|
|
||||||
status: fetched.status,
|
|
||||||
contentType: "application/json",
|
|
||||||
body: JSON.stringify(fetched.body),
|
|
||||||
});
|
});
|
||||||
});
|
};
|
||||||
|
for (const procedure of ["bootstrap", "threads/get"]) {
|
||||||
|
await page.route(`**/rpc/${procedure}`, async (route) => {
|
||||||
|
const fetched = await fetchRpcEnvelope(route);
|
||||||
|
const json = fetched.body.json as (Snapshot & { thread?: Snapshot }) | undefined;
|
||||||
|
seed(procedure === "bootstrap" ? json?.thread : json);
|
||||||
|
// Fresh headers: the original content length no longer matches the patched body.
|
||||||
|
await route.fulfill({
|
||||||
|
status: fetched.status,
|
||||||
|
contentType: "application/json",
|
||||||
|
body: JSON.stringify(fetched.body),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
test("renders a media job card while it runs and when it is done", async ({ page }, testInfo) => {
|
test("renders a media job card while it runs and when it is done", async ({ page }, testInfo) => {
|
||||||
@ -168,13 +174,7 @@ test("renders a media job card while it runs and when it is done", async ({ page
|
|||||||
body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }),
|
body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }),
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
const seeded = page.waitForResponse(
|
|
||||||
async (response) =>
|
|
||||||
response.url().includes("/rpc/threads/get") &&
|
|
||||||
(await response.text()).includes("media-job-1"),
|
|
||||||
);
|
|
||||||
await page.reload();
|
await page.reload();
|
||||||
await seeded;
|
|
||||||
|
|
||||||
const card = page.getByTestId("media-card");
|
const card = page.getByTestId("media-card");
|
||||||
await expect(card).toBeVisible({ timeout: 30_000 });
|
await expect(card).toBeVisible({ timeout: 30_000 });
|
||||||
@ -206,6 +206,6 @@ test("renders a media job card while it runs and when it is done", async ({ page
|
|||||||
botId,
|
botId,
|
||||||
messageId: expect.stringMatching(/^media-message-\d+$/),
|
messageId: expect.stringMatching(/^media-message-\d+$/),
|
||||||
operation: "transcribe",
|
operation: "transcribe",
|
||||||
input: { artifactId: voice.id },
|
input: { artifactId: voice.id, title: "voice-note.wav" },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -32,7 +32,7 @@ const CARD_CLASS =
|
|||||||
|
|
||||||
export type MediaCardAction =
|
export type MediaCardAction =
|
||||||
| { kind: "export"; artifactId: string; preset: MediaPresetId }
|
| { kind: "export"; artifactId: string; preset: MediaPresetId }
|
||||||
| { kind: "transcribe"; artifactId: string }
|
| { kind: "transcribe"; artifactId: string; name: string }
|
||||||
| { kind: "cancel" };
|
| { kind: "cancel" };
|
||||||
|
|
||||||
/** Which follow-up jobs the space can run; actions it cannot run are not offered. */
|
/** Which follow-up jobs the space can run; actions it cannot run are not offered. */
|
||||||
@ -156,7 +156,9 @@ function OutputActions({
|
|||||||
variant="outline"
|
variant="outline"
|
||||||
size="sm"
|
size="sm"
|
||||||
disabled={busy}
|
disabled={busy}
|
||||||
onClick={() => onAction({ kind: "transcribe", artifactId: output.artifactId })}
|
onClick={() =>
|
||||||
|
onAction({ kind: "transcribe", artifactId: output.artifactId, name: output.name })
|
||||||
|
}
|
||||||
>
|
>
|
||||||
<Captions />
|
<Captions />
|
||||||
{t`Transcribe`}
|
{t`Transcribe`}
|
||||||
@ -378,7 +380,11 @@ export function MediaCard({
|
|||||||
operation: "export_video",
|
operation: "export_video",
|
||||||
input: { artifact_id: action.artifactId, presets: [action.preset] },
|
input: { artifact_id: action.artifactId, presets: [action.preset] },
|
||||||
}
|
}
|
||||||
: { operation: "transcribe", input: { artifactId: action.artifactId } }),
|
: // The new card is titled with the file it transcribes.
|
||||||
|
{
|
||||||
|
operation: "transcribe",
|
||||||
|
input: { artifactId: action.artifactId, title: action.name },
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
return <MediaCardView block={block} target={target} available={available} onAction={onAction} />;
|
return <MediaCardView block={block} target={target} available={available} onAction={onAction} />;
|
||||||
|
|||||||
@ -201,6 +201,7 @@ describe("transcribe operation", () => {
|
|||||||
const input = registration.inputSchema.parse({ artifactId: "audio-1", language: "de" });
|
const input = registration.inputSchema.parse({ artifactId: "audio-1", language: "de" });
|
||||||
expect(registration.artifactIds?.(input)).toEqual(["audio-1"]);
|
expect(registration.artifactIds?.(input)).toEqual(["audio-1"]);
|
||||||
expect(registration.title(input)).toBe("Transcript");
|
expect(registration.title(input)).toBe("Transcript");
|
||||||
|
expect(registration.title({ ...input, title: "voice-note.wav" })).toBe("voice-note.wav");
|
||||||
expect(
|
expect(
|
||||||
registration.inputSchema.safeParse({ artifactId: "audio-1", language: "German!" }).success,
|
registration.inputSchema.safeParse({ artifactId: "audio-1", language: "German!" }).success,
|
||||||
).toBe(false);
|
).toBe(false);
|
||||||
|
|||||||
@ -146,7 +146,7 @@ export function createTranscriptionOperations(
|
|||||||
{
|
{
|
||||||
operation: "transcribe",
|
operation: "transcribe",
|
||||||
inputSchema: TranscribeInputSchema,
|
inputSchema: TranscribeInputSchema,
|
||||||
title: () => "Transcript",
|
title: (input) => input.title ?? "Transcript",
|
||||||
available: async () => Boolean((await deps.loadSettings()).transcription),
|
available: async () => Boolean((await deps.loadSettings()).transcription),
|
||||||
artifactIds: (input) => [input.artifactId],
|
artifactIds: (input) => [input.artifactId],
|
||||||
handler: {
|
handler: {
|
||||||
|
|||||||
@ -9,5 +9,12 @@ export const TranscribeInputSchema = z.object({
|
|||||||
.regex(/^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/, "Use a language code such as en or de")
|
.regex(/^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/, "Use a language code such as en or de")
|
||||||
.optional()
|
.optional()
|
||||||
.describe("Spoken language as a code such as en or de; detected when omitted."),
|
.describe("Spoken language as a code such as en or de; detected when omitted."),
|
||||||
|
title: z
|
||||||
|
.string()
|
||||||
|
.trim()
|
||||||
|
.min(1)
|
||||||
|
.max(120)
|
||||||
|
.optional()
|
||||||
|
.describe("A few words in the user's language for the media card."),
|
||||||
});
|
});
|
||||||
export type TranscribeInput = z.infer<typeof TranscribeInputSchema>;
|
export type TranscribeInput = z.infer<typeof TranscribeInputSchema>;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user