diff --git a/apps/web/e2e/media-card.spec.ts b/apps/web/e2e/media-card.spec.ts new file mode 100644 index 0000000..e777374 --- /dev/null +++ b/apps/web/e2e/media-card.spec.ts @@ -0,0 +1,164 @@ +import { expect, type Page, test } from "@playwright/test"; +import { + activeBotId, + captureScreenshot, + completeOnboarding, + fetchRpcEnvelope, + fulfillFetchedRpc, + rpc, + signup, +} from "./helpers"; + +type MediaBlock = Record & { kind: "media" }; +type Snapshot = { + threadId: string; + messages: Array<{ seq: number } & Record>; +}; + +/** One second of a quiet 440 Hz tone as 8 kHz mono 16-bit WAV, small enough to inline. */ +function toneWavBase64(): string { + const rate = 8000; + const samples = rate; + const buffer = Buffer.alloc(44 + samples * 2); + buffer.write("RIFF", 0); + buffer.writeUInt32LE(36 + samples * 2, 4); + buffer.write("WAVEfmt ", 8); + buffer.writeUInt32LE(16, 16); + buffer.writeUInt16LE(1, 20); + buffer.writeUInt16LE(1, 22); + buffer.writeUInt32LE(rate, 24); + buffer.writeUInt32LE(rate * 2, 28); + buffer.writeUInt16LE(2, 32); + buffer.writeUInt16LE(16, 34); + buffer.write("data", 36); + buffer.writeUInt32LE(samples * 2, 40); + for (let index = 0; index < samples; index += 1) { + buffer.writeInt16LE( + Math.round(Math.sin((2 * Math.PI * 440 * index) / rate) * 3000), + 44 + index * 2, + ); + } + return buffer.toString("base64"); +} + +/** A small grey gradient still, drawn by the browser so no binary fixture is needed. */ +async function stillPngBase64(page: Page): Promise { + return page.evaluate(() => { + const canvas = document.createElement("canvas"); + canvas.width = 320; + canvas.height = 180; + const context = canvas.getContext("2d")!; + const gradient = context.createLinearGradient(0, 0, 320, 180); + gradient.addColorStop(0, "rgb(40, 40, 40)"); + gradient.addColorStop(1, "rgb(200, 200, 200)"); + context.fillStyle = gradient; + context.fillRect(0, 0, 320, 180); + return canvas.toDataURL("image/png").split(",")[1]!; + }); +} + +/** + * Media jobs run on a GPU server or ffmpeg that CI does not have, so the card is seeded: real + * artifacts are uploaded, and the thread snapshot carries a bot message with the job's card. + */ +async function seedMediaMessage(page: Page, botId: string, block: () => MediaBlock) { + await page.route("**/rpc/threads/get", async (route) => { + const fetched = await fetchRpcEnvelope(route); + const snapshot = fetched.body.json as (Snapshot & { botId?: string }) | undefined; + if (snapshot?.botId === botId) { + const seq = Math.max(0, ...snapshot.messages.map((message) => message.seq)) + 1; + snapshot.messages.push({ + id: `media-message-${seq}`, + threadId: snapshot.threadId, + seq, + role: "bot", + botId, + blocks: [block()], + createdAt: new Date().toISOString(), + }); + } + await fulfillFetchedRpc(route, fetched); + }); +} + +test("renders a media job card while it runs and when it is done", async ({ page }, testInfo) => { + const stamp = Date.now(); + await signup(page, `media-card-${stamp}@rakazo.test`, "password12", "Media Card"); + await completeOnboarding(page); + const botId = activeBotId(page); + + const still = await rpc<{ id: string; size: number }>(page, "artifacts/create", { + botId, + name: "still.png", + mimeType: "image/png", + contentBase64: await stillPngBase64(page), + }); + const voice = await rpc<{ id: string; size: number }>(page, "artifacts/create", { + botId, + name: "voice-note.wav", + mimeType: "audio/wav", + contentBase64: toneWavBase64(), + }); + + let state: "running" | "done" = "running"; + await seedMediaMessage(page, botId, () => + state === "running" + ? { + kind: "media", + jobId: "media-job-1", + operation: "edit_video", + status: "running", + title: "Cut the pauses and reframe for 9:16", + progress: 0.4, + outputs: [], + } + : { + kind: "media", + jobId: "media-job-1", + operation: "edit_video", + status: "done", + title: "Cut the pauses and reframe for 9:16", + outputs: [ + { + artifactId: still.id, + kind: "image", + mimeType: "image/png", + name: "still.png", + size: still.size, + width: 320, + height: 180, + }, + { + artifactId: voice.id, + kind: "audio", + mimeType: "audio/wav", + name: "voice-note.wav", + size: voice.size, + durationMs: 1000, + }, + ], + }, + ); + await page.reload(); + + const card = page.getByTestId("media-card"); + await expect(card).toBeVisible({ timeout: 30_000 }); + await expect(card).toHaveAttribute("data-status", "running"); + await expect(card).toContainText("Cut the pauses and reframe for 9:16"); + await expect(card).toContainText("Working…"); + await expect(card.getByRole("progressbar")).toHaveAttribute("aria-valuenow", "40"); + await captureScreenshot(page, testInfo, "media-card-running"); + + state = "done"; + await page.reload(); + await expect(card).toHaveAttribute("data-status", "done", { timeout: 30_000 }); + await expect(card).toContainText("Done"); + await expect(card.getByRole("progressbar")).toHaveCount(0); + await expect(card.getByRole("img", { name: "still.png" })).toBeVisible(); + await expect(card.locator("audio")).toHaveCount(1); + await expect(card.getByRole("link", { name: "Download voice-note.wav" })).toHaveAttribute( + "download", + "voice-note.wav", + ); + await captureScreenshot(page, testInfo, "media-card-done"); +}); diff --git a/apps/web/src/components/media/MediaCard.test.tsx b/apps/web/src/components/media/MediaCard.test.tsx new file mode 100644 index 0000000..ff38de3 --- /dev/null +++ b/apps/web/src/components/media/MediaCard.test.tsx @@ -0,0 +1,193 @@ +import type { MediaJobBlock } from "@rakazo/contracts"; +import type { ReactNode } from "react"; +import { renderToString } from "react-dom/server"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@lingui/react/macro", () => { + const t = (parts: TemplateStringsArray, ...values: unknown[]) => + parts.reduce((text, part, index) => `${text}${part}${values[index] ?? ""}`, ""); + return { useLingui: () => ({ t }), Trans: ({ children }: { children: ReactNode }) => children }; +}); +vi.mock("../../pages/shell/message-cards", () => ({ + ArtifactImage: ({ artifactId, name }: { artifactId: string; name: string }) => ( + {name} + ), +})); +vi.mock("../ArtifactFileCard", () => ({ + ArtifactFileCard: ({ name }: { name: string }) => {name}, +})); +vi.mock("./MediaPlayer", () => ({ + MediaPlayer: ({ artifactId, kind }: { artifactId: string; kind: string }) => ( + + ), +})); +vi.mock("../../lib/artifact-url", () => ({ + artifactUrl: (artifactId: string) => `/artifact/${artifactId}`, +})); + +import { exportPresetsFor, isActiveMediaStatus, MediaCardView } from "./MediaCard"; + +const target = { groupId: "group-1" }; +const video = { + artifactId: "video-1", + kind: "video" as const, + mimeType: "video/mp4", + name: "clip.mp4", + size: 2048, + width: 1920, + height: 1080, +}; + +function block(overrides: Partial): MediaJobBlock { + return { + kind: "media", + jobId: "job-1", + operation: "edit_video", + status: "done", + title: "Cut the intro", + outputs: [], + ...overrides, + }; +} + +const everything = { export: true, transcribe: true }; +const noop = async () => undefined; + +describe("media card", () => { + it.each([ + ["queued", "Queued"], + ["waiting", "Waiting for GPU"], + ["running", "Working…"], + ["done", "Done"], + ["failed", "Failed"], + ["cancelled", "Cancelled"], + ] as const)("labels a %s job as %s", (status, label) => { + const html = renderToString(); + expect(html).toContain(`data-status="${status}"`); + expect(html).toContain(label); + expect(html).toContain("Cut the intro"); + }); + + it("shows progress and a cancel action while the job runs, and no result actions", () => { + const html = renderToString( + , + ); + expect(html).toContain('role="progressbar"'); + expect(html).toContain('aria-valuenow="42"'); + expect(html).toContain("width:42%"); + expect(html).toContain("Cancel"); + expect(html).not.toContain("Export"); + expect(html).not.toContain("Transcribe"); + }); + + it("leaves the progress value out when the backend does not report one", () => { + const html = renderToString( + , + ); + expect(html).toContain('role="progressbar"'); + expect(html).not.toContain("aria-valuenow"); + }); + + it("offers download, export and transcribe for a finished video", () => { + const html = renderToString( + , + ); + expect(html).toContain('data-player="video"'); + expect(html).toContain('href="/artifact/video-1"'); + expect(html).toContain('download="clip.mp4"'); + expect(html).toContain('aria-label="Download clip.mp4"'); + expect(html).toContain("Export"); + expect(html).toContain("Transcribe"); + expect(html).not.toContain("progressbar"); + expect(html).not.toContain("Cancel"); + }); + + it("offers only what the space can run and what fits the output", () => { + const audio = { ...video, artifactId: "audio-1", kind: "audio" as const, name: "take.m4a" }; + const onlyTranscribe = renderToString( + , + ); + expect(onlyTranscribe).toContain('data-player="audio"'); + expect(onlyTranscribe).toContain("Transcribe"); + expect(onlyTranscribe).not.toContain("Export"); + + const nothingAvailable = renderToString( + , + ); + expect(nothingAvailable).toContain("Download clip.mp4"); + expect(nothingAvailable).not.toContain("Export"); + expect(nothingAvailable).not.toContain("Transcribe"); + }); + + it("renders images as thumbnails and subtitle files as file cards", () => { + const html = renderToString( + , + ); + expect(html).toContain('data-artifact="image-1"'); + expect(html).toContain("clip.srt"); + expect(html).toContain("clip.vtt"); + expect(html).not.toContain("Transcribe"); + }); + + it("shows why a job failed", () => { + const html = renderToString( + , + ); + expect(html).toContain("GPU server is unreachable"); + }); +}); + +describe("exportPresetsFor", () => { + it("leaves out the shape the video already has, labelled by aspect ratio elsewhere", () => { + expect(exportPresetsFor({ width: 1920, height: 1080 })).toEqual([ + "vertical", + "square", + "portrait", + "classic", + "cinema", + ]); + expect(exportPresetsFor({ width: 1080, height: 1920, preset: "vertical" })).not.toContain( + "vertical", + ); + expect(exportPresetsFor({})).toHaveLength(6); + }); +}); + +describe("isActiveMediaStatus", () => { + it("treats queued, waiting and running as active", () => { + expect(["queued", "waiting", "running"].every((s) => isActiveMediaStatus(s as never))).toBe( + true, + ); + expect(isActiveMediaStatus("done")).toBe(false); + }); +}); diff --git a/apps/web/src/components/media/MediaCard.tsx b/apps/web/src/components/media/MediaCard.tsx new file mode 100644 index 0000000..e4ba52c --- /dev/null +++ b/apps/web/src/components/media/MediaCard.tsx @@ -0,0 +1,319 @@ +import { useLingui } from "@lingui/react/macro"; +import { + MEDIA_JOB_ACTIVE_STATUSES, + MEDIA_PRESETS, + type MediaJobBlock, + type MediaJobStatus, + type MediaOperation, + type MediaOutput, + type MediaPresetId, + MediaPresetIdSchema, +} from "@rakazo/contracts"; +import { + Button, + buttonVariants, + cn, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@rakazo/ui-web"; +import { + Captions, + ChevronDown, + Download, + Film, + ImageIcon, + type LucideIcon, + Scissors, + Sparkles, +} from "lucide-react"; +import { useId, useState } from "react"; +import type { ArtifactTarget } from "../../lib/artifact-open"; +import { artifactUrl } from "../../lib/artifact-url"; +import { ArtifactImage } from "../../pages/shell/message-cards"; +import { ArtifactFileCard } from "../ArtifactFileCard"; +import { Shimmer } from "../ai/primitives"; +import { MediaPlayer } from "./MediaPlayer"; + +const CARD_CLASS = + "w-[min(480px,96%)] rounded-[18px] border border-border bg-card/75 p-3.5 text-start shadow-xs"; + +export type MediaCardAction = + | { kind: "export"; artifactId: string; preset: MediaPresetId } + | { kind: "transcribe"; artifactId: string } + | { kind: "cancel" }; + +/** Which follow-up jobs the space can run; actions it cannot run are not offered. */ +export interface MediaCardAvailability { + export: boolean; + transcribe: boolean; +} + +const OPERATION_ICONS: Record = { + generate_image: Sparkles, + generate_video: Sparkles, + edit_image: ImageIcon, + edit_video: Scissors, + export_video: Film, + transcribe: Captions, +}; + +export function isActiveMediaStatus(status: MediaJobStatus): boolean { + return (MEDIA_JOB_ACTIVE_STATUSES as readonly MediaJobStatus[]).includes(status); +} + +/** Formats a video can be exported to: every preset except the shape it already has. */ +export function exportPresetsFor(output: Pick) { + const ratio = output.width && output.height ? output.width / output.height : null; + return MediaPresetIdSchema.options.filter((id) => { + if (id === output.preset) return false; + const preset = MEDIA_PRESETS[id]; + return ratio === null || Math.abs(preset.width / preset.height - ratio) > 0.01; + }); +} + +function StatusLabel({ status }: { status: MediaJobStatus }) { + const { t } = useLingui(); + if (status === "running") { + return ( + + {t`Working…`} + + ); + } + const label = + status === "queued" + ? t`Queued` + : status === "waiting" + ? t`Waiting for GPU` + : status === "done" + ? t`Done` + : status === "failed" + ? t`Failed` + : t`Cancelled`; + return ( + + {label} + + ); +} + +function OutputActions({ + output, + available, + busy, + onAction, +}: { + output: MediaOutput; + available: MediaCardAvailability; + busy: boolean; + onAction?: (action: MediaCardAction) => void; +}) { + const { t } = useLingui(); + const name = output.name; + const presets = output.kind === "video" ? exportPresetsFor(output) : []; + return ( +
+ + + + {onAction && available.export && presets.length > 0 ? ( + + } + > + + {t`Export`} + + + + {presets.map((preset) => ( + onAction({ kind: "export", artifactId: output.artifactId, preset })} + > + {MEDIA_PRESETS[preset].aspect} + + ))} + + + ) : null} + {onAction && available.transcribe && (output.kind === "video" || output.kind === "audio") ? ( + + ) : null} +
+ ); +} + +/** + * The live card of a media job: what it makes, how far it is, and the results with the + * follow-ups that fit them (export other formats, transcribe). Nothing else is shown. + */ +export function MediaCardView({ + block, + target, + available = { export: false, transcribe: false }, + onAction, +}: { + block: MediaJobBlock; + target: ArtifactTarget; + available?: MediaCardAvailability; + /** Starts a follow-up job or cancels this one; rejects with a message to show. */ + onAction?: (action: MediaCardAction) => Promise; +}) { + const { t } = useLingui(); + const titleId = useId(); + const [busy, setBusy] = useState(false); + const [actionError, setActionError] = useState(null); + const active = isActiveMediaStatus(block.status); + const Icon = OPERATION_ICONS[block.operation]; + const percent = Math.round((block.progress ?? 0) * 100); + const images = block.outputs.filter((output) => output.kind === "image"); + const others = block.outputs.filter((output) => output.kind !== "image"); + + const act = onAction + ? (action: MediaCardAction) => { + setBusy(true); + setActionError(null); + void onAction(action) + .catch((error: unknown) => + setActionError(error instanceof Error && error.message ? error.message : t`Failed`), + ) + .finally(() => setBusy(false)); + } + : undefined; + + return ( +
+
+
+ + + + + {active && act ? ( + + ) : null} + +
+ {active ? ( +
+
+
+ ) : null} + {block.status === "failed" && block.error ? ( +

+ {block.error} +

+ ) : null} + {images.length > 0 ? ( +
+ {images.map((output) => ( + + ))} +
+ ) : null} + {others.map((output) => ( +
+ {output.kind === "video" || output.kind === "audio" ? ( + <> + + {active ? null : ( + + )} + + ) : ( + + )} +
+ ))} + {actionError ? ( +

+ {actionError} +

+ ) : null} +
+
+ ); +} + +/** The media card in a conversation, with its follow-up actions wired to the media job RPCs. */ +export function MediaCard({ block, target }: { block: MediaJobBlock; target: ArtifactTarget }) { + return ; +} diff --git a/apps/web/src/components/media/MediaPlayer.tsx b/apps/web/src/components/media/MediaPlayer.tsx new file mode 100644 index 0000000..ade576e --- /dev/null +++ b/apps/web/src/components/media/MediaPlayer.tsx @@ -0,0 +1,33 @@ +// Stand-in for B1's media player; replaced by B1's component when its branch is merged. +import { artifactUrl } from "../../lib/artifact-url"; + +export function MediaPlayer({ + artifactId, + kind, + name, + posterArtifactId, +}: { + artifactId: string; + kind: "video" | "audio"; + mimeType: string; + name: string; + posterArtifactId?: string; + width?: number; + height?: number; +}) { + if (kind === "audio") { + // biome-ignore lint/a11y/useMediaCaption: generated media has no caption track yet. + return