From e66ce42d73271dcd41ed07d0e562d0937727f535 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 14:00:34 +0000 Subject: [PATCH] feat(web): start exports, transcriptions and cancels from the media card The card's follow-ups now reach the media job RPCs, and it only offers what the space can run. - media.capabilities answers which operations the actor's space can run (booleans only). The card asks once per minute per space, and only when it shows finished video or audio, so other cards cost nothing. - Export and Transcribe call media.jobs.start with the card's message, so the same bot posts the new card; Cancel calls media.jobs.cancel. A refused start shows the server's message under the card. - New files use top-level import type. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- apps/api/src/media-capabilities.test.ts | 34 ++++++ apps/api/src/media-jobs.ts | 14 ++- apps/api/src/router.ts | 4 + apps/api/src/voice-transcribe.test.ts | 6 +- apps/web/e2e/media-card.spec.ts | 40 ++++++- .../src/components/media/MediaCard.test.tsx | 1 + apps/web/src/components/media/MediaCard.tsx | 111 ++++++++++++++---- apps/web/src/pages/Shell.tsx | 5 +- .../src/media/transcription.postgres.test.ts | 6 +- .../adapters/src/media/transcription.test.ts | 10 +- packages/adapters/src/media/transcription.ts | 19 +-- packages/adapters/src/studio-specialist.ts | 3 +- packages/contracts/src/media-jobs.ts | 3 + packages/contracts/src/rpc.ts | 2 + packages/db/src/studio-agent.postgres.test.ts | 3 +- 15 files changed, 206 insertions(+), 55 deletions(-) create mode 100644 apps/api/src/media-capabilities.test.ts diff --git a/apps/api/src/media-capabilities.test.ts b/apps/api/src/media-capabilities.test.ts new file mode 100644 index 0000000..1da582d --- /dev/null +++ b/apps/api/src/media-capabilities.test.ts @@ -0,0 +1,34 @@ +import type { MediaCapabilities } from "@rakazo/adapter-kit"; +import type { Actor } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { MediaRouteDeps } from "./media-jobs.js"; +import { getMediaCapabilities } from "./media-jobs.js"; + +const actor: Actor = { + userId: "user-1", + spaceId: "space-1", + email: "user@rakazo.test", + isDeploymentOwner: false, +}; + +describe("media capabilities", () => { + it("answers for the actor's space", async () => { + const asked: string[] = []; + const deps = { + media: { + capabilities: async (spaceId: string) => { + asked.push(spaceId); + return { transcribe: true, export_video: false } as MediaCapabilities; + }, + }, + } as unknown as MediaRouteDeps; + await expect(getMediaCapabilities(deps, actor)).resolves.toMatchObject({ transcribe: true }); + expect(asked).toEqual(["space-1"]); + }); + + it("reports nothing available without a media runtime", async () => { + const capabilities = await getMediaCapabilities({} as MediaRouteDeps, actor); + expect(Object.values(capabilities)).toHaveLength(6); + expect(Object.values(capabilities).every((value) => value === false)).toBe(true); + }); +}); diff --git a/apps/api/src/media-jobs.ts b/apps/api/src/media-jobs.ts index 4a289e8..45dd6f3 100644 --- a/apps/api/src/media-jobs.ts +++ b/apps/api/src/media-jobs.ts @@ -1,5 +1,5 @@ import { ORPCError } from "@orpc/server"; -import type { MediaRuntime } from "@rakazo/adapter-kit"; +import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit"; import { type EncryptedSecretStore, loadMediaSettings, @@ -9,6 +9,7 @@ import { saveMediaSettings, } from "@rakazo/adapters"; import type { Actor, MediaJobStartInput, MediaSettingsUpdate } from "@rakazo/contracts"; +import { MediaOperationSchema } from "@rakazo/contracts"; import type { PrismaClient } from "@rakazo/db"; import { resolveThreadTarget } from "./thread-target.js"; @@ -92,6 +93,17 @@ export async function startMediaJob(deps: MediaRouteDeps, actor: Actor, input: M } } +/** Booleans only: which operations the actor's space can run, never how they are configured. */ +export async function getMediaCapabilities( + deps: MediaRouteDeps, + actor: Actor, +): Promise { + if (deps.media) return deps.media.capabilities(actor.spaceId); + return Object.fromEntries( + MediaOperationSchema.options.map((operation) => [operation, false]), + ) as MediaCapabilities; +} + async function ownedJobId(deps: MediaRouteDeps, actor: Actor, jobId: string): Promise { const job = await deps.prisma.mediaJob.findFirst({ where: { id: jobId, spaceId: actor.spaceId, userId: actor.userId }, diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index e1447e8..7f8562a 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -169,6 +169,7 @@ import { import { buildMcpUpdateMaterial } from "./mcp-material.js"; import { cancelMediaJob, + getMediaCapabilities, getMediaJob, getMediaSettings, startMediaJob, @@ -3500,6 +3501,9 @@ export function createRouter(deps: RouterDeps) { updateMediaSettings(deps, context.actor, input), ), }, + capabilities: authed.media.capabilities.handler(({ context }) => + getMediaCapabilities(deps, context.actor), + ), jobs: { start: authed.media.jobs.start.handler(({ context, input }) => startMediaJob(deps, context.actor, input), diff --git a/apps/api/src/voice-transcribe.test.ts b/apps/api/src/voice-transcribe.test.ts index a023d9e..19a1ee0 100644 --- a/apps/api/src/voice-transcribe.test.ts +++ b/apps/api/src/voice-transcribe.test.ts @@ -1,9 +1,11 @@ -import { createServer, type Server } from "node:http"; +import type { Server } from "node:http"; +import { createServer } from "node:http"; import type { AddressInfo } from "node:net"; import { NoVoiceConfigured } from "@rakazo/adapters"; import type { Actor } from "@rakazo/contracts"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { toVoiceStatus, transcribeVoice, type VoiceDeps } from "./voice.js"; +import type { VoiceDeps } from "./voice.js"; +import { toVoiceStatus, transcribeVoice } from "./voice.js"; const actor: Actor = { userId: "user-1", diff --git a/apps/web/e2e/media-card.spec.ts b/apps/web/e2e/media-card.spec.ts index 26de164..1db958f 100644 --- a/apps/web/e2e/media-card.spec.ts +++ b/apps/web/e2e/media-card.spec.ts @@ -1,4 +1,5 @@ -import { expect, type Page, test } from "@playwright/test"; +import type { Page } from "@playwright/test"; +import { expect, test } from "@playwright/test"; import { activeBotId, captureScreenshot, @@ -143,8 +144,29 @@ test("renders a media job card while it runs and when it is done", async ({ page ], }, ); - page.on("console", (message) => { - if (message.type() === "error") console.log(`browser: ${message.text()}`); + // CI has no transcription server; the card asks what the space can run. + await page.route("**/rpc/media/capabilities", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ + json: { + generate_image: false, + generate_video: false, + edit_image: false, + edit_video: false, + export_video: true, + transcribe: true, + }, + }), + }), + ); + const started: unknown[] = []; + await page.route("**/rpc/media/jobs/start", async (route) => { + started.push((route.request().postDataJSON() as { json: unknown }).json); + await route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }), + }); }); const seeded = page.waitForResponse( async (response) => @@ -173,5 +195,17 @@ test("renders a media job card while it runs and when it is done", async ({ page "download", "voice-note.wav", ); + await expect(card.getByRole("button", { name: "Transcribe" })).toBeVisible(); + // Export applies to video only. + await expect(card.getByRole("button", { name: "Export" })).toHaveCount(0); await captureScreenshot(page, testInfo, "media-card-done"); + + await card.getByRole("button", { name: "Transcribe" }).click(); + await expect.poll(() => started.length).toBe(1); + expect(started[0]).toEqual({ + botId, + messageId: expect.stringMatching(/^media-message-\d+$/), + operation: "transcribe", + input: { artifactId: voice.id }, + }); }); diff --git a/apps/web/src/components/media/MediaCard.test.tsx b/apps/web/src/components/media/MediaCard.test.tsx index 7e24923..4ebaca4 100644 --- a/apps/web/src/components/media/MediaCard.test.tsx +++ b/apps/web/src/components/media/MediaCard.test.tsx @@ -21,6 +21,7 @@ vi.mock("./MediaPlayer", () => ({ ), })); +vi.mock("../../lib/rpc", () => ({ rpc: {}, selectedSpaceId: () => null })); vi.mock("../../lib/artifact-url", () => ({ artifactUrl: (artifactId: string, options?: { download?: boolean }) => `/artifact/${artifactId}${options?.download ? "?download=1" : ""}`, diff --git a/apps/web/src/components/media/MediaCard.tsx b/apps/web/src/components/media/MediaCard.tsx index 6134a16..4231a37 100644 --- a/apps/web/src/components/media/MediaCard.tsx +++ b/apps/web/src/components/media/MediaCard.tsx @@ -1,14 +1,12 @@ import { useLingui } from "@lingui/react/macro"; -import { - MEDIA_JOB_ACTIVE_STATUSES, - MEDIA_PRESETS, - type MediaJobBlock, - type MediaJobStatus, - type MediaOperation, - type MediaOutput, - type MediaPresetId, - MediaPresetIdSchema, +import type { + MediaJobBlock, + MediaJobStatus, + MediaOperation, + MediaOutput, + MediaPresetId, } from "@rakazo/contracts"; +import { MEDIA_JOB_ACTIVE_STATUSES, MEDIA_PRESETS, MediaPresetIdSchema } from "@rakazo/contracts"; import { Button, buttonVariants, @@ -18,19 +16,12 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from "@rakazo/ui-web"; -import { - Captions, - ChevronDown, - Download, - Film, - ImageIcon, - type LucideIcon, - Scissors, - Sparkles, -} from "lucide-react"; -import { useId, useState } from "react"; +import type { LucideIcon } from "lucide-react"; +import { Captions, ChevronDown, Download, Film, ImageIcon, Scissors, Sparkles } from "lucide-react"; +import { useEffect, useId, useState } from "react"; import type { ArtifactTarget } from "../../lib/artifact-open"; import { artifactUrl } from "../../lib/artifact-url"; +import { rpc, selectedSpaceId } from "../../lib/rpc"; import { ArtifactImage } from "../../pages/shell/message-cards"; import { ArtifactFileCard } from "../ArtifactFileCard"; import { Shimmer } from "../ai/primitives"; @@ -313,7 +304,81 @@ export function MediaCardView({ ); } -/** The media card in a conversation, with its follow-up actions wired to the media job RPCs. */ -export function MediaCard({ block, target }: { block: MediaJobBlock; target: ArtifactTarget }) { - return ; +const NOTHING_AVAILABLE: MediaCardAvailability = { export: false, transcribe: false }; +const CAPABILITIES_TTL_MS = 60_000; +let capabilities: + | { spaceId: string | null; at: number; value: Promise } + | undefined; + +/** What the space can run, shared by every card on screen and refreshed after a minute. */ +function loadAvailability(): Promise { + const spaceId = selectedSpaceId(); + if ( + !capabilities || + capabilities.spaceId !== spaceId || + Date.now() - capabilities.at > CAPABILITIES_TTL_MS + ) { + capabilities = { + spaceId, + at: Date.now(), + value: rpc.media + .capabilities() + .then((value) => ({ + export: value.export_video === true, + transcribe: value.transcribe === true, + })) + .catch(() => NOTHING_AVAILABLE), + }; + } + return capabilities.value; +} + +function useAvailability(enabled: boolean): MediaCardAvailability { + const [available, setAvailable] = useState(NOTHING_AVAILABLE); + useEffect(() => { + if (!enabled) return; + let current = true; + void loadAvailability().then((value) => { + if (current) setAvailable(value); + }); + return () => { + current = false; + }; + }, [enabled]); + return available; +} + +/** The media card in a conversation, with its follow-up actions wired to the media job RPCs. */ +export function MediaCard({ + block, + target, + messageId, +}: { + block: MediaJobBlock; + target: ArtifactTarget; + /** The message carrying the card; a follow-up job is posted by the same bot. */ + messageId: string; +}) { + // Follow-ups apply to finished video and audio only; other cards never ask. + const available = useAvailability( + block.status === "done" && + block.outputs.some((output) => output.kind === "video" || output.kind === "audio"), + ); + const onAction = async (action: MediaCardAction) => { + if (action.kind === "cancel") { + await rpc.media.jobs.cancel({ jobId: block.jobId }); + return; + } + await rpc.media.jobs.start({ + ...target, + messageId, + ...(action.kind === "export" + ? { + operation: "export_video", + input: { artifact_id: action.artifactId, presets: [action.preset] }, + } + : { operation: "transcribe", input: { artifactId: action.artifactId } }), + }); + }; + return ; } diff --git a/apps/web/src/pages/Shell.tsx b/apps/web/src/pages/Shell.tsx index c27cb0e..a56d68f 100644 --- a/apps/web/src/pages/Shell.tsx +++ b/apps/web/src/pages/Shell.tsx @@ -7961,8 +7961,9 @@ const MessageView = memo(function MessageView({ ); } if (block.kind === "cloud_agent") return ; - if (block.kind === "media") - return ; + if (block.kind === "media") { + return ; + } if (block.kind === "skill_draft") { return (
diff --git a/packages/adapters/src/media/transcription.postgres.test.ts b/packages/adapters/src/media/transcription.postgres.test.ts index 2c28a68..fc31366 100644 --- a/packages/adapters/src/media/transcription.postgres.test.ts +++ b/packages/adapters/src/media/transcription.postgres.test.ts @@ -4,11 +4,13 @@ import { tmpdir } from "node:os"; import path from "node:path"; import type { AdapterContext, JobPublisher } from "@rakazo/adapter-kit"; import type { MediaJobBlock, MediaSettings, MessageBlock } from "@rakazo/contracts"; -import { createDb, type PrismaClient } from "@rakazo/db"; +import type { PrismaClient } from "@rakazo/db"; +import { createDb } from "@rakazo/db"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { storeArtifactFromFile } from "../artifact-files.js"; import { LocalArtifactStore } from "../artifacts.js"; -import { createMediaRuntime, type MediaOperationRegistration } from "../media-runtime.js"; +import type { MediaOperationRegistration } from "../media-runtime.js"; +import { createMediaRuntime } from "../media-runtime.js"; import { createOpenAiCompatibleTranscriptionProvider } from "../openai-compatible-transcription.js"; import { OpenAiCompatibleTranscriptionServerEmulator } from "../transcription-emulator.js"; import { createTranscriptionOperations } from "./transcription.js"; diff --git a/packages/adapters/src/media/transcription.test.ts b/packages/adapters/src/media/transcription.test.ts index 0fab810..158acff 100644 --- a/packages/adapters/src/media/transcription.test.ts +++ b/packages/adapters/src/media/transcription.test.ts @@ -1,12 +1,8 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { - type MediaJobContext, - MediaJobDeferredError, - type MediaJobInputFile, - type MediaJobOutputFile, -} from "@rakazo/adapter-kit"; +import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit"; +import { MediaJobDeferredError } from "@rakazo/adapter-kit"; import type { MediaOutput, MediaSettings } from "@rakazo/contracts"; import { afterEach, describe, expect, it } from "vitest"; import { createOpenAiCompatibleTranscriptionProvider } from "../openai-compatible-transcription.js"; @@ -14,11 +10,11 @@ import { EmulatorTranscriptionProvider, OpenAiCompatibleTranscriptionServerEmulator, } from "../transcription-emulator.js"; +import type { TranscriptionAudioTools } from "./transcription.js"; import { createTranscriptionOperations, planAudioChunks, TRANSCRIPT_SUMMARY_MAX_CHARS, - type TranscriptionAudioTools, transcribeChunks, transcriptionChunkSeconds, transcriptSummary, diff --git a/packages/adapters/src/media/transcription.ts b/packages/adapters/src/media/transcription.ts index 9e80ed2..023dbef 100644 --- a/packages/adapters/src/media/transcription.ts +++ b/packages/adapters/src/media/transcription.ts @@ -1,19 +1,12 @@ import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; -import { - MediaJobDeferredError, - type Transcript, - type TranscriptionProvider, - type TranscriptSegment, -} from "@rakazo/adapter-kit"; -import { - type MediaSettings, - type TranscribeInput, - TranscribeInputSchema, - type TranscriptionSettings, -} from "@rakazo/contracts"; +import type { Transcript, TranscriptionProvider, TranscriptSegment } from "@rakazo/adapter-kit"; +import { MediaJobDeferredError } from "@rakazo/adapter-kit"; +import type { MediaSettings, TranscribeInput, TranscriptionSettings } from "@rakazo/contracts"; +import { TranscribeInputSchema } from "@rakazo/contracts"; import { toSrt, toVtt } from "@rakazo/core"; -import { MediaJobRejectedError, type MediaOperationRegistration } from "../media-runtime.js"; +import type { MediaOperationRegistration } from "../media-runtime.js"; +import { MediaJobRejectedError } from "../media-runtime.js"; import { createOpenAiCompatibleTranscriptionProvider, TranscriptionServerUnavailableError, diff --git a/packages/adapters/src/studio-specialist.ts b/packages/adapters/src/studio-specialist.ts index 5d3eab6..0c08d71 100644 --- a/packages/adapters/src/studio-specialist.ts +++ b/packages/adapters/src/studio-specialist.ts @@ -1,5 +1,6 @@ import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit"; -import { ensureStudioBot, type PrismaClient } from "@rakazo/db"; +import type { PrismaClient } from "@rakazo/db"; +import { ensureStudioBot } from "@rakazo/db"; import { getLogger } from "@rakazo/logging"; /** diff --git a/packages/contracts/src/media-jobs.ts b/packages/contracts/src/media-jobs.ts index 477a6fe..fa16a30 100644 --- a/packages/contracts/src/media-jobs.ts +++ b/packages/contracts/src/media-jobs.ts @@ -34,3 +34,6 @@ export const MediaJobSnapshotSchema = z.object({ export type MediaJobSnapshotDto = z.infer; export const MediaJobStartOutputSchema = z.object({ jobId: Id, messageId: Id }); + +/** Which media operations the actor's space can run now, so cards offer only those. */ +export const MediaCapabilitiesSchema = z.record(MediaOperationSchema, z.boolean()); diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 30de55a..b294b44 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -83,6 +83,7 @@ import { IntegrationSetupStateSchema, } from "./integration-settings.js"; import { + MediaCapabilitiesSchema, MediaJobSnapshotSchema, MediaJobStartInputSchema, MediaJobStartOutputSchema, @@ -702,6 +703,7 @@ export const appContract = { get: oc.output(MediaSettingsViewSchema), update: oc.input(MediaSettingsUpdateSchema).output(MediaSettingsViewSchema), }, + capabilities: oc.output(MediaCapabilitiesSchema), jobs: { start: oc.input(MediaJobStartInputSchema).output(MediaJobStartOutputSchema), get: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema), diff --git a/packages/db/src/studio-agent.postgres.test.ts b/packages/db/src/studio-agent.postgres.test.ts index 540b213..5c25f4a 100644 --- a/packages/db/src/studio-agent.postgres.test.ts +++ b/packages/db/src/studio-agent.postgres.test.ts @@ -1,7 +1,8 @@ import { randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { bootstrapUserSpace } from "./bootstrap-user.js"; -import { createDb, type PrismaClient } from "./client.js"; +import type { PrismaClient } from "./client.js"; +import { createDb } from "./client.js"; import { ensureMainBot } from "./main-agent.js"; import { ensureStudioBot,