diff --git a/apps/api/src/media-capabilities.test.ts b/apps/api/src/media-capabilities.test.ts new file mode 100644 index 0000000..1da582d --- /dev/null +++ b/apps/api/src/media-capabilities.test.ts @@ -0,0 +1,34 @@ +import type { MediaCapabilities } from "@rakazo/adapter-kit"; +import type { Actor } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { MediaRouteDeps } from "./media-jobs.js"; +import { getMediaCapabilities } from "./media-jobs.js"; + +const actor: Actor = { + userId: "user-1", + spaceId: "space-1", + email: "user@rakazo.test", + isDeploymentOwner: false, +}; + +describe("media capabilities", () => { + it("answers for the actor's space", async () => { + const asked: string[] = []; + const deps = { + media: { + capabilities: async (spaceId: string) => { + asked.push(spaceId); + return { transcribe: true, export_video: false } as MediaCapabilities; + }, + }, + } as unknown as MediaRouteDeps; + await expect(getMediaCapabilities(deps, actor)).resolves.toMatchObject({ transcribe: true }); + expect(asked).toEqual(["space-1"]); + }); + + it("reports nothing available without a media runtime", async () => { + const capabilities = await getMediaCapabilities({} as MediaRouteDeps, actor); + expect(Object.values(capabilities)).toHaveLength(6); + expect(Object.values(capabilities).every((value) => value === false)).toBe(true); + }); +}); diff --git a/apps/api/src/media-jobs.ts b/apps/api/src/media-jobs.ts index 4a289e8..45dd6f3 100644 --- a/apps/api/src/media-jobs.ts +++ b/apps/api/src/media-jobs.ts @@ -1,5 +1,5 @@ import { ORPCError } from "@orpc/server"; -import type { MediaRuntime } from "@rakazo/adapter-kit"; +import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit"; import { type EncryptedSecretStore, loadMediaSettings, @@ -9,6 +9,7 @@ import { saveMediaSettings, } from "@rakazo/adapters"; import type { Actor, MediaJobStartInput, MediaSettingsUpdate } from "@rakazo/contracts"; +import { MediaOperationSchema } from "@rakazo/contracts"; import type { PrismaClient } from "@rakazo/db"; import { resolveThreadTarget } from "./thread-target.js"; @@ -92,6 +93,17 @@ export async function startMediaJob(deps: MediaRouteDeps, actor: Actor, input: M } } +/** Booleans only: which operations the actor's space can run, never how they are configured. */ +export async function getMediaCapabilities( + deps: MediaRouteDeps, + actor: Actor, +): Promise { + if (deps.media) return deps.media.capabilities(actor.spaceId); + return Object.fromEntries( + MediaOperationSchema.options.map((operation) => [operation, false]), + ) as MediaCapabilities; +} + async function ownedJobId(deps: MediaRouteDeps, actor: Actor, jobId: string): Promise { const job = await deps.prisma.mediaJob.findFirst({ where: { id: jobId, spaceId: actor.spaceId, userId: actor.userId }, diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index e1447e8..7f8562a 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -169,6 +169,7 @@ import { import { buildMcpUpdateMaterial } from "./mcp-material.js"; import { cancelMediaJob, + getMediaCapabilities, getMediaJob, getMediaSettings, startMediaJob, @@ -3500,6 +3501,9 @@ export function createRouter(deps: RouterDeps) { updateMediaSettings(deps, context.actor, input), ), }, + capabilities: authed.media.capabilities.handler(({ context }) => + getMediaCapabilities(deps, context.actor), + ), jobs: { start: authed.media.jobs.start.handler(({ context, input }) => startMediaJob(deps, context.actor, input), diff --git a/apps/api/src/voice-transcribe.test.ts b/apps/api/src/voice-transcribe.test.ts index a023d9e..19a1ee0 100644 --- a/apps/api/src/voice-transcribe.test.ts +++ b/apps/api/src/voice-transcribe.test.ts @@ -1,9 +1,11 @@ -import { createServer, type Server } from "node:http"; +import type { Server } from "node:http"; +import { createServer } from "node:http"; import type { AddressInfo } from "node:net"; import { NoVoiceConfigured } from "@rakazo/adapters"; import type { Actor } from "@rakazo/contracts"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { toVoiceStatus, transcribeVoice, type VoiceDeps } from "./voice.js"; +import type { VoiceDeps } from "./voice.js"; +import { toVoiceStatus, transcribeVoice } from "./voice.js"; const actor: Actor = { userId: "user-1", diff --git a/apps/web/e2e/media-card.spec.ts b/apps/web/e2e/media-card.spec.ts index 26de164..1db958f 100644 --- a/apps/web/e2e/media-card.spec.ts +++ b/apps/web/e2e/media-card.spec.ts @@ -1,4 +1,5 @@ -import { expect, type Page, test } from "@playwright/test"; +import type { Page } from "@playwright/test"; +import { expect, test } from "@playwright/test"; import { activeBotId, captureScreenshot, @@ -143,8 +144,29 @@ test("renders a media job card while it runs and when it is done", async ({ page ], }, ); - page.on("console", (message) => { - if (message.type() === "error") console.log(`browser: ${message.text()}`); + // CI has no transcription server; the card asks what the space can run. + await page.route("**/rpc/media/capabilities", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ + json: { + generate_image: false, + generate_video: false, + edit_image: false, + edit_video: false, + export_video: true, + transcribe: true, + }, + }), + }), + ); + const started: unknown[] = []; + await page.route("**/rpc/media/jobs/start", async (route) => { + started.push((route.request().postDataJSON() as { json: unknown }).json); + await route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }), + }); }); const seeded = page.waitForResponse( async (response) => @@ -173,5 +195,17 @@ test("renders a media job card while it runs and when it is done", async ({ page "download", "voice-note.wav", ); + await expect(card.getByRole("button", { name: "Transcribe" })).toBeVisible(); + // Export applies to video only. + await expect(card.getByRole("button", { name: "Export" })).toHaveCount(0); await captureScreenshot(page, testInfo, "media-card-done"); + + await card.getByRole("button", { name: "Transcribe" }).click(); + await expect.poll(() => started.length).toBe(1); + expect(started[0]).toEqual({ + botId, + messageId: expect.stringMatching(/^media-message-\d+$/), + operation: "transcribe", + input: { artifactId: voice.id }, + }); }); diff --git a/apps/web/src/components/media/MediaCard.test.tsx b/apps/web/src/components/media/MediaCard.test.tsx index 7e24923..4ebaca4 100644 --- a/apps/web/src/components/media/MediaCard.test.tsx +++ b/apps/web/src/components/media/MediaCard.test.tsx @@ -21,6 +21,7 @@ vi.mock("./MediaPlayer", () => ({ ), })); +vi.mock("../../lib/rpc", () => ({ rpc: {}, selectedSpaceId: () => null })); vi.mock("../../lib/artifact-url", () => ({ artifactUrl: (artifactId: string, options?: { download?: boolean }) => `/artifact/${artifactId}${options?.download ? "?download=1" : ""}`, diff --git a/apps/web/src/components/media/MediaCard.tsx b/apps/web/src/components/media/MediaCard.tsx index 6134a16..4231a37 100644 --- a/apps/web/src/components/media/MediaCard.tsx +++ b/apps/web/src/components/media/MediaCard.tsx @@ -1,14 +1,12 @@ import { useLingui } from "@lingui/react/macro"; -import { - MEDIA_JOB_ACTIVE_STATUSES, - MEDIA_PRESETS, - type MediaJobBlock, - type MediaJobStatus, - type MediaOperation, - type MediaOutput, - type MediaPresetId, - MediaPresetIdSchema, +import type { + MediaJobBlock, + MediaJobStatus, + MediaOperation, + MediaOutput, + MediaPresetId, } from "@rakazo/contracts"; +import { MEDIA_JOB_ACTIVE_STATUSES, MEDIA_PRESETS, MediaPresetIdSchema } from "@rakazo/contracts"; import { Button, buttonVariants, @@ -18,19 +16,12 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from "@rakazo/ui-web"; -import { - Captions, - ChevronDown, - Download, - Film, - ImageIcon, - type LucideIcon, - Scissors, - Sparkles, -} from "lucide-react"; -import { useId, useState } from "react"; +import type { LucideIcon } from "lucide-react"; +import { Captions, ChevronDown, Download, Film, ImageIcon, Scissors, Sparkles } from "lucide-react"; +import { useEffect, useId, useState } from "react"; import type { ArtifactTarget } from "../../lib/artifact-open"; import { artifactUrl } from "../../lib/artifact-url"; +import { rpc, selectedSpaceId } from "../../lib/rpc"; import { ArtifactImage } from "../../pages/shell/message-cards"; import { ArtifactFileCard } from "../ArtifactFileCard"; import { Shimmer } from "../ai/primitives"; @@ -313,7 +304,81 @@ export function MediaCardView({ ); } -/** The media card in a conversation, with its follow-up actions wired to the media job RPCs. */ -export function MediaCard({ block, target }: { block: MediaJobBlock; target: ArtifactTarget }) { - return ; +const NOTHING_AVAILABLE: MediaCardAvailability = { export: false, transcribe: false }; +const CAPABILITIES_TTL_MS = 60_000; +let capabilities: + | { spaceId: string | null; at: number; value: Promise } + | undefined; + +/** What the space can run, shared by every card on screen and refreshed after a minute. */ +function loadAvailability(): Promise { + const spaceId = selectedSpaceId(); + if ( + !capabilities || + capabilities.spaceId !== spaceId || + Date.now() - capabilities.at > CAPABILITIES_TTL_MS + ) { + capabilities = { + spaceId, + at: Date.now(), + value: rpc.media + .capabilities() + .then((value) => ({ + export: value.export_video === true, + transcribe: value.transcribe === true, + })) + .catch(() => NOTHING_AVAILABLE), + }; + } + return capabilities.value; +} + +function useAvailability(enabled: boolean): MediaCardAvailability { + const [available, setAvailable] = useState(NOTHING_AVAILABLE); + useEffect(() => { + if (!enabled) return; + let current = true; + void loadAvailability().then((value) => { + if (current) setAvailable(value); + }); + return () => { + current = false; + }; + }, [enabled]); + return available; +} + +/** The media card in a conversation, with its follow-up actions wired to the media job RPCs. */ +export function MediaCard({ + block, + target, + messageId, +}: { + block: MediaJobBlock; + target: ArtifactTarget; + /** The message carrying the card; a follow-up job is posted by the same bot. */ + messageId: string; +}) { + // Follow-ups apply to finished video and audio only; other cards never ask. + const available = useAvailability( + block.status === "done" && + block.outputs.some((output) => output.kind === "video" || output.kind === "audio"), + ); + const onAction = async (action: MediaCardAction) => { + if (action.kind === "cancel") { + await rpc.media.jobs.cancel({ jobId: block.jobId }); + return; + } + await rpc.media.jobs.start({ + ...target, + messageId, + ...(action.kind === "export" + ? { + operation: "export_video", + input: { artifact_id: action.artifactId, presets: [action.preset] }, + } + : { operation: "transcribe", input: { artifactId: action.artifactId } }), + }); + }; + return ; } diff --git a/apps/web/src/pages/Shell.tsx b/apps/web/src/pages/Shell.tsx index c27cb0e..a56d68f 100644 --- a/apps/web/src/pages/Shell.tsx +++ b/apps/web/src/pages/Shell.tsx @@ -7961,8 +7961,9 @@ const MessageView = memo(function MessageView({ ); } if (block.kind === "cloud_agent") return ; - if (block.kind === "media") - return ; + if (block.kind === "media") { + return ; + } if (block.kind === "skill_draft") { return (
diff --git a/packages/adapters/src/media/transcription.postgres.test.ts b/packages/adapters/src/media/transcription.postgres.test.ts index 2c28a68..fc31366 100644 --- a/packages/adapters/src/media/transcription.postgres.test.ts +++ b/packages/adapters/src/media/transcription.postgres.test.ts @@ -4,11 +4,13 @@ import { tmpdir } from "node:os"; import path from "node:path"; import type { AdapterContext, JobPublisher } from "@rakazo/adapter-kit"; import type { MediaJobBlock, MediaSettings, MessageBlock } from "@rakazo/contracts"; -import { createDb, type PrismaClient } from "@rakazo/db"; +import type { PrismaClient } from "@rakazo/db"; +import { createDb } from "@rakazo/db"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { storeArtifactFromFile } from "../artifact-files.js"; import { LocalArtifactStore } from "../artifacts.js"; -import { createMediaRuntime, type MediaOperationRegistration } from "../media-runtime.js"; +import type { MediaOperationRegistration } from "../media-runtime.js"; +import { createMediaRuntime } from "../media-runtime.js"; import { createOpenAiCompatibleTranscriptionProvider } from "../openai-compatible-transcription.js"; import { OpenAiCompatibleTranscriptionServerEmulator } from "../transcription-emulator.js"; import { createTranscriptionOperations } from "./transcription.js"; diff --git a/packages/adapters/src/media/transcription.test.ts b/packages/adapters/src/media/transcription.test.ts index 0fab810..158acff 100644 --- a/packages/adapters/src/media/transcription.test.ts +++ b/packages/adapters/src/media/transcription.test.ts @@ -1,12 +1,8 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { - type MediaJobContext, - MediaJobDeferredError, - type MediaJobInputFile, - type MediaJobOutputFile, -} from "@rakazo/adapter-kit"; +import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit"; +import { MediaJobDeferredError } from "@rakazo/adapter-kit"; import type { MediaOutput, MediaSettings } from "@rakazo/contracts"; import { afterEach, describe, expect, it } from "vitest"; import { createOpenAiCompatibleTranscriptionProvider } from "../openai-compatible-transcription.js"; @@ -14,11 +10,11 @@ import { EmulatorTranscriptionProvider, OpenAiCompatibleTranscriptionServerEmulator, } from "../transcription-emulator.js"; +import type { TranscriptionAudioTools } from "./transcription.js"; import { createTranscriptionOperations, planAudioChunks, TRANSCRIPT_SUMMARY_MAX_CHARS, - type TranscriptionAudioTools, transcribeChunks, transcriptionChunkSeconds, transcriptSummary, diff --git a/packages/adapters/src/media/transcription.ts b/packages/adapters/src/media/transcription.ts index 9e80ed2..023dbef 100644 --- a/packages/adapters/src/media/transcription.ts +++ b/packages/adapters/src/media/transcription.ts @@ -1,19 +1,12 @@ import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; -import { - MediaJobDeferredError, - type Transcript, - type TranscriptionProvider, - type TranscriptSegment, -} from "@rakazo/adapter-kit"; -import { - type MediaSettings, - type TranscribeInput, - TranscribeInputSchema, - type TranscriptionSettings, -} from "@rakazo/contracts"; +import type { Transcript, TranscriptionProvider, TranscriptSegment } from "@rakazo/adapter-kit"; +import { MediaJobDeferredError } from "@rakazo/adapter-kit"; +import type { MediaSettings, TranscribeInput, TranscriptionSettings } from "@rakazo/contracts"; +import { TranscribeInputSchema } from "@rakazo/contracts"; import { toSrt, toVtt } from "@rakazo/core"; -import { MediaJobRejectedError, type MediaOperationRegistration } from "../media-runtime.js"; +import type { MediaOperationRegistration } from "../media-runtime.js"; +import { MediaJobRejectedError } from "../media-runtime.js"; import { createOpenAiCompatibleTranscriptionProvider, TranscriptionServerUnavailableError, diff --git a/packages/adapters/src/studio-specialist.ts b/packages/adapters/src/studio-specialist.ts index 5d3eab6..0c08d71 100644 --- a/packages/adapters/src/studio-specialist.ts +++ b/packages/adapters/src/studio-specialist.ts @@ -1,5 +1,6 @@ import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit"; -import { ensureStudioBot, type PrismaClient } from "@rakazo/db"; +import type { PrismaClient } from "@rakazo/db"; +import { ensureStudioBot } from "@rakazo/db"; import { getLogger } from "@rakazo/logging"; /** diff --git a/packages/contracts/src/media-jobs.ts b/packages/contracts/src/media-jobs.ts index 477a6fe..fa16a30 100644 --- a/packages/contracts/src/media-jobs.ts +++ b/packages/contracts/src/media-jobs.ts @@ -34,3 +34,6 @@ export const MediaJobSnapshotSchema = z.object({ export type MediaJobSnapshotDto = z.infer; export const MediaJobStartOutputSchema = z.object({ jobId: Id, messageId: Id }); + +/** Which media operations the actor's space can run now, so cards offer only those. */ +export const MediaCapabilitiesSchema = z.record(MediaOperationSchema, z.boolean()); diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 30de55a..b294b44 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -83,6 +83,7 @@ import { IntegrationSetupStateSchema, } from "./integration-settings.js"; import { + MediaCapabilitiesSchema, MediaJobSnapshotSchema, MediaJobStartInputSchema, MediaJobStartOutputSchema, @@ -702,6 +703,7 @@ export const appContract = { get: oc.output(MediaSettingsViewSchema), update: oc.input(MediaSettingsUpdateSchema).output(MediaSettingsViewSchema), }, + capabilities: oc.output(MediaCapabilitiesSchema), jobs: { start: oc.input(MediaJobStartInputSchema).output(MediaJobStartOutputSchema), get: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema), diff --git a/packages/db/src/studio-agent.postgres.test.ts b/packages/db/src/studio-agent.postgres.test.ts index 540b213..5c25f4a 100644 --- a/packages/db/src/studio-agent.postgres.test.ts +++ b/packages/db/src/studio-agent.postgres.test.ts @@ -1,7 +1,8 @@ import { randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { bootstrapUserSpace } from "./bootstrap-user.js"; -import { createDb, type PrismaClient } from "./client.js"; +import type { PrismaClient } from "./client.js"; +import { createDb } from "./client.js"; import { ensureMainBot } from "./main-agent.js"; import { ensureStudioBot,