feat(web): start exports, transcriptions and cancels from the media card

The card's follow-ups now reach the media job RPCs, and it only offers what
the space can run.

- media.capabilities answers which operations the actor's space can run
  (booleans only). The card asks once per minute per space, and only when it
  shows finished video or audio, so other cards cost nothing.
- Export and Transcribe call media.jobs.start with the card's message, so the
  same bot posts the new card; Cancel calls media.jobs.cancel. A refused
  start shows the server's message under the card.
- New files use top-level import type.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude 2026-10-03 14:00:34 +00:00
parent 114103579f
commit e66ce42d73
No known key found for this signature in database
15 changed files with 206 additions and 55 deletions

View File

@ -0,0 +1,34 @@
import type { MediaCapabilities } from "@rakazo/adapter-kit";
import type { Actor } from "@rakazo/contracts";
import { describe, expect, it } from "vitest";
import type { MediaRouteDeps } from "./media-jobs.js";
import { getMediaCapabilities } from "./media-jobs.js";
const actor: Actor = {
userId: "user-1",
spaceId: "space-1",
email: "user@rakazo.test",
isDeploymentOwner: false,
};
describe("media capabilities", () => {
it("answers for the actor's space", async () => {
const asked: string[] = [];
const deps = {
media: {
capabilities: async (spaceId: string) => {
asked.push(spaceId);
return { transcribe: true, export_video: false } as MediaCapabilities;
},
},
} as unknown as MediaRouteDeps;
await expect(getMediaCapabilities(deps, actor)).resolves.toMatchObject({ transcribe: true });
expect(asked).toEqual(["space-1"]);
});
it("reports nothing available without a media runtime", async () => {
const capabilities = await getMediaCapabilities({} as MediaRouteDeps, actor);
expect(Object.values(capabilities)).toHaveLength(6);
expect(Object.values(capabilities).every((value) => value === false)).toBe(true);
});
});

View File

@ -1,5 +1,5 @@
import { ORPCError } from "@orpc/server";
import type { MediaRuntime } from "@rakazo/adapter-kit";
import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit";
import {
type EncryptedSecretStore,
loadMediaSettings,
@ -9,6 +9,7 @@ import {
saveMediaSettings,
} from "@rakazo/adapters";
import type { Actor, MediaJobStartInput, MediaSettingsUpdate } from "@rakazo/contracts";
import { MediaOperationSchema } from "@rakazo/contracts";
import type { PrismaClient } from "@rakazo/db";
import { resolveThreadTarget } from "./thread-target.js";
@ -92,6 +93,17 @@ export async function startMediaJob(deps: MediaRouteDeps, actor: Actor, input: M
}
}
/** Booleans only: which operations the actor's space can run, never how they are configured. */
export async function getMediaCapabilities(
deps: MediaRouteDeps,
actor: Actor,
): Promise<MediaCapabilities> {
if (deps.media) return deps.media.capabilities(actor.spaceId);
return Object.fromEntries(
MediaOperationSchema.options.map((operation) => [operation, false]),
) as MediaCapabilities;
}
async function ownedJobId(deps: MediaRouteDeps, actor: Actor, jobId: string): Promise<string> {
const job = await deps.prisma.mediaJob.findFirst({
where: { id: jobId, spaceId: actor.spaceId, userId: actor.userId },

View File

@ -169,6 +169,7 @@ import {
import { buildMcpUpdateMaterial } from "./mcp-material.js";
import {
cancelMediaJob,
getMediaCapabilities,
getMediaJob,
getMediaSettings,
startMediaJob,
@ -3500,6 +3501,9 @@ export function createRouter(deps: RouterDeps) {
updateMediaSettings(deps, context.actor, input),
),
},
capabilities: authed.media.capabilities.handler(({ context }) =>
getMediaCapabilities(deps, context.actor),
),
jobs: {
start: authed.media.jobs.start.handler(({ context, input }) =>
startMediaJob(deps, context.actor, input),

View File

@ -1,9 +1,11 @@
import { createServer, type Server } from "node:http";
import type { Server } from "node:http";
import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import { NoVoiceConfigured } from "@rakazo/adapters";
import type { Actor } from "@rakazo/contracts";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { toVoiceStatus, transcribeVoice, type VoiceDeps } from "./voice.js";
import type { VoiceDeps } from "./voice.js";
import { toVoiceStatus, transcribeVoice } from "./voice.js";
const actor: Actor = {
userId: "user-1",

View File

@ -1,4 +1,5 @@
import { expect, type Page, test } from "@playwright/test";
import type { Page } from "@playwright/test";
import { expect, test } from "@playwright/test";
import {
activeBotId,
captureScreenshot,
@ -143,8 +144,29 @@ test("renders a media job card while it runs and when it is done", async ({ page
],
},
);
page.on("console", (message) => {
if (message.type() === "error") console.log(`browser: ${message.text()}`);
// CI has no transcription server; the card asks what the space can run.
await page.route("**/rpc/media/capabilities", (route) =>
route.fulfill({
contentType: "application/json",
body: JSON.stringify({
json: {
generate_image: false,
generate_video: false,
edit_image: false,
edit_video: false,
export_video: true,
transcribe: true,
},
}),
}),
);
const started: unknown[] = [];
await page.route("**/rpc/media/jobs/start", async (route) => {
started.push((route.request().postDataJSON() as { json: unknown }).json);
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }),
});
});
const seeded = page.waitForResponse(
async (response) =>
@ -173,5 +195,17 @@ test("renders a media job card while it runs and when it is done", async ({ page
"download",
"voice-note.wav",
);
await expect(card.getByRole("button", { name: "Transcribe" })).toBeVisible();
// Export applies to video only.
await expect(card.getByRole("button", { name: "Export" })).toHaveCount(0);
await captureScreenshot(page, testInfo, "media-card-done");
await card.getByRole("button", { name: "Transcribe" }).click();
await expect.poll(() => started.length).toBe(1);
expect(started[0]).toEqual({
botId,
messageId: expect.stringMatching(/^media-message-\d+$/),
operation: "transcribe",
input: { artifactId: voice.id },
});
});

View File

@ -21,6 +21,7 @@ vi.mock("./MediaPlayer", () => ({
<span data-player={kind} data-artifact={artifactId} />
),
}));
vi.mock("../../lib/rpc", () => ({ rpc: {}, selectedSpaceId: () => null }));
vi.mock("../../lib/artifact-url", () => ({
artifactUrl: (artifactId: string, options?: { download?: boolean }) =>
`/artifact/${artifactId}${options?.download ? "?download=1" : ""}`,

View File

@ -1,14 +1,12 @@
import { useLingui } from "@lingui/react/macro";
import {
MEDIA_JOB_ACTIVE_STATUSES,
MEDIA_PRESETS,
type MediaJobBlock,
type MediaJobStatus,
type MediaOperation,
type MediaOutput,
type MediaPresetId,
MediaPresetIdSchema,
import type {
MediaJobBlock,
MediaJobStatus,
MediaOperation,
MediaOutput,
MediaPresetId,
} from "@rakazo/contracts";
import { MEDIA_JOB_ACTIVE_STATUSES, MEDIA_PRESETS, MediaPresetIdSchema } from "@rakazo/contracts";
import {
Button,
buttonVariants,
@ -18,19 +16,12 @@ import {
DropdownMenuItem,
DropdownMenuTrigger,
} from "@rakazo/ui-web";
import {
Captions,
ChevronDown,
Download,
Film,
ImageIcon,
type LucideIcon,
Scissors,
Sparkles,
} from "lucide-react";
import { useId, useState } from "react";
import type { LucideIcon } from "lucide-react";
import { Captions, ChevronDown, Download, Film, ImageIcon, Scissors, Sparkles } from "lucide-react";
import { useEffect, useId, useState } from "react";
import type { ArtifactTarget } from "../../lib/artifact-open";
import { artifactUrl } from "../../lib/artifact-url";
import { rpc, selectedSpaceId } from "../../lib/rpc";
import { ArtifactImage } from "../../pages/shell/message-cards";
import { ArtifactFileCard } from "../ArtifactFileCard";
import { Shimmer } from "../ai/primitives";
@ -313,7 +304,81 @@ export function MediaCardView({
);
}
/** The media card in a conversation, with its follow-up actions wired to the media job RPCs. */
export function MediaCard({ block, target }: { block: MediaJobBlock; target: ArtifactTarget }) {
return <MediaCardView block={block} target={target} />;
const NOTHING_AVAILABLE: MediaCardAvailability = { export: false, transcribe: false };
const CAPABILITIES_TTL_MS = 60_000;
let capabilities:
| { spaceId: string | null; at: number; value: Promise<MediaCardAvailability> }
| undefined;
/** What the space can run, shared by every card on screen and refreshed after a minute. */
function loadAvailability(): Promise<MediaCardAvailability> {
const spaceId = selectedSpaceId();
if (
!capabilities ||
capabilities.spaceId !== spaceId ||
Date.now() - capabilities.at > CAPABILITIES_TTL_MS
) {
capabilities = {
spaceId,
at: Date.now(),
value: rpc.media
.capabilities()
.then((value) => ({
export: value.export_video === true,
transcribe: value.transcribe === true,
}))
.catch(() => NOTHING_AVAILABLE),
};
}
return capabilities.value;
}
function useAvailability(enabled: boolean): MediaCardAvailability {
const [available, setAvailable] = useState(NOTHING_AVAILABLE);
useEffect(() => {
if (!enabled) return;
let current = true;
void loadAvailability().then((value) => {
if (current) setAvailable(value);
});
return () => {
current = false;
};
}, [enabled]);
return available;
}
/** The media card in a conversation, with its follow-up actions wired to the media job RPCs. */
export function MediaCard({
block,
target,
messageId,
}: {
block: MediaJobBlock;
target: ArtifactTarget;
/** The message carrying the card; a follow-up job is posted by the same bot. */
messageId: string;
}) {
// Follow-ups apply to finished video and audio only; other cards never ask.
const available = useAvailability(
block.status === "done" &&
block.outputs.some((output) => output.kind === "video" || output.kind === "audio"),
);
const onAction = async (action: MediaCardAction) => {
if (action.kind === "cancel") {
await rpc.media.jobs.cancel({ jobId: block.jobId });
return;
}
await rpc.media.jobs.start({
...target,
messageId,
...(action.kind === "export"
? {
operation: "export_video",
input: { artifact_id: action.artifactId, presets: [action.preset] },
}
: { operation: "transcribe", input: { artifactId: action.artifactId } }),
});
};
return <MediaCardView block={block} target={target} available={available} onAction={onAction} />;
}

View File

@ -7961,8 +7961,9 @@ const MessageView = memo(function MessageView({
);
}
if (block.kind === "cloud_agent") return <CloudAgentCard key={i} block={block} />;
if (block.kind === "media")
return <MediaCard key={i} block={block} target={artifactTarget} />;
if (block.kind === "media") {
return <MediaCard key={i} block={block} target={artifactTarget} messageId={message.id} />;
}
if (block.kind === "skill_draft") {
return (
<div key={i} className="flex justify-start">

View File

@ -4,11 +4,13 @@ import { tmpdir } from "node:os";
import path from "node:path";
import type { AdapterContext, JobPublisher } from "@rakazo/adapter-kit";
import type { MediaJobBlock, MediaSettings, MessageBlock } from "@rakazo/contracts";
import { createDb, type PrismaClient } from "@rakazo/db";
import type { PrismaClient } from "@rakazo/db";
import { createDb } from "@rakazo/db";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { storeArtifactFromFile } from "../artifact-files.js";
import { LocalArtifactStore } from "../artifacts.js";
import { createMediaRuntime, type MediaOperationRegistration } from "../media-runtime.js";
import type { MediaOperationRegistration } from "../media-runtime.js";
import { createMediaRuntime } from "../media-runtime.js";
import { createOpenAiCompatibleTranscriptionProvider } from "../openai-compatible-transcription.js";
import { OpenAiCompatibleTranscriptionServerEmulator } from "../transcription-emulator.js";
import { createTranscriptionOperations } from "./transcription.js";

View File

@ -1,12 +1,8 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
type MediaJobContext,
MediaJobDeferredError,
type MediaJobInputFile,
type MediaJobOutputFile,
} from "@rakazo/adapter-kit";
import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit";
import { MediaJobDeferredError } from "@rakazo/adapter-kit";
import type { MediaOutput, MediaSettings } from "@rakazo/contracts";
import { afterEach, describe, expect, it } from "vitest";
import { createOpenAiCompatibleTranscriptionProvider } from "../openai-compatible-transcription.js";
@ -14,11 +10,11 @@ import {
EmulatorTranscriptionProvider,
OpenAiCompatibleTranscriptionServerEmulator,
} from "../transcription-emulator.js";
import type { TranscriptionAudioTools } from "./transcription.js";
import {
createTranscriptionOperations,
planAudioChunks,
TRANSCRIPT_SUMMARY_MAX_CHARS,
type TranscriptionAudioTools,
transcribeChunks,
transcriptionChunkSeconds,
transcriptSummary,

View File

@ -1,19 +1,12 @@
import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import {
MediaJobDeferredError,
type Transcript,
type TranscriptionProvider,
type TranscriptSegment,
} from "@rakazo/adapter-kit";
import {
type MediaSettings,
type TranscribeInput,
TranscribeInputSchema,
type TranscriptionSettings,
} from "@rakazo/contracts";
import type { Transcript, TranscriptionProvider, TranscriptSegment } from "@rakazo/adapter-kit";
import { MediaJobDeferredError } from "@rakazo/adapter-kit";
import type { MediaSettings, TranscribeInput, TranscriptionSettings } from "@rakazo/contracts";
import { TranscribeInputSchema } from "@rakazo/contracts";
import { toSrt, toVtt } from "@rakazo/core";
import { MediaJobRejectedError, type MediaOperationRegistration } from "../media-runtime.js";
import type { MediaOperationRegistration } from "../media-runtime.js";
import { MediaJobRejectedError } from "../media-runtime.js";
import {
createOpenAiCompatibleTranscriptionProvider,
TranscriptionServerUnavailableError,

View File

@ -1,5 +1,6 @@
import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit";
import { ensureStudioBot, type PrismaClient } from "@rakazo/db";
import type { PrismaClient } from "@rakazo/db";
import { ensureStudioBot } from "@rakazo/db";
import { getLogger } from "@rakazo/logging";
/**

View File

@ -34,3 +34,6 @@ export const MediaJobSnapshotSchema = z.object({
export type MediaJobSnapshotDto = z.infer<typeof MediaJobSnapshotSchema>;
export const MediaJobStartOutputSchema = z.object({ jobId: Id, messageId: Id });
/** Which media operations the actor's space can run now, so cards offer only those. */
export const MediaCapabilitiesSchema = z.record(MediaOperationSchema, z.boolean());

View File

@ -83,6 +83,7 @@ import {
IntegrationSetupStateSchema,
} from "./integration-settings.js";
import {
MediaCapabilitiesSchema,
MediaJobSnapshotSchema,
MediaJobStartInputSchema,
MediaJobStartOutputSchema,
@ -702,6 +703,7 @@ export const appContract = {
get: oc.output(MediaSettingsViewSchema),
update: oc.input(MediaSettingsUpdateSchema).output(MediaSettingsViewSchema),
},
capabilities: oc.output(MediaCapabilitiesSchema),
jobs: {
start: oc.input(MediaJobStartInputSchema).output(MediaJobStartOutputSchema),
get: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema),

View File

@ -1,7 +1,8 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { bootstrapUserSpace } from "./bootstrap-user.js";
import { createDb, type PrismaClient } from "./client.js";
import type { PrismaClient } from "./client.js";
import { createDb } from "./client.js";
import { ensureMainBot } from "./main-agent.js";
import {
ensureStudioBot,