feat(web): start exports, transcriptions and cancels from the media card
The card's follow-ups now reach the media job RPCs, and it only offers what the space can run. - media.capabilities answers which operations the actor's space can run (booleans only). The card asks once per minute per space, and only when it shows finished video or audio, so other cards cost nothing. - Export and Transcribe call media.jobs.start with the card's message, so the same bot posts the new card; Cancel calls media.jobs.cancel. A refused start shows the server's message under the card. - New files use top-level import type. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
parent
114103579f
commit
e66ce42d73
34
apps/api/src/media-capabilities.test.ts
Normal file
34
apps/api/src/media-capabilities.test.ts
Normal file
@ -0,0 +1,34 @@
|
||||
import type { MediaCapabilities } from "@rakazo/adapter-kit";
|
||||
import type { Actor } from "@rakazo/contracts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { MediaRouteDeps } from "./media-jobs.js";
|
||||
import { getMediaCapabilities } from "./media-jobs.js";
|
||||
|
||||
const actor: Actor = {
|
||||
userId: "user-1",
|
||||
spaceId: "space-1",
|
||||
email: "user@rakazo.test",
|
||||
isDeploymentOwner: false,
|
||||
};
|
||||
|
||||
describe("media capabilities", () => {
|
||||
it("answers for the actor's space", async () => {
|
||||
const asked: string[] = [];
|
||||
const deps = {
|
||||
media: {
|
||||
capabilities: async (spaceId: string) => {
|
||||
asked.push(spaceId);
|
||||
return { transcribe: true, export_video: false } as MediaCapabilities;
|
||||
},
|
||||
},
|
||||
} as unknown as MediaRouteDeps;
|
||||
await expect(getMediaCapabilities(deps, actor)).resolves.toMatchObject({ transcribe: true });
|
||||
expect(asked).toEqual(["space-1"]);
|
||||
});
|
||||
|
||||
it("reports nothing available without a media runtime", async () => {
|
||||
const capabilities = await getMediaCapabilities({} as MediaRouteDeps, actor);
|
||||
expect(Object.values(capabilities)).toHaveLength(6);
|
||||
expect(Object.values(capabilities).every((value) => value === false)).toBe(true);
|
||||
});
|
||||
});
|
||||
@ -1,5 +1,5 @@
|
||||
import { ORPCError } from "@orpc/server";
|
||||
import type { MediaRuntime } from "@rakazo/adapter-kit";
|
||||
import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit";
|
||||
import {
|
||||
type EncryptedSecretStore,
|
||||
loadMediaSettings,
|
||||
@ -9,6 +9,7 @@ import {
|
||||
saveMediaSettings,
|
||||
} from "@rakazo/adapters";
|
||||
import type { Actor, MediaJobStartInput, MediaSettingsUpdate } from "@rakazo/contracts";
|
||||
import { MediaOperationSchema } from "@rakazo/contracts";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { resolveThreadTarget } from "./thread-target.js";
|
||||
|
||||
@ -92,6 +93,17 @@ export async function startMediaJob(deps: MediaRouteDeps, actor: Actor, input: M
|
||||
}
|
||||
}
|
||||
|
||||
/** Booleans only: which operations the actor's space can run, never how they are configured. */
|
||||
export async function getMediaCapabilities(
|
||||
deps: MediaRouteDeps,
|
||||
actor: Actor,
|
||||
): Promise<MediaCapabilities> {
|
||||
if (deps.media) return deps.media.capabilities(actor.spaceId);
|
||||
return Object.fromEntries(
|
||||
MediaOperationSchema.options.map((operation) => [operation, false]),
|
||||
) as MediaCapabilities;
|
||||
}
|
||||
|
||||
async function ownedJobId(deps: MediaRouteDeps, actor: Actor, jobId: string): Promise<string> {
|
||||
const job = await deps.prisma.mediaJob.findFirst({
|
||||
where: { id: jobId, spaceId: actor.spaceId, userId: actor.userId },
|
||||
|
||||
@ -169,6 +169,7 @@ import {
|
||||
import { buildMcpUpdateMaterial } from "./mcp-material.js";
|
||||
import {
|
||||
cancelMediaJob,
|
||||
getMediaCapabilities,
|
||||
getMediaJob,
|
||||
getMediaSettings,
|
||||
startMediaJob,
|
||||
@ -3500,6 +3501,9 @@ export function createRouter(deps: RouterDeps) {
|
||||
updateMediaSettings(deps, context.actor, input),
|
||||
),
|
||||
},
|
||||
capabilities: authed.media.capabilities.handler(({ context }) =>
|
||||
getMediaCapabilities(deps, context.actor),
|
||||
),
|
||||
jobs: {
|
||||
start: authed.media.jobs.start.handler(({ context, input }) =>
|
||||
startMediaJob(deps, context.actor, input),
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { Server } from "node:http";
|
||||
import { createServer } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { NoVoiceConfigured } from "@rakazo/adapters";
|
||||
import type { Actor } from "@rakazo/contracts";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { toVoiceStatus, transcribeVoice, type VoiceDeps } from "./voice.js";
|
||||
import type { VoiceDeps } from "./voice.js";
|
||||
import { toVoiceStatus, transcribeVoice } from "./voice.js";
|
||||
|
||||
const actor: Actor = {
|
||||
userId: "user-1",
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { expect, type Page, test } from "@playwright/test";
|
||||
import type { Page } from "@playwright/test";
|
||||
import { expect, test } from "@playwright/test";
|
||||
import {
|
||||
activeBotId,
|
||||
captureScreenshot,
|
||||
@ -143,8 +144,29 @@ test("renders a media job card while it runs and when it is done", async ({ page
|
||||
],
|
||||
},
|
||||
);
|
||||
page.on("console", (message) => {
|
||||
if (message.type() === "error") console.log(`browser: ${message.text()}`);
|
||||
// CI has no transcription server; the card asks what the space can run.
|
||||
await page.route("**/rpc/media/capabilities", (route) =>
|
||||
route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
json: {
|
||||
generate_image: false,
|
||||
generate_video: false,
|
||||
edit_image: false,
|
||||
edit_video: false,
|
||||
export_video: true,
|
||||
transcribe: true,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const started: unknown[] = [];
|
||||
await page.route("**/rpc/media/jobs/start", async (route) => {
|
||||
started.push((route.request().postDataJSON() as { json: unknown }).json);
|
||||
await route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }),
|
||||
});
|
||||
});
|
||||
const seeded = page.waitForResponse(
|
||||
async (response) =>
|
||||
@ -173,5 +195,17 @@ test("renders a media job card while it runs and when it is done", async ({ page
|
||||
"download",
|
||||
"voice-note.wav",
|
||||
);
|
||||
await expect(card.getByRole("button", { name: "Transcribe" })).toBeVisible();
|
||||
// Export applies to video only.
|
||||
await expect(card.getByRole("button", { name: "Export" })).toHaveCount(0);
|
||||
await captureScreenshot(page, testInfo, "media-card-done");
|
||||
|
||||
await card.getByRole("button", { name: "Transcribe" }).click();
|
||||
await expect.poll(() => started.length).toBe(1);
|
||||
expect(started[0]).toEqual({
|
||||
botId,
|
||||
messageId: expect.stringMatching(/^media-message-\d+$/),
|
||||
operation: "transcribe",
|
||||
input: { artifactId: voice.id },
|
||||
});
|
||||
});
|
||||
|
||||
@ -21,6 +21,7 @@ vi.mock("./MediaPlayer", () => ({
|
||||
<span data-player={kind} data-artifact={artifactId} />
|
||||
),
|
||||
}));
|
||||
vi.mock("../../lib/rpc", () => ({ rpc: {}, selectedSpaceId: () => null }));
|
||||
vi.mock("../../lib/artifact-url", () => ({
|
||||
artifactUrl: (artifactId: string, options?: { download?: boolean }) =>
|
||||
`/artifact/${artifactId}${options?.download ? "?download=1" : ""}`,
|
||||
|
||||
@ -1,14 +1,12 @@
|
||||
import { useLingui } from "@lingui/react/macro";
|
||||
import {
|
||||
MEDIA_JOB_ACTIVE_STATUSES,
|
||||
MEDIA_PRESETS,
|
||||
type MediaJobBlock,
|
||||
type MediaJobStatus,
|
||||
type MediaOperation,
|
||||
type MediaOutput,
|
||||
type MediaPresetId,
|
||||
MediaPresetIdSchema,
|
||||
import type {
|
||||
MediaJobBlock,
|
||||
MediaJobStatus,
|
||||
MediaOperation,
|
||||
MediaOutput,
|
||||
MediaPresetId,
|
||||
} from "@rakazo/contracts";
|
||||
import { MEDIA_JOB_ACTIVE_STATUSES, MEDIA_PRESETS, MediaPresetIdSchema } from "@rakazo/contracts";
|
||||
import {
|
||||
Button,
|
||||
buttonVariants,
|
||||
@ -18,19 +16,12 @@ import {
|
||||
DropdownMenuItem,
|
||||
DropdownMenuTrigger,
|
||||
} from "@rakazo/ui-web";
|
||||
import {
|
||||
Captions,
|
||||
ChevronDown,
|
||||
Download,
|
||||
Film,
|
||||
ImageIcon,
|
||||
type LucideIcon,
|
||||
Scissors,
|
||||
Sparkles,
|
||||
} from "lucide-react";
|
||||
import { useId, useState } from "react";
|
||||
import type { LucideIcon } from "lucide-react";
|
||||
import { Captions, ChevronDown, Download, Film, ImageIcon, Scissors, Sparkles } from "lucide-react";
|
||||
import { useEffect, useId, useState } from "react";
|
||||
import type { ArtifactTarget } from "../../lib/artifact-open";
|
||||
import { artifactUrl } from "../../lib/artifact-url";
|
||||
import { rpc, selectedSpaceId } from "../../lib/rpc";
|
||||
import { ArtifactImage } from "../../pages/shell/message-cards";
|
||||
import { ArtifactFileCard } from "../ArtifactFileCard";
|
||||
import { Shimmer } from "../ai/primitives";
|
||||
@ -313,7 +304,81 @@ export function MediaCardView({
|
||||
);
|
||||
}
|
||||
|
||||
/** The media card in a conversation, with its follow-up actions wired to the media job RPCs. */
|
||||
export function MediaCard({ block, target }: { block: MediaJobBlock; target: ArtifactTarget }) {
|
||||
return <MediaCardView block={block} target={target} />;
|
||||
const NOTHING_AVAILABLE: MediaCardAvailability = { export: false, transcribe: false };
|
||||
const CAPABILITIES_TTL_MS = 60_000;
|
||||
let capabilities:
|
||||
| { spaceId: string | null; at: number; value: Promise<MediaCardAvailability> }
|
||||
| undefined;
|
||||
|
||||
/** What the space can run, shared by every card on screen and refreshed after a minute. */
|
||||
function loadAvailability(): Promise<MediaCardAvailability> {
|
||||
const spaceId = selectedSpaceId();
|
||||
if (
|
||||
!capabilities ||
|
||||
capabilities.spaceId !== spaceId ||
|
||||
Date.now() - capabilities.at > CAPABILITIES_TTL_MS
|
||||
) {
|
||||
capabilities = {
|
||||
spaceId,
|
||||
at: Date.now(),
|
||||
value: rpc.media
|
||||
.capabilities()
|
||||
.then((value) => ({
|
||||
export: value.export_video === true,
|
||||
transcribe: value.transcribe === true,
|
||||
}))
|
||||
.catch(() => NOTHING_AVAILABLE),
|
||||
};
|
||||
}
|
||||
return capabilities.value;
|
||||
}
|
||||
|
||||
function useAvailability(enabled: boolean): MediaCardAvailability {
|
||||
const [available, setAvailable] = useState(NOTHING_AVAILABLE);
|
||||
useEffect(() => {
|
||||
if (!enabled) return;
|
||||
let current = true;
|
||||
void loadAvailability().then((value) => {
|
||||
if (current) setAvailable(value);
|
||||
});
|
||||
return () => {
|
||||
current = false;
|
||||
};
|
||||
}, [enabled]);
|
||||
return available;
|
||||
}
|
||||
|
||||
/** The media card in a conversation, with its follow-up actions wired to the media job RPCs. */
|
||||
export function MediaCard({
|
||||
block,
|
||||
target,
|
||||
messageId,
|
||||
}: {
|
||||
block: MediaJobBlock;
|
||||
target: ArtifactTarget;
|
||||
/** The message carrying the card; a follow-up job is posted by the same bot. */
|
||||
messageId: string;
|
||||
}) {
|
||||
// Follow-ups apply to finished video and audio only; other cards never ask.
|
||||
const available = useAvailability(
|
||||
block.status === "done" &&
|
||||
block.outputs.some((output) => output.kind === "video" || output.kind === "audio"),
|
||||
);
|
||||
const onAction = async (action: MediaCardAction) => {
|
||||
if (action.kind === "cancel") {
|
||||
await rpc.media.jobs.cancel({ jobId: block.jobId });
|
||||
return;
|
||||
}
|
||||
await rpc.media.jobs.start({
|
||||
...target,
|
||||
messageId,
|
||||
...(action.kind === "export"
|
||||
? {
|
||||
operation: "export_video",
|
||||
input: { artifact_id: action.artifactId, presets: [action.preset] },
|
||||
}
|
||||
: { operation: "transcribe", input: { artifactId: action.artifactId } }),
|
||||
});
|
||||
};
|
||||
return <MediaCardView block={block} target={target} available={available} onAction={onAction} />;
|
||||
}
|
||||
|
||||
@ -7961,8 +7961,9 @@ const MessageView = memo(function MessageView({
|
||||
);
|
||||
}
|
||||
if (block.kind === "cloud_agent") return <CloudAgentCard key={i} block={block} />;
|
||||
if (block.kind === "media")
|
||||
return <MediaCard key={i} block={block} target={artifactTarget} />;
|
||||
if (block.kind === "media") {
|
||||
return <MediaCard key={i} block={block} target={artifactTarget} messageId={message.id} />;
|
||||
}
|
||||
if (block.kind === "skill_draft") {
|
||||
return (
|
||||
<div key={i} className="flex justify-start">
|
||||
|
||||
@ -4,11 +4,13 @@ import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { AdapterContext, JobPublisher } from "@rakazo/adapter-kit";
|
||||
import type { MediaJobBlock, MediaSettings, MessageBlock } from "@rakazo/contracts";
|
||||
import { createDb, type PrismaClient } from "@rakazo/db";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { createDb } from "@rakazo/db";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { storeArtifactFromFile } from "../artifact-files.js";
|
||||
import { LocalArtifactStore } from "../artifacts.js";
|
||||
import { createMediaRuntime, type MediaOperationRegistration } from "../media-runtime.js";
|
||||
import type { MediaOperationRegistration } from "../media-runtime.js";
|
||||
import { createMediaRuntime } from "../media-runtime.js";
|
||||
import { createOpenAiCompatibleTranscriptionProvider } from "../openai-compatible-transcription.js";
|
||||
import { OpenAiCompatibleTranscriptionServerEmulator } from "../transcription-emulator.js";
|
||||
import { createTranscriptionOperations } from "./transcription.js";
|
||||
|
||||
@ -1,12 +1,8 @@
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
type MediaJobContext,
|
||||
MediaJobDeferredError,
|
||||
type MediaJobInputFile,
|
||||
type MediaJobOutputFile,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit";
|
||||
import { MediaJobDeferredError } from "@rakazo/adapter-kit";
|
||||
import type { MediaOutput, MediaSettings } from "@rakazo/contracts";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createOpenAiCompatibleTranscriptionProvider } from "../openai-compatible-transcription.js";
|
||||
@ -14,11 +10,11 @@ import {
|
||||
EmulatorTranscriptionProvider,
|
||||
OpenAiCompatibleTranscriptionServerEmulator,
|
||||
} from "../transcription-emulator.js";
|
||||
import type { TranscriptionAudioTools } from "./transcription.js";
|
||||
import {
|
||||
createTranscriptionOperations,
|
||||
planAudioChunks,
|
||||
TRANSCRIPT_SUMMARY_MAX_CHARS,
|
||||
type TranscriptionAudioTools,
|
||||
transcribeChunks,
|
||||
transcriptionChunkSeconds,
|
||||
transcriptSummary,
|
||||
|
||||
@ -1,19 +1,12 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
MediaJobDeferredError,
|
||||
type Transcript,
|
||||
type TranscriptionProvider,
|
||||
type TranscriptSegment,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import {
|
||||
type MediaSettings,
|
||||
type TranscribeInput,
|
||||
TranscribeInputSchema,
|
||||
type TranscriptionSettings,
|
||||
} from "@rakazo/contracts";
|
||||
import type { Transcript, TranscriptionProvider, TranscriptSegment } from "@rakazo/adapter-kit";
|
||||
import { MediaJobDeferredError } from "@rakazo/adapter-kit";
|
||||
import type { MediaSettings, TranscribeInput, TranscriptionSettings } from "@rakazo/contracts";
|
||||
import { TranscribeInputSchema } from "@rakazo/contracts";
|
||||
import { toSrt, toVtt } from "@rakazo/core";
|
||||
import { MediaJobRejectedError, type MediaOperationRegistration } from "../media-runtime.js";
|
||||
import type { MediaOperationRegistration } from "../media-runtime.js";
|
||||
import { MediaJobRejectedError } from "../media-runtime.js";
|
||||
import {
|
||||
createOpenAiCompatibleTranscriptionProvider,
|
||||
TranscriptionServerUnavailableError,
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit";
|
||||
import { ensureStudioBot, type PrismaClient } from "@rakazo/db";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { ensureStudioBot } from "@rakazo/db";
|
||||
import { getLogger } from "@rakazo/logging";
|
||||
|
||||
/**
|
||||
|
||||
@ -34,3 +34,6 @@ export const MediaJobSnapshotSchema = z.object({
|
||||
export type MediaJobSnapshotDto = z.infer<typeof MediaJobSnapshotSchema>;
|
||||
|
||||
export const MediaJobStartOutputSchema = z.object({ jobId: Id, messageId: Id });
|
||||
|
||||
/** Which media operations the actor's space can run now, so cards offer only those. */
|
||||
export const MediaCapabilitiesSchema = z.record(MediaOperationSchema, z.boolean());
|
||||
|
||||
@ -83,6 +83,7 @@ import {
|
||||
IntegrationSetupStateSchema,
|
||||
} from "./integration-settings.js";
|
||||
import {
|
||||
MediaCapabilitiesSchema,
|
||||
MediaJobSnapshotSchema,
|
||||
MediaJobStartInputSchema,
|
||||
MediaJobStartOutputSchema,
|
||||
@ -702,6 +703,7 @@ export const appContract = {
|
||||
get: oc.output(MediaSettingsViewSchema),
|
||||
update: oc.input(MediaSettingsUpdateSchema).output(MediaSettingsViewSchema),
|
||||
},
|
||||
capabilities: oc.output(MediaCapabilitiesSchema),
|
||||
jobs: {
|
||||
start: oc.input(MediaJobStartInputSchema).output(MediaJobStartOutputSchema),
|
||||
get: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema),
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { bootstrapUserSpace } from "./bootstrap-user.js";
|
||||
import { createDb, type PrismaClient } from "./client.js";
|
||||
import type { PrismaClient } from "./client.js";
|
||||
import { createDb } from "./client.js";
|
||||
import { ensureMainBot } from "./main-agent.js";
|
||||
import {
|
||||
ensureStudioBot,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user