feat(media): media tools, media RPCs and owner-only media settings API
Bots and users need to start and follow media jobs, and the deployment owner needs to point the deployment at their GPU and transcription servers. - Media tools are named like their operations. A run sees one only when the space can run that operation (MediaRuntime.capabilities); the per-bot allowlist still applies. applyTool hands every media tool to one dispatcher (media-tools.ts) that splits off continue_after, starts the job under the tool call's effect key (a replay returns the same job) and returns at once; refusals (invalid input, unknown artifact, unavailable) go back to the model as tool errors. - media.jobs.start starts an export, transcription or image edit from a card: the conversation is resolved like every thread procedure, the card's bot (or the group's lead) posts the new card, and the runtime refuses source artifacts outside the actor's space. media.jobs.get and media.jobs.cancel only see the actor's own jobs. - media.settings.get/update are deployment-owner only, never return keys and answer invalid URLs with a 400 that says what is wrong. - createApp accepts media operations as a test seam. - Tests: a Postgres suite over the RPCs (owner-only settings, encryption, key retention per server, URL policy, start/get/cancel isolation between users, card jobs end to end) and unit tests for tool exposure and dispatch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
parent
e0277985f9
commit
2796965256
@ -15,6 +15,7 @@ import type {
|
||||
ComposioProvider,
|
||||
ConnectorRegistry,
|
||||
DestinationEmulator,
|
||||
MediaOperationRegistration,
|
||||
RemoteConnectorDependencies,
|
||||
} from "@rakazo/adapters";
|
||||
import {
|
||||
@ -143,6 +144,8 @@ export async function createApp(
|
||||
email?: TransactionalEmailProvider;
|
||||
remoteConnectors?: RemoteConnectorDependencies;
|
||||
logger?: Logger;
|
||||
/** Test seam: media operations instead of the configured ones. */
|
||||
mediaOperations?: MediaOperationRegistration[];
|
||||
} = {},
|
||||
): Promise<AppHandles> {
|
||||
const {
|
||||
@ -155,6 +158,7 @@ export async function createApp(
|
||||
email: emailOverride,
|
||||
remoteConnectors,
|
||||
logger: loggerOverride,
|
||||
mediaOperations,
|
||||
...envOverrides
|
||||
} = overrides;
|
||||
const env = { ...loadEnv(process.env), ...envOverrides };
|
||||
@ -384,9 +388,9 @@ export async function createApp(
|
||||
events,
|
||||
jobs,
|
||||
artifacts,
|
||||
registrations: createMediaOperations({
|
||||
loadSettings: () => loadMediaSettings({ prisma, secrets }),
|
||||
}),
|
||||
registrations:
|
||||
mediaOperations ??
|
||||
createMediaOperations({ loadSettings: () => loadMediaSettings({ prisma, secrets }) }),
|
||||
});
|
||||
const executor = createRunExecutor({
|
||||
prisma,
|
||||
@ -468,6 +472,7 @@ export async function createApp(
|
||||
|
||||
const router = createRouter({
|
||||
cloudAgent,
|
||||
media,
|
||||
prisma,
|
||||
events,
|
||||
auth,
|
||||
|
||||
118
apps/api/src/media-jobs.ts
Normal file
118
apps/api/src/media-jobs.ts
Normal file
@ -0,0 +1,118 @@
|
||||
import { ORPCError } from "@orpc/server";
|
||||
import type { MediaRuntime } from "@rakazo/adapter-kit";
|
||||
import {
|
||||
type EncryptedSecretStore,
|
||||
loadMediaSettings,
|
||||
MediaJobRejectedError,
|
||||
MediaSettingsInvalidError,
|
||||
mediaSettingsView,
|
||||
saveMediaSettings,
|
||||
} from "@rakazo/adapters";
|
||||
import type { Actor, MediaJobStartInput, MediaSettingsUpdate } from "@rakazo/contracts";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { resolveThreadTarget } from "./thread-target.js";
|
||||
|
||||
export interface MediaRouteDeps {
|
||||
prisma: PrismaClient;
|
||||
secrets: EncryptedSecretStore;
|
||||
media?: MediaRuntime;
|
||||
}
|
||||
|
||||
/** Media backends run on the deployment owner's own server, so only they configure them. */
|
||||
function requireDeploymentOwner(actor: Actor) {
|
||||
if (!actor.isDeploymentOwner) throw new ORPCError("FORBIDDEN");
|
||||
}
|
||||
|
||||
function requireMedia(deps: MediaRouteDeps): MediaRuntime {
|
||||
if (!deps.media) throw new ORPCError("NOT_IMPLEMENTED");
|
||||
return deps.media;
|
||||
}
|
||||
|
||||
export async function getMediaSettings(deps: MediaRouteDeps, actor: Actor) {
|
||||
requireDeploymentOwner(actor);
|
||||
return mediaSettingsView(await loadMediaSettings(deps));
|
||||
}
|
||||
|
||||
export async function updateMediaSettings(
|
||||
deps: MediaRouteDeps,
|
||||
actor: Actor,
|
||||
input: MediaSettingsUpdate,
|
||||
) {
|
||||
requireDeploymentOwner(actor);
|
||||
try {
|
||||
return await saveMediaSettings(deps, input);
|
||||
} catch (error) {
|
||||
if (error instanceof MediaSettingsInvalidError) {
|
||||
throw new ORPCError("BAD_REQUEST", { message: error.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts an operation a user picked on a card (export, transcribe, edit) in the same conversation.
|
||||
* The thread is authorized like every thread procedure; the runtime checks that the operation's
|
||||
* source artifacts belong to the actor, and the job can only ever read the actor's artifacts.
|
||||
*/
|
||||
export async function startMediaJob(deps: MediaRouteDeps, actor: Actor, input: MediaJobStartInput) {
|
||||
const media = requireMedia(deps);
|
||||
const target = await resolveThreadTarget(deps.prisma, actor, input);
|
||||
const source = await deps.prisma.message.findFirst({
|
||||
where: { id: input.messageId, threadId: target.threadId },
|
||||
select: { botId: true },
|
||||
});
|
||||
if (!source) throw new ORPCError("NOT_FOUND");
|
||||
const bots =
|
||||
target.kind === "bot"
|
||||
? [target.botId]
|
||||
: [
|
||||
...(target.leadBotId && target.memberBotIds.includes(target.leadBotId)
|
||||
? [target.leadBotId]
|
||||
: []),
|
||||
...target.memberBotIds,
|
||||
];
|
||||
const botId = source.botId && bots.includes(source.botId) ? source.botId : bots[0];
|
||||
if (!botId) throw new ORPCError("NOT_FOUND");
|
||||
try {
|
||||
return await media.start({
|
||||
operation: input.operation,
|
||||
input: input.input,
|
||||
spaceId: actor.spaceId,
|
||||
userId: actor.userId,
|
||||
botId,
|
||||
groupId: target.kind === "group" ? target.groupId : null,
|
||||
threadId: target.threadId,
|
||||
runId: null,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof MediaJobRejectedError) {
|
||||
throw new ORPCError("BAD_REQUEST", { message: error.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function ownedJobId(deps: MediaRouteDeps, actor: Actor, jobId: string): Promise<string> {
|
||||
const job = await deps.prisma.mediaJob.findFirst({
|
||||
where: { id: jobId, spaceId: actor.spaceId, userId: actor.userId },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!job) throw new ORPCError("NOT_FOUND");
|
||||
return job.id;
|
||||
}
|
||||
|
||||
export async function getMediaJob(deps: MediaRouteDeps, actor: Actor, jobId: string) {
|
||||
const media = requireMedia(deps);
|
||||
const snapshot = await media.get(await ownedJobId(deps, actor, jobId));
|
||||
if (!snapshot) throw new ORPCError("NOT_FOUND");
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
export async function cancelMediaJob(deps: MediaRouteDeps, actor: Actor, jobId: string) {
|
||||
const media = requireMedia(deps);
|
||||
const id = await ownedJobId(deps, actor, jobId);
|
||||
await media.cancel(id);
|
||||
const snapshot = await media.get(id);
|
||||
if (!snapshot) throw new ORPCError("NOT_FOUND");
|
||||
return snapshot;
|
||||
}
|
||||
@ -6,6 +6,7 @@ import type {
|
||||
ArtifactStore,
|
||||
ConnectorCatalogItem,
|
||||
JobPublisher,
|
||||
MediaRuntime,
|
||||
MemoryStore,
|
||||
SandboxProvider,
|
||||
} from "@rakazo/adapter-kit";
|
||||
@ -166,6 +167,13 @@ import {
|
||||
renameMainConversation,
|
||||
} from "./main-conversation.js";
|
||||
import { buildMcpUpdateMaterial } from "./mcp-material.js";
|
||||
import {
|
||||
cancelMediaJob,
|
||||
getMediaJob,
|
||||
getMediaSettings,
|
||||
startMediaJob,
|
||||
updateMediaSettings,
|
||||
} from "./media-jobs.js";
|
||||
import {
|
||||
disconnectMemoryProvider,
|
||||
getMemoryProviderConfig,
|
||||
@ -446,6 +454,7 @@ function mcpAssignmentDto(row: {
|
||||
|
||||
export interface RouterDeps {
|
||||
cloudAgent?: CloudAgentConnection | null;
|
||||
media?: MediaRuntime;
|
||||
prisma: PrismaClient;
|
||||
events: ThreadEvents;
|
||||
auth: Auth;
|
||||
@ -3481,6 +3490,27 @@ export function createRouter(deps: RouterDeps) {
|
||||
return { ok: true as const };
|
||||
}),
|
||||
},
|
||||
media: {
|
||||
settings: {
|
||||
get: authed.media.settings.get.handler(({ context }) =>
|
||||
getMediaSettings(deps, context.actor),
|
||||
),
|
||||
update: authed.media.settings.update.handler(({ context, input }) =>
|
||||
updateMediaSettings(deps, context.actor, input),
|
||||
),
|
||||
},
|
||||
jobs: {
|
||||
start: authed.media.jobs.start.handler(({ context, input }) =>
|
||||
startMediaJob(deps, context.actor, input),
|
||||
),
|
||||
get: authed.media.jobs.get.handler(({ context, input }) =>
|
||||
getMediaJob(deps, context.actor, input.jobId),
|
||||
),
|
||||
cancel: authed.media.jobs.cancel.handler(({ context, input }) =>
|
||||
cancelMediaJob(deps, context.actor, input.jobId),
|
||||
),
|
||||
},
|
||||
},
|
||||
connections: {
|
||||
catalog: authed.connections.catalog.handler(async ({ context, input }) => {
|
||||
const adapterContext = connectionContext(
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import type { ConnectorTool } from "@rakazo/adapter-kit";
|
||||
import type { ConnectorTool, MediaCapabilities } from "@rakazo/adapter-kit";
|
||||
import { agentConnectionTools, builtinAgentTools } from "./builtin-tools.js";
|
||||
import { selectCloudAgentTools } from "./cloud-agent-tools-select.js";
|
||||
import { selectMediaTools } from "./media-tools.js";
|
||||
import { selectMemoryTools } from "./memory-tools.js";
|
||||
import { filterImageReturningComputerTools } from "./model-vision.js";
|
||||
import {
|
||||
@ -81,6 +82,8 @@ export function selectBuiltinToolsForRun(options: {
|
||||
trigger: string;
|
||||
semanticMemoryEnabled: boolean;
|
||||
cloudAgentEnabled?: boolean;
|
||||
/** Media tools appear only for operations the space can run. */
|
||||
mediaCapabilities?: MediaCapabilities;
|
||||
messagingChannelRun: boolean;
|
||||
/** Cross-owner agent connections only exist for chat-linked bots. */
|
||||
agentConnectionsEnabled?: boolean;
|
||||
@ -88,21 +91,24 @@ export function selectBuiltinToolsForRun(options: {
|
||||
foreignAgentRun?: boolean;
|
||||
bot: BuiltinToolPolicy;
|
||||
}) {
|
||||
const selected = selectCloudAgentTools(
|
||||
selectMemoryTools(
|
||||
filterBuiltinToolsForRun(
|
||||
filterBuiltinToolsForThread(
|
||||
filterPageBrowserTools(
|
||||
filterImageReturningComputerTools(builtinAgentTools, options.graphicalToolsAllowed),
|
||||
options.pageBrowserAllowed ?? options.graphicalToolsAllowed,
|
||||
const selected = selectMediaTools(
|
||||
selectCloudAgentTools(
|
||||
selectMemoryTools(
|
||||
filterBuiltinToolsForRun(
|
||||
filterBuiltinToolsForThread(
|
||||
filterPageBrowserTools(
|
||||
filterImageReturningComputerTools(builtinAgentTools, options.graphicalToolsAllowed),
|
||||
options.pageBrowserAllowed ?? options.graphicalToolsAllowed,
|
||||
),
|
||||
options.groupId,
|
||||
),
|
||||
options.groupId,
|
||||
options.trigger,
|
||||
),
|
||||
options.trigger,
|
||||
options.semanticMemoryEnabled,
|
||||
),
|
||||
options.semanticMemoryEnabled,
|
||||
Boolean(options.cloudAgentEnabled),
|
||||
),
|
||||
Boolean(options.cloudAgentEnabled),
|
||||
options.mediaCapabilities,
|
||||
).filter(
|
||||
(tool) =>
|
||||
!(options.foreignAgentRun && tool.name === "update_user_profile") &&
|
||||
|
||||
@ -283,6 +283,7 @@ import {
|
||||
needsOAuthProbe,
|
||||
parseMcpServerToolArgs,
|
||||
} from "./mcp-server-tool.js";
|
||||
import { executeMediaTool, isMediaToolName } from "./media-tools.js";
|
||||
import { loadAgentMemoryContext } from "./memory-context.js";
|
||||
import type { MemoryProviderResolver } from "./memory-provider-factory.js";
|
||||
import { selectMemoryTools } from "./memory-tools.js";
|
||||
@ -1477,6 +1478,7 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
trigger: run.trigger,
|
||||
semanticMemoryEnabled,
|
||||
cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId),
|
||||
mediaCapabilities: await deps.media?.capabilities(run.spaceId).catch(() => undefined),
|
||||
messagingChannelRun,
|
||||
agentConnectionsEnabled: hasMessagingIdentity,
|
||||
foreignAgentRun,
|
||||
@ -2598,6 +2600,24 @@ export function createRunExecutor(deps: ExecutorDeps) {
|
||||
return computerScreenToolResult(() => tool(browser, computer, context, args), finish);
|
||||
}
|
||||
|
||||
if (isMediaToolName(name)) {
|
||||
return finish(
|
||||
await executeMediaTool(
|
||||
deps.media,
|
||||
{
|
||||
spaceId: run.spaceId,
|
||||
userId: run.userId,
|
||||
botId: bot.id,
|
||||
groupId: thread.groupId,
|
||||
threadId: run.threadId,
|
||||
runId: run.id,
|
||||
operationKey: effectKey,
|
||||
},
|
||||
name,
|
||||
args,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (name.startsWith("cloud_agent_")) {
|
||||
return finish(
|
||||
await executeCloudAgentTool(
|
||||
|
||||
@ -82,31 +82,48 @@ function keptKey(
|
||||
return new URL(previous.baseUrl).origin === new URL(baseUrl).origin ? previous.apiKey : undefined;
|
||||
}
|
||||
|
||||
/** A media setting the deployment owner has to correct; the message says what is wrong. */
|
||||
export class MediaSettingsInvalidError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "MediaSettingsInvalidError";
|
||||
}
|
||||
}
|
||||
|
||||
function resolveSettings(previous: MediaSettings, update: MediaSettingsUpdate): MediaSettings {
|
||||
const parsed = MediaSettingsUpdateSchema.parse(update);
|
||||
try {
|
||||
let comfyui: MediaSettings["comfyui"] = null;
|
||||
if (parsed.comfyui) {
|
||||
const baseUrl = normalizeComfyUiBaseUrl(parsed.comfyui.baseUrl);
|
||||
const apiKey = keptKey(parsed.comfyui.apiKey, previous.comfyui, baseUrl);
|
||||
assertHttpsForKeyedOpenAiCompatibleUrl(new URL(baseUrl), apiKey);
|
||||
comfyui = { ...parsed.comfyui, baseUrl, apiKey };
|
||||
}
|
||||
let transcription: MediaSettings["transcription"] = null;
|
||||
if (parsed.transcription) {
|
||||
const baseUrl = normalizeTranscriptionBaseUrl(parsed.transcription.baseUrl);
|
||||
const apiKey = keptKey(parsed.transcription.apiKey, previous.transcription, baseUrl);
|
||||
assertHttpsForKeyedOpenAiCompatibleUrl(new URL(baseUrl), apiKey);
|
||||
transcription = { ...parsed.transcription, baseUrl, apiKey };
|
||||
}
|
||||
return MediaSettingsSchema.parse({ comfyui, transcription });
|
||||
} catch (error) {
|
||||
throw new MediaSettingsInvalidError(
|
||||
error instanceof Error ? error.message : "These media settings are not valid",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates and stores the media backends (deployment owner only; the caller authorizes).
|
||||
* Throws an Error with a user-facing message when a URL is not allowed.
|
||||
* Throws MediaSettingsInvalidError when a URL is not allowed.
|
||||
*/
|
||||
export async function saveMediaSettings(
|
||||
deps: MediaSettingsDeps,
|
||||
update: MediaSettingsUpdate,
|
||||
): Promise<MediaSettingsView> {
|
||||
const parsed = MediaSettingsUpdateSchema.parse(update);
|
||||
const previous = await loadMediaSettings(deps);
|
||||
let comfyui: MediaSettings["comfyui"] = null;
|
||||
if (parsed.comfyui) {
|
||||
const baseUrl = normalizeComfyUiBaseUrl(parsed.comfyui.baseUrl);
|
||||
const apiKey = keptKey(parsed.comfyui.apiKey, previous.comfyui, baseUrl);
|
||||
assertHttpsForKeyedOpenAiCompatibleUrl(new URL(baseUrl), apiKey);
|
||||
comfyui = { ...parsed.comfyui, baseUrl, apiKey };
|
||||
}
|
||||
let transcription: MediaSettings["transcription"] = null;
|
||||
if (parsed.transcription) {
|
||||
const baseUrl = normalizeTranscriptionBaseUrl(parsed.transcription.baseUrl);
|
||||
const apiKey = keptKey(parsed.transcription.apiKey, previous.transcription, baseUrl);
|
||||
assertHttpsForKeyedOpenAiCompatibleUrl(new URL(baseUrl), apiKey);
|
||||
transcription = { ...parsed.transcription, baseUrl, apiKey };
|
||||
}
|
||||
const settings = MediaSettingsSchema.parse({ comfyui, transcription });
|
||||
const settings = resolveSettings(await loadMediaSettings(deps), update);
|
||||
if (!settings.comfyui && !settings.transcription) {
|
||||
await deps.prisma.integrationProviderConfig.deleteMany({ where: { id: MEDIA_SETTINGS_ID } });
|
||||
return mediaSettingsView(settings);
|
||||
|
||||
125
packages/adapters/src/media-tools.test.ts
Normal file
125
packages/adapters/src/media-tools.test.ts
Normal file
@ -0,0 +1,125 @@
|
||||
import type { MediaCapabilities, MediaRuntime, StartMediaJobInput } from "@rakazo/adapter-kit";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { selectBuiltinToolsForRun } from "./executor-tools.js";
|
||||
import { MediaJobRejectedError } from "./media-runtime.js";
|
||||
import {
|
||||
executeMediaTool,
|
||||
isMediaToolName,
|
||||
MEDIA_CONTINUE_AFTER_PROPERTY,
|
||||
mediaToolInputSchema,
|
||||
selectMediaTools,
|
||||
} from "./media-tools.js";
|
||||
|
||||
const none: MediaCapabilities = {
|
||||
generate_image: false,
|
||||
generate_video: false,
|
||||
edit_image: false,
|
||||
edit_video: false,
|
||||
export_video: false,
|
||||
transcribe: false,
|
||||
};
|
||||
|
||||
const scope = {
|
||||
spaceId: "space-1",
|
||||
userId: "user-1",
|
||||
botId: "bot-1",
|
||||
groupId: null,
|
||||
threadId: "thread-1",
|
||||
runId: "run-1",
|
||||
operationKey: "effect-1",
|
||||
};
|
||||
|
||||
function runtime(start: MediaRuntime["start"]): MediaRuntime {
|
||||
return {
|
||||
capabilities: async () => none,
|
||||
start,
|
||||
get: async () => null,
|
||||
cancel: async () => undefined,
|
||||
};
|
||||
}
|
||||
|
||||
describe("media tools", () => {
|
||||
it("names tools after their operations", () => {
|
||||
expect(isMediaToolName("generate_image")).toBe(true);
|
||||
expect(isMediaToolName("transcribe")).toBe(true);
|
||||
expect(isMediaToolName("render_plot")).toBe(false);
|
||||
});
|
||||
|
||||
it("shows a media tool only when the space can run its operation", () => {
|
||||
const tools = [{ name: "generate_image" }, { name: "transcribe" }, { name: "web_search" }];
|
||||
expect(selectMediaTools(tools, undefined).map((tool) => tool.name)).toEqual(["web_search"]);
|
||||
expect(selectMediaTools(tools, { ...none, transcribe: true }).map((tool) => tool.name)).toEqual(
|
||||
["transcribe", "web_search"],
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves non-media tools of a run untouched", () => {
|
||||
const base = {
|
||||
graphicalToolsAllowed: true,
|
||||
groupId: null,
|
||||
trigger: "user",
|
||||
semanticMemoryEnabled: false,
|
||||
messagingChannelRun: false,
|
||||
bot: { kind: "specialist", restrictBuiltinTools: false, allowedBuiltinTools: [] },
|
||||
};
|
||||
const names = (mediaCapabilities?: MediaCapabilities) =>
|
||||
selectBuiltinToolsForRun({ ...base, mediaCapabilities })
|
||||
.map((tool) => tool.name)
|
||||
.filter((name) => !isMediaToolName(name));
|
||||
expect(names({ ...none, generate_image: true, transcribe: true })).toEqual(names(undefined));
|
||||
expect(selectBuiltinToolsForRun(base).some((tool) => isMediaToolName(tool.name))).toBe(false);
|
||||
});
|
||||
|
||||
it("adds continue_after to every media tool schema", () => {
|
||||
expect(
|
||||
mediaToolInputSchema({
|
||||
properties: { prompt: { type: "string" } },
|
||||
required: ["prompt"],
|
||||
}),
|
||||
).toEqual({
|
||||
type: "object",
|
||||
properties: { prompt: { type: "string" }, ...MEDIA_CONTINUE_AFTER_PROPERTY },
|
||||
required: ["prompt"],
|
||||
});
|
||||
});
|
||||
|
||||
it("starts the job with the tool scope and returns at once", async () => {
|
||||
const start = vi.fn(async (_input: StartMediaJobInput) => ({
|
||||
jobId: "job-1",
|
||||
messageId: "message-1",
|
||||
}));
|
||||
const result = await executeMediaTool(runtime(start), scope, "generate_image", {
|
||||
prompt: "a fox",
|
||||
continue_after: true,
|
||||
});
|
||||
expect(start).toHaveBeenCalledWith({
|
||||
operation: "generate_image",
|
||||
input: { prompt: "a fox" },
|
||||
...scope,
|
||||
continueAfter: true,
|
||||
});
|
||||
expect(result).toMatchObject({ jobId: "job-1", status: "queued" });
|
||||
expect((result as { note: string }).note).toContain("woken");
|
||||
|
||||
await executeMediaTool(runtime(start), scope, "generate_image", { prompt: "a fox" });
|
||||
expect(start.mock.calls[1]?.[0]).toMatchObject({ continueAfter: false });
|
||||
});
|
||||
|
||||
it("returns refusals to the model and rethrows other failures", async () => {
|
||||
const refused = runtime(async () => {
|
||||
throw new MediaJobRejectedError("Invalid generate_image input: prompt: Required");
|
||||
});
|
||||
expect(await executeMediaTool(refused, scope, "generate_image", {})).toEqual({
|
||||
error: "Invalid generate_image input: prompt: Required",
|
||||
});
|
||||
expect(await executeMediaTool(undefined, scope, "transcribe", {})).toEqual({
|
||||
error: "transcribe is not available.",
|
||||
});
|
||||
const broken = runtime(async () => {
|
||||
throw new Error("database down");
|
||||
});
|
||||
await expect(executeMediaTool(broken, scope, "transcribe", {})).rejects.toThrow(
|
||||
"database down",
|
||||
);
|
||||
});
|
||||
});
|
||||
@ -10,6 +10,7 @@ export * from "./integration-settings.js";
|
||||
export * from "./local-settings.js";
|
||||
export * from "./mcp.js";
|
||||
export * from "./media.js";
|
||||
export * from "./media-jobs.js";
|
||||
export * from "./media-settings.js";
|
||||
export * from "./model-tiers.js";
|
||||
export * from "./openai-compatible-ui.js";
|
||||
|
||||
36
packages/contracts/src/media-jobs.ts
Normal file
36
packages/contracts/src/media-jobs.ts
Normal file
@ -0,0 +1,36 @@
|
||||
import * as z from "zod";
|
||||
import { Id } from "./ids.js";
|
||||
import { MediaJobStatusSchema, MediaOperationSchema, MediaOutputSchema } from "./media.js";
|
||||
|
||||
/** Operations a user starts from a media card (export, transcribe, edit); generation goes through a bot. */
|
||||
export const MediaCardOperationSchema = z.enum(["edit_image", "export_video", "transcribe"]);
|
||||
export type MediaCardOperation = z.infer<typeof MediaCardOperationSchema>;
|
||||
|
||||
export const MediaJobStartInputSchema = z
|
||||
.object({
|
||||
/** The conversation, like other thread procedures: a bot chat or a group. */
|
||||
botId: Id.optional(),
|
||||
groupId: Id.optional(),
|
||||
/** The message the action came from; its bot posts the new card. */
|
||||
messageId: Id,
|
||||
operation: MediaCardOperationSchema,
|
||||
/** The operation's input; validated by the operation. */
|
||||
input: z.record(z.string(), z.unknown()),
|
||||
})
|
||||
.refine((input) => Boolean(input.botId) !== Boolean(input.groupId), {
|
||||
message: "Pass botId or groupId",
|
||||
});
|
||||
export type MediaJobStartInput = z.infer<typeof MediaJobStartInputSchema>;
|
||||
|
||||
export const MediaJobSnapshotSchema = z.object({
|
||||
id: Id,
|
||||
operation: MediaOperationSchema,
|
||||
status: MediaJobStatusSchema,
|
||||
title: z.string(),
|
||||
progress: z.number().min(0).max(1).nullable(),
|
||||
error: z.string().nullable(),
|
||||
outputs: z.array(MediaOutputSchema),
|
||||
});
|
||||
export type MediaJobSnapshotDto = z.infer<typeof MediaJobSnapshotSchema>;
|
||||
|
||||
export const MediaJobStartOutputSchema = z.object({ jobId: Id, messageId: Id });
|
||||
@ -82,6 +82,12 @@ import {
|
||||
IntegrationProviderConfigSchema,
|
||||
IntegrationSetupStateSchema,
|
||||
} from "./integration-settings.js";
|
||||
import {
|
||||
MediaJobSnapshotSchema,
|
||||
MediaJobStartInputSchema,
|
||||
MediaJobStartOutputSchema,
|
||||
} from "./media-jobs.js";
|
||||
import { MediaSettingsUpdateSchema, MediaSettingsViewSchema } from "./media-settings.js";
|
||||
import { ModelTierMappingSchema, SetModelTierInput } from "./model-tiers.js";
|
||||
import {
|
||||
CreateProjectGoalInput,
|
||||
@ -690,6 +696,18 @@ export const appContract = {
|
||||
get: oc.output(IntegrationSetupStateSchema),
|
||||
save: oc.input(IntegrationProviderConfigSchema).output(z.object({ ok: z.literal(true) })),
|
||||
},
|
||||
media: {
|
||||
/** Deployment-wide media backends; deployment owner only. */
|
||||
settings: {
|
||||
get: oc.output(MediaSettingsViewSchema),
|
||||
update: oc.input(MediaSettingsUpdateSchema).output(MediaSettingsViewSchema),
|
||||
},
|
||||
jobs: {
|
||||
start: oc.input(MediaJobStartInputSchema).output(MediaJobStartOutputSchema),
|
||||
get: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema),
|
||||
cancel: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema),
|
||||
},
|
||||
},
|
||||
connections: {
|
||||
catalog: oc
|
||||
.input(z.object({ query: z.string().optional(), connectorId: z.string().optional() }))
|
||||
|
||||
@ -106,6 +106,7 @@ async function main() {
|
||||
"packages/testkit/src/connections.test.ts",
|
||||
"packages/testkit/src/bot-secrets.test.ts",
|
||||
"packages/testkit/src/delegation.postgres.test.ts",
|
||||
"packages/testkit/src/media.postgres.test.ts",
|
||||
"packages/testkit/src/user-profile.postgres.test.ts",
|
||||
"packages/db/src/space-membership.postgres.test.ts",
|
||||
"packages/db/src/main-agent.postgres.test.ts",
|
||||
|
||||
378
packages/testkit/src/media.postgres.test.ts
Normal file
378
packages/testkit/src/media.postgres.test.ts
Normal file
@ -0,0 +1,378 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { ComposioEmulator, type MediaOperationRegistration } from "@rakazo/adapters";
|
||||
import {
|
||||
type MediaJobBlock,
|
||||
MediaOutputSchema,
|
||||
type MediaSettingsView,
|
||||
type MessageBlock,
|
||||
} from "@rakazo/contracts";
|
||||
import { createThreadMessage } from "@rakazo/db";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { createApp } from "../../../apps/api/src/app.ts";
|
||||
import { sessionCookieHeader } from "./index.js";
|
||||
|
||||
type App = { request: (input: string, init?: RequestInit) => Response | Promise<Response> };
|
||||
type AppHandles = Awaited<ReturnType<typeof createApp>>;
|
||||
|
||||
process.env.WAKEUP_DRIVER = "memory";
|
||||
process.env.SANDBOX_PROVIDER = "fake";
|
||||
process.env.AGENT_RUNTIME = "scripted";
|
||||
|
||||
const hasDb = process.env.VERIFY_DATABASE === "1" && Boolean(process.env.DATABASE_URL);
|
||||
const describeWithDatabase = hasDb ? describe.sequential : describe.skip;
|
||||
|
||||
const PNG_BASE64 = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]).toString(
|
||||
"base64",
|
||||
);
|
||||
|
||||
// testkit has no zod of its own; the contracts' schemas carry it.
|
||||
const SourceInputSchema = MediaOutputSchema.pick({ artifactId: true });
|
||||
|
||||
/** Fake card operations: an export that copies its source, and a transcription that waits. */
|
||||
function fakeOperations(): MediaOperationRegistration[] {
|
||||
const exportVideo: MediaOperationRegistration<{ artifactId: string }> = {
|
||||
operation: "export_video",
|
||||
inputSchema: SourceInputSchema,
|
||||
title: () => "Export",
|
||||
available: async () => true,
|
||||
artifactIds: (input) => [input.artifactId],
|
||||
handler: {
|
||||
operation: "export_video",
|
||||
run: async (context) => {
|
||||
const { artifactId } = context.input as { artifactId: string };
|
||||
const source = await context.fetchArtifact(artifactId);
|
||||
const out = path.join(context.tempDir, "export.png");
|
||||
await writeFile(out, Buffer.from(PNG_BASE64, "base64"));
|
||||
await context.addOutput({
|
||||
path: out,
|
||||
name: "export.png",
|
||||
mimeType: "image/png",
|
||||
parentArtifactId: source.artifactId,
|
||||
preset: "square",
|
||||
});
|
||||
return { summary: "Exported." };
|
||||
},
|
||||
},
|
||||
};
|
||||
const transcribe: MediaOperationRegistration<{ artifactId: string }> = {
|
||||
operation: "transcribe",
|
||||
inputSchema: SourceInputSchema,
|
||||
title: () => "Transcript",
|
||||
available: async () => true,
|
||||
artifactIds: (input) => [input.artifactId],
|
||||
handler: {
|
||||
operation: "transcribe",
|
||||
run: async (context) => {
|
||||
await new Promise<void>((resolve) => {
|
||||
context.signal.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
context.signal.throwIfAborted();
|
||||
return { summary: "" };
|
||||
},
|
||||
},
|
||||
};
|
||||
return [exportVideo, transcribe] as MediaOperationRegistration[];
|
||||
}
|
||||
|
||||
describeWithDatabase("media RPCs (PostgreSQL)", () => {
|
||||
let handles: AppHandles;
|
||||
let app: App;
|
||||
const stamp = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
||||
const dataDir = mkdtempSync(path.join(tmpdir(), "rakazo-media-"));
|
||||
|
||||
beforeAll(async () => {
|
||||
const { createApp } = await import("../../../apps/api/src/app.ts");
|
||||
handles = await createApp({
|
||||
databaseUrl: process.env.DATABASE_URL!,
|
||||
dataDir,
|
||||
sandboxProvider: "fake",
|
||||
agentRuntime: "scripted",
|
||||
wakeupDriver: "memory",
|
||||
signupsEnabled: "true",
|
||||
composio: new ComposioEmulator(),
|
||||
mediaOperations: fakeOperations(),
|
||||
});
|
||||
app = handles.app;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await handles?.prisma.integrationProviderConfig.deleteMany({ where: { id: "media" } });
|
||||
await handles?.stop();
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function user(label: string) {
|
||||
const cookie = await signup(app, `${label}-${stamp}@rakazo.test`, label);
|
||||
const me = await rpc<{ userId: string; spaceId: string }>(app, cookie, "me");
|
||||
const bot = await rpc<{ id: string }>(app, cookie, "bots/create", {
|
||||
name: `${label} bot`,
|
||||
title: "",
|
||||
description: "",
|
||||
instructions: "",
|
||||
notifyOnFinish: false,
|
||||
});
|
||||
const thread = await handles.prisma.thread.findUniqueOrThrow({ where: { botId: bot.id } });
|
||||
const card = await createThreadMessage(handles.prisma, {
|
||||
threadId: thread.id,
|
||||
role: "bot",
|
||||
botId: bot.id,
|
||||
blocks: [{ kind: "text", text: "Here is your clip." }],
|
||||
});
|
||||
const artifact = await rpc<{ id: string }>(app, cookie, "artifacts/create", {
|
||||
botId: bot.id,
|
||||
name: "clip.png",
|
||||
mimeType: "image/png",
|
||||
contentBase64: PNG_BASE64,
|
||||
});
|
||||
return { cookie, me, bot, thread, card, artifact };
|
||||
}
|
||||
|
||||
async function waitForStatus(jobId: string, status: string) {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < 20_000) {
|
||||
const job = await handles.prisma.mediaJob.findUnique({ where: { id: jobId } });
|
||||
if (job?.status === status) return job;
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
throw new Error(`media job ${jobId} never reached ${status}`);
|
||||
}
|
||||
|
||||
it("lets only the deployment owner read and change media settings, and never returns keys", async () => {
|
||||
const owner = await user("media-owner");
|
||||
const other = await user("media-other");
|
||||
await handles.prisma.deploymentSettings.upsert({
|
||||
where: { id: "default" },
|
||||
create: { id: "default", ownerUserId: owner.me.userId },
|
||||
update: { ownerUserId: owner.me.userId },
|
||||
});
|
||||
|
||||
const update = {
|
||||
comfyui: {
|
||||
baseUrl: "http://127.0.0.1:8188/",
|
||||
apiKey: "comfy-test-key",
|
||||
imageModel: "sd_xl_base_1.0.safetensors",
|
||||
imageWorkflow: '{"3":{"class_type":"KSampler","inputs":{}}}',
|
||||
videoWorkflow: "",
|
||||
},
|
||||
transcription: { baseUrl: "http://127.0.0.1:8080", model: "whisper-1", apiKey: "" },
|
||||
};
|
||||
expect((await raw(app, other.cookie, "media/settings/get", {})).status).toBe(403);
|
||||
expect((await raw(app, other.cookie, "media/settings/update", update)).status).toBe(403);
|
||||
expect(await handles.prisma.integrationProviderConfig.count({ where: { id: "media" } })).toBe(
|
||||
0,
|
||||
);
|
||||
|
||||
const saved = await rpc<MediaSettingsView>(app, owner.cookie, "media/settings/update", update);
|
||||
expect(saved).toEqual({
|
||||
comfyui: {
|
||||
baseUrl: "http://127.0.0.1:8188",
|
||||
hasApiKey: true,
|
||||
imageModel: "sd_xl_base_1.0.safetensors",
|
||||
imageWorkflow: update.comfyui.imageWorkflow,
|
||||
},
|
||||
transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "whisper-1", hasApiKey: false },
|
||||
});
|
||||
expect(JSON.stringify(saved)).not.toContain("comfy-test-key");
|
||||
expect(await rpc(app, owner.cookie, "media/settings/get")).toEqual(saved);
|
||||
const row = await handles.prisma.integrationProviderConfig.findUniqueOrThrow({
|
||||
where: { id: "media" },
|
||||
});
|
||||
expect(row.ciphertext).not.toContain("comfy-test-key");
|
||||
expect(row.ciphertext).not.toContain("127.0.0.1");
|
||||
|
||||
// An omitted key stays with the same server, never follows the settings elsewhere.
|
||||
const { apiKey: _key, ...withoutKey } = update.comfyui;
|
||||
const kept = await rpc<MediaSettingsView>(app, owner.cookie, "media/settings/update", {
|
||||
...update,
|
||||
comfyui: withoutKey,
|
||||
});
|
||||
expect(kept.comfyui?.hasApiKey).toBe(true);
|
||||
const moved = await rpc<MediaSettingsView>(app, owner.cookie, "media/settings/update", {
|
||||
...update,
|
||||
comfyui: { ...withoutKey, baseUrl: "http://127.0.0.1:8189" },
|
||||
});
|
||||
expect(moved.comfyui?.hasApiKey).toBe(false);
|
||||
|
||||
for (const baseUrl of ["http://169.254.169.254", "http://user:pass@127.0.0.1:8188"]) {
|
||||
const refused = await raw(app, owner.cookie, "media/settings/update", {
|
||||
...update,
|
||||
comfyui: { ...update.comfyui, baseUrl },
|
||||
});
|
||||
expect(refused.status, baseUrl).toBe(400);
|
||||
}
|
||||
const badWorkflow = await raw(app, owner.cookie, "media/settings/update", {
|
||||
...update,
|
||||
comfyui: { ...update.comfyui, imageWorkflow: "[1, 2" },
|
||||
});
|
||||
expect(badWorkflow.status).toBe(400);
|
||||
|
||||
const previous = process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC;
|
||||
try {
|
||||
delete process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC;
|
||||
expect(
|
||||
(
|
||||
await raw(app, owner.cookie, "media/settings/update", {
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "http://transcribe.example.com" },
|
||||
})
|
||||
).status,
|
||||
).toBe(400);
|
||||
process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC = "1";
|
||||
// A key is never sent in cleartext to a public host.
|
||||
expect(
|
||||
(
|
||||
await raw(app, owner.cookie, "media/settings/update", {
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "http://transcribe.example.com", apiKey: "secret" },
|
||||
})
|
||||
).status,
|
||||
).toBe(400);
|
||||
const https = await rpc<MediaSettingsView>(app, owner.cookie, "media/settings/update", {
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "https://transcribe.example.com", apiKey: "secret" },
|
||||
});
|
||||
expect(https.transcription).toMatchObject({
|
||||
baseUrl: "https://transcribe.example.com/v1",
|
||||
hasApiKey: true,
|
||||
});
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC;
|
||||
else process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC = previous;
|
||||
}
|
||||
|
||||
expect(
|
||||
await rpc(app, owner.cookie, "media/settings/update", { comfyui: null, transcription: null }),
|
||||
).toEqual({ comfyui: null, transcription: null });
|
||||
expect(await handles.prisma.integrationProviderConfig.count({ where: { id: "media" } })).toBe(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it("starts, reads and cancels card jobs only for the actor's own threads and files", async () => {
|
||||
const alice = await user("media-alice");
|
||||
const mallory = await user("media-mallory");
|
||||
|
||||
const started = await rpc<{ jobId: string; messageId: string }>(
|
||||
app,
|
||||
alice.cookie,
|
||||
"media/jobs/start",
|
||||
{
|
||||
botId: alice.bot.id,
|
||||
messageId: alice.card.id,
|
||||
operation: "export_video",
|
||||
input: { artifactId: alice.artifact.id },
|
||||
},
|
||||
);
|
||||
const job = await waitForStatus(started.jobId, "done");
|
||||
expect(job).toMatchObject({
|
||||
spaceId: alice.me.spaceId,
|
||||
userId: alice.me.userId,
|
||||
botId: alice.bot.id,
|
||||
threadId: alice.thread.id,
|
||||
runId: null,
|
||||
continueAfter: false,
|
||||
});
|
||||
const snapshot = await rpc<{ status: string; outputs: MediaJobBlock["outputs"] }>(
|
||||
app,
|
||||
alice.cookie,
|
||||
"media/jobs/get",
|
||||
{ jobId: started.jobId },
|
||||
);
|
||||
expect(snapshot.status).toBe("done");
|
||||
expect(snapshot.outputs).toEqual([
|
||||
expect.objectContaining({
|
||||
name: "export.png",
|
||||
parentArtifactId: alice.artifact.id,
|
||||
preset: "square",
|
||||
}),
|
||||
]);
|
||||
const card = await handles.prisma.message.findUniqueOrThrow({
|
||||
where: { id: started.messageId },
|
||||
});
|
||||
expect(card).toMatchObject({ threadId: alice.thread.id, botId: alice.bot.id, role: "bot" });
|
||||
expect((card.blocks as MessageBlock[])[0]).toMatchObject({ kind: "media", status: "done" });
|
||||
|
||||
// Another user can neither read nor cancel the job, nor start one in Alice's conversation
|
||||
// or on her files.
|
||||
expect(
|
||||
(await raw(app, mallory.cookie, "media/jobs/get", { jobId: started.jobId })).status,
|
||||
).toBe(404);
|
||||
expect(
|
||||
(await raw(app, mallory.cookie, "media/jobs/cancel", { jobId: started.jobId })).status,
|
||||
).toBe(404);
|
||||
for (const body of [
|
||||
{
|
||||
botId: alice.bot.id,
|
||||
messageId: alice.card.id,
|
||||
operation: "export_video",
|
||||
input: { artifactId: mallory.artifact.id },
|
||||
},
|
||||
{
|
||||
botId: mallory.bot.id,
|
||||
messageId: alice.card.id,
|
||||
operation: "export_video",
|
||||
input: { artifactId: mallory.artifact.id },
|
||||
},
|
||||
{
|
||||
botId: mallory.bot.id,
|
||||
messageId: mallory.card.id,
|
||||
operation: "export_video",
|
||||
input: { artifactId: alice.artifact.id },
|
||||
},
|
||||
{
|
||||
botId: mallory.bot.id,
|
||||
messageId: mallory.card.id,
|
||||
operation: "generate_image",
|
||||
input: { prompt: "anything" },
|
||||
},
|
||||
]) {
|
||||
const response = await raw(app, mallory.cookie, "media/jobs/start", body);
|
||||
expect(response.status, JSON.stringify(body)).toBeGreaterThanOrEqual(400);
|
||||
}
|
||||
expect(await handles.prisma.mediaJob.count({ where: { userId: mallory.me.userId } })).toBe(0);
|
||||
|
||||
const slow = await rpc<{ jobId: string }>(app, alice.cookie, "media/jobs/start", {
|
||||
botId: alice.bot.id,
|
||||
messageId: alice.card.id,
|
||||
operation: "transcribe",
|
||||
input: { artifactId: alice.artifact.id },
|
||||
});
|
||||
await waitForStatus(slow.jobId, "running");
|
||||
const cancelled = await rpc<{ status: string }>(app, alice.cookie, "media/jobs/cancel", {
|
||||
jobId: slow.jobId,
|
||||
});
|
||||
expect(["running", "cancelled"]).toContain(cancelled.status);
|
||||
await waitForStatus(slow.jobId, "cancelled");
|
||||
});
|
||||
});
|
||||
|
||||
async function signup(app: App, email: string, name: string) {
|
||||
const res = await app.request("/api/auth/sign-up/email", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", origin: "http://127.0.0.1:5173" },
|
||||
body: JSON.stringify({ email, password: "password12", name }),
|
||||
});
|
||||
if (res.status >= 400) throw new Error(`signup failed ${res.status}: ${await res.text()}`);
|
||||
return sessionCookieHeader(res);
|
||||
}
|
||||
|
||||
function raw(app: App, cookie: string, proc: string, body: unknown = {}) {
|
||||
return app.request(`/rpc/${proc}`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie, origin: "http://127.0.0.1:5173" },
|
||||
body: JSON.stringify({ json: body }),
|
||||
});
|
||||
}
|
||||
|
||||
async function rpc<T>(app: App, cookie: string, proc: string, body: unknown = {}): Promise<T> {
|
||||
const res = await raw(app, cookie, proc, body);
|
||||
const text = await res.text();
|
||||
const parsed = JSON.parse(text) as { json?: T; error?: { message?: string } };
|
||||
if (res.status >= 400 || parsed.error) {
|
||||
throw new Error(`${proc} ${res.status}: ${parsed.error?.message ?? text}`);
|
||||
}
|
||||
return parsed.json as T;
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user