diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 2a9e5c1..d3ff4e7 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -15,6 +15,7 @@ import type { ComposioProvider, ConnectorRegistry, DestinationEmulator, + MediaOperationRegistration, RemoteConnectorDependencies, } from "@rakazo/adapters"; import { @@ -143,6 +144,8 @@ export async function createApp( email?: TransactionalEmailProvider; remoteConnectors?: RemoteConnectorDependencies; logger?: Logger; + /** Test seam: media operations instead of the configured ones. */ + mediaOperations?: MediaOperationRegistration[]; } = {}, ): Promise { const { @@ -155,6 +158,7 @@ export async function createApp( email: emailOverride, remoteConnectors, logger: loggerOverride, + mediaOperations, ...envOverrides } = overrides; const env = { ...loadEnv(process.env), ...envOverrides }; @@ -384,9 +388,9 @@ export async function createApp( events, jobs, artifacts, - registrations: createMediaOperations({ - loadSettings: () => loadMediaSettings({ prisma, secrets }), - }), + registrations: + mediaOperations ?? + createMediaOperations({ loadSettings: () => loadMediaSettings({ prisma, secrets }) }), }); const executor = createRunExecutor({ prisma, @@ -468,6 +472,7 @@ export async function createApp( const router = createRouter({ cloudAgent, + media, prisma, events, auth, diff --git a/apps/api/src/media-jobs.ts b/apps/api/src/media-jobs.ts new file mode 100644 index 0000000..4a289e8 --- /dev/null +++ b/apps/api/src/media-jobs.ts @@ -0,0 +1,118 @@ +import { ORPCError } from "@orpc/server"; +import type { MediaRuntime } from "@rakazo/adapter-kit"; +import { + type EncryptedSecretStore, + loadMediaSettings, + MediaJobRejectedError, + MediaSettingsInvalidError, + mediaSettingsView, + saveMediaSettings, +} from "@rakazo/adapters"; +import type { Actor, MediaJobStartInput, MediaSettingsUpdate } from "@rakazo/contracts"; +import type { PrismaClient } from "@rakazo/db"; +import { resolveThreadTarget } from "./thread-target.js"; + +export interface MediaRouteDeps { + prisma: PrismaClient; + secrets: EncryptedSecretStore; + media?: MediaRuntime; +} + +/** Media backends run on the deployment owner's own server, so only they configure them. */ +function requireDeploymentOwner(actor: Actor) { + if (!actor.isDeploymentOwner) throw new ORPCError("FORBIDDEN"); +} + +function requireMedia(deps: MediaRouteDeps): MediaRuntime { + if (!deps.media) throw new ORPCError("NOT_IMPLEMENTED"); + return deps.media; +} + +export async function getMediaSettings(deps: MediaRouteDeps, actor: Actor) { + requireDeploymentOwner(actor); + return mediaSettingsView(await loadMediaSettings(deps)); +} + +export async function updateMediaSettings( + deps: MediaRouteDeps, + actor: Actor, + input: MediaSettingsUpdate, +) { + requireDeploymentOwner(actor); + try { + return await saveMediaSettings(deps, input); + } catch (error) { + if (error instanceof MediaSettingsInvalidError) { + throw new ORPCError("BAD_REQUEST", { message: error.message }); + } + throw error; + } +} + +/** + * Starts an operation a user picked on a card (export, transcribe, edit) in the same conversation. + * The thread is authorized like every thread procedure; the runtime checks that the operation's + * source artifacts belong to the actor, and the job can only ever read the actor's artifacts. + */ +export async function startMediaJob(deps: MediaRouteDeps, actor: Actor, input: MediaJobStartInput) { + const media = requireMedia(deps); + const target = await resolveThreadTarget(deps.prisma, actor, input); + const source = await deps.prisma.message.findFirst({ + where: { id: input.messageId, threadId: target.threadId }, + select: { botId: true }, + }); + if (!source) throw new ORPCError("NOT_FOUND"); + const bots = + target.kind === "bot" + ? [target.botId] + : [ + ...(target.leadBotId && target.memberBotIds.includes(target.leadBotId) + ? [target.leadBotId] + : []), + ...target.memberBotIds, + ]; + const botId = source.botId && bots.includes(source.botId) ? source.botId : bots[0]; + if (!botId) throw new ORPCError("NOT_FOUND"); + try { + return await media.start({ + operation: input.operation, + input: input.input, + spaceId: actor.spaceId, + userId: actor.userId, + botId, + groupId: target.kind === "group" ? target.groupId : null, + threadId: target.threadId, + runId: null, + }); + } catch (error) { + if (error instanceof MediaJobRejectedError) { + throw new ORPCError("BAD_REQUEST", { message: error.message }); + } + throw error; + } +} + +async function ownedJobId(deps: MediaRouteDeps, actor: Actor, jobId: string): Promise { + const job = await deps.prisma.mediaJob.findFirst({ + where: { id: jobId, spaceId: actor.spaceId, userId: actor.userId }, + select: { id: true }, + }); + if (!job) throw new ORPCError("NOT_FOUND"); + return job.id; +} + +export async function getMediaJob(deps: MediaRouteDeps, actor: Actor, jobId: string) { + const media = requireMedia(deps); + const snapshot = await media.get(await ownedJobId(deps, actor, jobId)); + if (!snapshot) throw new ORPCError("NOT_FOUND"); + return snapshot; +} + +export async function cancelMediaJob(deps: MediaRouteDeps, actor: Actor, jobId: string) { + const media = requireMedia(deps); + const id = await ownedJobId(deps, actor, jobId); + await media.cancel(id); + const snapshot = await media.get(id); + if (!snapshot) throw new ORPCError("NOT_FOUND"); + return snapshot; +} diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 2baf21e..4255c3d 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -6,6 +6,7 @@ import type { ArtifactStore, ConnectorCatalogItem, JobPublisher, + MediaRuntime, MemoryStore, SandboxProvider, } from "@rakazo/adapter-kit"; @@ -166,6 +167,13 @@ import { renameMainConversation, } from "./main-conversation.js"; import { buildMcpUpdateMaterial } from "./mcp-material.js"; +import { + cancelMediaJob, + getMediaJob, + getMediaSettings, + startMediaJob, + updateMediaSettings, +} from "./media-jobs.js"; import { disconnectMemoryProvider, getMemoryProviderConfig, @@ -446,6 +454,7 @@ function mcpAssignmentDto(row: { export interface RouterDeps { cloudAgent?: CloudAgentConnection | null; + media?: MediaRuntime; prisma: PrismaClient; events: ThreadEvents; auth: Auth; @@ -3481,6 +3490,27 @@ export function createRouter(deps: RouterDeps) { return { ok: true as const }; }), }, + media: { + settings: { + get: authed.media.settings.get.handler(({ context }) => + getMediaSettings(deps, context.actor), + ), + update: authed.media.settings.update.handler(({ context, input }) => + updateMediaSettings(deps, context.actor, input), + ), + }, + jobs: { + start: authed.media.jobs.start.handler(({ context, input }) => + startMediaJob(deps, context.actor, input), + ), + get: authed.media.jobs.get.handler(({ context, input }) => + getMediaJob(deps, context.actor, input.jobId), + ), + cancel: authed.media.jobs.cancel.handler(({ context, input }) => + cancelMediaJob(deps, context.actor, input.jobId), + ), + }, + }, connections: { catalog: authed.connections.catalog.handler(async ({ context, input }) => { const adapterContext = connectionContext( diff --git a/packages/adapters/src/executor-tools.ts b/packages/adapters/src/executor-tools.ts index 2e819b8..1045aa2 100644 --- a/packages/adapters/src/executor-tools.ts +++ b/packages/adapters/src/executor-tools.ts @@ -1,6 +1,7 @@ -import type { ConnectorTool } from "@rakazo/adapter-kit"; +import type { ConnectorTool, MediaCapabilities } from "@rakazo/adapter-kit"; import { agentConnectionTools, builtinAgentTools } from "./builtin-tools.js"; import { selectCloudAgentTools } from "./cloud-agent-tools-select.js"; +import { selectMediaTools } from "./media-tools.js"; import { selectMemoryTools } from "./memory-tools.js"; import { filterImageReturningComputerTools } from "./model-vision.js"; import { @@ -81,6 +82,8 @@ export function selectBuiltinToolsForRun(options: { trigger: string; semanticMemoryEnabled: boolean; cloudAgentEnabled?: boolean; + /** Media tools appear only for operations the space can run. */ + mediaCapabilities?: MediaCapabilities; messagingChannelRun: boolean; /** Cross-owner agent connections only exist for chat-linked bots. */ agentConnectionsEnabled?: boolean; @@ -88,21 +91,24 @@ export function selectBuiltinToolsForRun(options: { foreignAgentRun?: boolean; bot: BuiltinToolPolicy; }) { - const selected = selectCloudAgentTools( - selectMemoryTools( - filterBuiltinToolsForRun( - filterBuiltinToolsForThread( - filterPageBrowserTools( - filterImageReturningComputerTools(builtinAgentTools, options.graphicalToolsAllowed), - options.pageBrowserAllowed ?? options.graphicalToolsAllowed, + const selected = selectMediaTools( + selectCloudAgentTools( + selectMemoryTools( + filterBuiltinToolsForRun( + filterBuiltinToolsForThread( + filterPageBrowserTools( + filterImageReturningComputerTools(builtinAgentTools, options.graphicalToolsAllowed), + options.pageBrowserAllowed ?? options.graphicalToolsAllowed, + ), + options.groupId, ), - options.groupId, + options.trigger, ), - options.trigger, + options.semanticMemoryEnabled, ), - options.semanticMemoryEnabled, + Boolean(options.cloudAgentEnabled), ), - Boolean(options.cloudAgentEnabled), + options.mediaCapabilities, ).filter( (tool) => !(options.foreignAgentRun && tool.name === "update_user_profile") && diff --git a/packages/adapters/src/executor.ts b/packages/adapters/src/executor.ts index 5cb9181..31b094d 100644 --- a/packages/adapters/src/executor.ts +++ b/packages/adapters/src/executor.ts @@ -283,6 +283,7 @@ import { needsOAuthProbe, parseMcpServerToolArgs, } from "./mcp-server-tool.js"; +import { executeMediaTool, isMediaToolName } from "./media-tools.js"; import { loadAgentMemoryContext } from "./memory-context.js"; import type { MemoryProviderResolver } from "./memory-provider-factory.js"; import { selectMemoryTools } from "./memory-tools.js"; @@ -1477,6 +1478,7 @@ export function createRunExecutor(deps: ExecutorDeps) { trigger: run.trigger, semanticMemoryEnabled, cloudAgentEnabled: cloudAgentsEnabled(cloudAgent, run.spaceId), + mediaCapabilities: await deps.media?.capabilities(run.spaceId).catch(() => undefined), messagingChannelRun, agentConnectionsEnabled: hasMessagingIdentity, foreignAgentRun, @@ -2598,6 +2600,24 @@ export function createRunExecutor(deps: ExecutorDeps) { return computerScreenToolResult(() => tool(browser, computer, context, args), finish); } + if (isMediaToolName(name)) { + return finish( + await executeMediaTool( + deps.media, + { + spaceId: run.spaceId, + userId: run.userId, + botId: bot.id, + groupId: thread.groupId, + threadId: run.threadId, + runId: run.id, + operationKey: effectKey, + }, + name, + args, + ), + ); + } if (name.startsWith("cloud_agent_")) { return finish( await executeCloudAgentTool( diff --git a/packages/adapters/src/media-settings.ts b/packages/adapters/src/media-settings.ts index d237a13..325c45c 100644 --- a/packages/adapters/src/media-settings.ts +++ b/packages/adapters/src/media-settings.ts @@ -82,31 +82,48 @@ function keptKey( return new URL(previous.baseUrl).origin === new URL(baseUrl).origin ? previous.apiKey : undefined; } +/** A media setting the deployment owner has to correct; the message says what is wrong. */ +export class MediaSettingsInvalidError extends Error { + constructor(message: string) { + super(message); + this.name = "MediaSettingsInvalidError"; + } +} + +function resolveSettings(previous: MediaSettings, update: MediaSettingsUpdate): MediaSettings { + const parsed = MediaSettingsUpdateSchema.parse(update); + try { + let comfyui: MediaSettings["comfyui"] = null; + if (parsed.comfyui) { + const baseUrl = normalizeComfyUiBaseUrl(parsed.comfyui.baseUrl); + const apiKey = keptKey(parsed.comfyui.apiKey, previous.comfyui, baseUrl); + assertHttpsForKeyedOpenAiCompatibleUrl(new URL(baseUrl), apiKey); + comfyui = { ...parsed.comfyui, baseUrl, apiKey }; + } + let transcription: MediaSettings["transcription"] = null; + if (parsed.transcription) { + const baseUrl = normalizeTranscriptionBaseUrl(parsed.transcription.baseUrl); + const apiKey = keptKey(parsed.transcription.apiKey, previous.transcription, baseUrl); + assertHttpsForKeyedOpenAiCompatibleUrl(new URL(baseUrl), apiKey); + transcription = { ...parsed.transcription, baseUrl, apiKey }; + } + return MediaSettingsSchema.parse({ comfyui, transcription }); + } catch (error) { + throw new MediaSettingsInvalidError( + error instanceof Error ? error.message : "These media settings are not valid", + ); + } +} + /** * Validates and stores the media backends (deployment owner only; the caller authorizes). - * Throws an Error with a user-facing message when a URL is not allowed. + * Throws MediaSettingsInvalidError when a URL is not allowed. */ export async function saveMediaSettings( deps: MediaSettingsDeps, update: MediaSettingsUpdate, ): Promise { - const parsed = MediaSettingsUpdateSchema.parse(update); - const previous = await loadMediaSettings(deps); - let comfyui: MediaSettings["comfyui"] = null; - if (parsed.comfyui) { - const baseUrl = normalizeComfyUiBaseUrl(parsed.comfyui.baseUrl); - const apiKey = keptKey(parsed.comfyui.apiKey, previous.comfyui, baseUrl); - assertHttpsForKeyedOpenAiCompatibleUrl(new URL(baseUrl), apiKey); - comfyui = { ...parsed.comfyui, baseUrl, apiKey }; - } - let transcription: MediaSettings["transcription"] = null; - if (parsed.transcription) { - const baseUrl = normalizeTranscriptionBaseUrl(parsed.transcription.baseUrl); - const apiKey = keptKey(parsed.transcription.apiKey, previous.transcription, baseUrl); - assertHttpsForKeyedOpenAiCompatibleUrl(new URL(baseUrl), apiKey); - transcription = { ...parsed.transcription, baseUrl, apiKey }; - } - const settings = MediaSettingsSchema.parse({ comfyui, transcription }); + const settings = resolveSettings(await loadMediaSettings(deps), update); if (!settings.comfyui && !settings.transcription) { await deps.prisma.integrationProviderConfig.deleteMany({ where: { id: MEDIA_SETTINGS_ID } }); return mediaSettingsView(settings); diff --git a/packages/adapters/src/media-tools.test.ts b/packages/adapters/src/media-tools.test.ts new file mode 100644 index 0000000..9af7370 --- /dev/null +++ b/packages/adapters/src/media-tools.test.ts @@ -0,0 +1,125 @@ +import type { MediaCapabilities, MediaRuntime, StartMediaJobInput } from "@rakazo/adapter-kit"; +import { describe, expect, it, vi } from "vitest"; +import { selectBuiltinToolsForRun } from "./executor-tools.js"; +import { MediaJobRejectedError } from "./media-runtime.js"; +import { + executeMediaTool, + isMediaToolName, + MEDIA_CONTINUE_AFTER_PROPERTY, + mediaToolInputSchema, + selectMediaTools, +} from "./media-tools.js"; + +const none: MediaCapabilities = { + generate_image: false, + generate_video: false, + edit_image: false, + edit_video: false, + export_video: false, + transcribe: false, +}; + +const scope = { + spaceId: "space-1", + userId: "user-1", + botId: "bot-1", + groupId: null, + threadId: "thread-1", + runId: "run-1", + operationKey: "effect-1", +}; + +function runtime(start: MediaRuntime["start"]): MediaRuntime { + return { + capabilities: async () => none, + start, + get: async () => null, + cancel: async () => undefined, + }; +} + +describe("media tools", () => { + it("names tools after their operations", () => { + expect(isMediaToolName("generate_image")).toBe(true); + expect(isMediaToolName("transcribe")).toBe(true); + expect(isMediaToolName("render_plot")).toBe(false); + }); + + it("shows a media tool only when the space can run its operation", () => { + const tools = [{ name: "generate_image" }, { name: "transcribe" }, { name: "web_search" }]; + expect(selectMediaTools(tools, undefined).map((tool) => tool.name)).toEqual(["web_search"]); + expect(selectMediaTools(tools, { ...none, transcribe: true }).map((tool) => tool.name)).toEqual( + ["transcribe", "web_search"], + ); + }); + + it("leaves non-media tools of a run untouched", () => { + const base = { + graphicalToolsAllowed: true, + groupId: null, + trigger: "user", + semanticMemoryEnabled: false, + messagingChannelRun: false, + bot: { kind: "specialist", restrictBuiltinTools: false, allowedBuiltinTools: [] }, + }; + const names = (mediaCapabilities?: MediaCapabilities) => + selectBuiltinToolsForRun({ ...base, mediaCapabilities }) + .map((tool) => tool.name) + .filter((name) => !isMediaToolName(name)); + expect(names({ ...none, generate_image: true, transcribe: true })).toEqual(names(undefined)); + expect(selectBuiltinToolsForRun(base).some((tool) => isMediaToolName(tool.name))).toBe(false); + }); + + it("adds continue_after to every media tool schema", () => { + expect( + mediaToolInputSchema({ + properties: { prompt: { type: "string" } }, + required: ["prompt"], + }), + ).toEqual({ + type: "object", + properties: { prompt: { type: "string" }, ...MEDIA_CONTINUE_AFTER_PROPERTY }, + required: ["prompt"], + }); + }); + + it("starts the job with the tool scope and returns at once", async () => { + const start = vi.fn(async (_input: StartMediaJobInput) => ({ + jobId: "job-1", + messageId: "message-1", + })); + const result = await executeMediaTool(runtime(start), scope, "generate_image", { + prompt: "a fox", + continue_after: true, + }); + expect(start).toHaveBeenCalledWith({ + operation: "generate_image", + input: { prompt: "a fox" }, + ...scope, + continueAfter: true, + }); + expect(result).toMatchObject({ jobId: "job-1", status: "queued" }); + expect((result as { note: string }).note).toContain("woken"); + + await executeMediaTool(runtime(start), scope, "generate_image", { prompt: "a fox" }); + expect(start.mock.calls[1]?.[0]).toMatchObject({ continueAfter: false }); + }); + + it("returns refusals to the model and rethrows other failures", async () => { + const refused = runtime(async () => { + throw new MediaJobRejectedError("Invalid generate_image input: prompt: Required"); + }); + expect(await executeMediaTool(refused, scope, "generate_image", {})).toEqual({ + error: "Invalid generate_image input: prompt: Required", + }); + expect(await executeMediaTool(undefined, scope, "transcribe", {})).toEqual({ + error: "transcribe is not available.", + }); + const broken = runtime(async () => { + throw new Error("database down"); + }); + await expect(executeMediaTool(broken, scope, "transcribe", {})).rejects.toThrow( + "database down", + ); + }); +}); diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 48aa488..9ed1f63 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -10,6 +10,7 @@ export * from "./integration-settings.js"; export * from "./local-settings.js"; export * from "./mcp.js"; export * from "./media.js"; +export * from "./media-jobs.js"; export * from "./media-settings.js"; export * from "./model-tiers.js"; export * from "./openai-compatible-ui.js"; diff --git a/packages/contracts/src/media-jobs.ts b/packages/contracts/src/media-jobs.ts new file mode 100644 index 0000000..477a6fe --- /dev/null +++ b/packages/contracts/src/media-jobs.ts @@ -0,0 +1,36 @@ +import * as z from "zod"; +import { Id } from "./ids.js"; +import { MediaJobStatusSchema, MediaOperationSchema, MediaOutputSchema } from "./media.js"; + +/** Operations a user starts from a media card (export, transcribe, edit); generation goes through a bot. */ +export const MediaCardOperationSchema = z.enum(["edit_image", "export_video", "transcribe"]); +export type MediaCardOperation = z.infer; + +export const MediaJobStartInputSchema = z + .object({ + /** The conversation, like other thread procedures: a bot chat or a group. */ + botId: Id.optional(), + groupId: Id.optional(), + /** The message the action came from; its bot posts the new card. */ + messageId: Id, + operation: MediaCardOperationSchema, + /** The operation's input; validated by the operation. */ + input: z.record(z.string(), z.unknown()), + }) + .refine((input) => Boolean(input.botId) !== Boolean(input.groupId), { + message: "Pass botId or groupId", + }); +export type MediaJobStartInput = z.infer; + +export const MediaJobSnapshotSchema = z.object({ + id: Id, + operation: MediaOperationSchema, + status: MediaJobStatusSchema, + title: z.string(), + progress: z.number().min(0).max(1).nullable(), + error: z.string().nullable(), + outputs: z.array(MediaOutputSchema), +}); +export type MediaJobSnapshotDto = z.infer; + +export const MediaJobStartOutputSchema = z.object({ jobId: Id, messageId: Id }); diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 6640d4c..30de55a 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -82,6 +82,12 @@ import { IntegrationProviderConfigSchema, IntegrationSetupStateSchema, } from "./integration-settings.js"; +import { + MediaJobSnapshotSchema, + MediaJobStartInputSchema, + MediaJobStartOutputSchema, +} from "./media-jobs.js"; +import { MediaSettingsUpdateSchema, MediaSettingsViewSchema } from "./media-settings.js"; import { ModelTierMappingSchema, SetModelTierInput } from "./model-tiers.js"; import { CreateProjectGoalInput, @@ -690,6 +696,18 @@ export const appContract = { get: oc.output(IntegrationSetupStateSchema), save: oc.input(IntegrationProviderConfigSchema).output(z.object({ ok: z.literal(true) })), }, + media: { + /** Deployment-wide media backends; deployment owner only. */ + settings: { + get: oc.output(MediaSettingsViewSchema), + update: oc.input(MediaSettingsUpdateSchema).output(MediaSettingsViewSchema), + }, + jobs: { + start: oc.input(MediaJobStartInputSchema).output(MediaJobStartOutputSchema), + get: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema), + cancel: oc.input(z.object({ jobId: Id })).output(MediaJobSnapshotSchema), + }, + }, connections: { catalog: oc .input(z.object({ query: z.string().optional(), connectorId: z.string().optional() })) diff --git a/packages/testkit/src/cli/harness.ts b/packages/testkit/src/cli/harness.ts index edc60ae..497a72d 100644 --- a/packages/testkit/src/cli/harness.ts +++ b/packages/testkit/src/cli/harness.ts @@ -106,6 +106,7 @@ async function main() { "packages/testkit/src/connections.test.ts", "packages/testkit/src/bot-secrets.test.ts", "packages/testkit/src/delegation.postgres.test.ts", + "packages/testkit/src/media.postgres.test.ts", "packages/testkit/src/user-profile.postgres.test.ts", "packages/db/src/space-membership.postgres.test.ts", "packages/db/src/main-agent.postgres.test.ts", diff --git a/packages/testkit/src/media.postgres.test.ts b/packages/testkit/src/media.postgres.test.ts new file mode 100644 index 0000000..f5de0c6 --- /dev/null +++ b/packages/testkit/src/media.postgres.test.ts @@ -0,0 +1,378 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { ComposioEmulator, type MediaOperationRegistration } from "@rakazo/adapters"; +import { + type MediaJobBlock, + MediaOutputSchema, + type MediaSettingsView, + type MessageBlock, +} from "@rakazo/contracts"; +import { createThreadMessage } from "@rakazo/db"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { createApp } from "../../../apps/api/src/app.ts"; +import { sessionCookieHeader } from "./index.js"; + +type App = { request: (input: string, init?: RequestInit) => Response | Promise }; +type AppHandles = Awaited>; + +process.env.WAKEUP_DRIVER = "memory"; +process.env.SANDBOX_PROVIDER = "fake"; +process.env.AGENT_RUNTIME = "scripted"; + +const hasDb = process.env.VERIFY_DATABASE === "1" && Boolean(process.env.DATABASE_URL); +const describeWithDatabase = hasDb ? describe.sequential : describe.skip; + +const PNG_BASE64 = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]).toString( + "base64", +); + +// testkit has no zod of its own; the contracts' schemas carry it. +const SourceInputSchema = MediaOutputSchema.pick({ artifactId: true }); + +/** Fake card operations: an export that copies its source, and a transcription that waits. */ +function fakeOperations(): MediaOperationRegistration[] { + const exportVideo: MediaOperationRegistration<{ artifactId: string }> = { + operation: "export_video", + inputSchema: SourceInputSchema, + title: () => "Export", + available: async () => true, + artifactIds: (input) => [input.artifactId], + handler: { + operation: "export_video", + run: async (context) => { + const { artifactId } = context.input as { artifactId: string }; + const source = await context.fetchArtifact(artifactId); + const out = path.join(context.tempDir, "export.png"); + await writeFile(out, Buffer.from(PNG_BASE64, "base64")); + await context.addOutput({ + path: out, + name: "export.png", + mimeType: "image/png", + parentArtifactId: source.artifactId, + preset: "square", + }); + return { summary: "Exported." }; + }, + }, + }; + const transcribe: MediaOperationRegistration<{ artifactId: string }> = { + operation: "transcribe", + inputSchema: SourceInputSchema, + title: () => "Transcript", + available: async () => true, + artifactIds: (input) => [input.artifactId], + handler: { + operation: "transcribe", + run: async (context) => { + await new Promise((resolve) => { + context.signal.addEventListener("abort", () => resolve(), { once: true }); + }); + context.signal.throwIfAborted(); + return { summary: "" }; + }, + }, + }; + return [exportVideo, transcribe] as MediaOperationRegistration[]; +} + +describeWithDatabase("media RPCs (PostgreSQL)", () => { + let handles: AppHandles; + let app: App; + const stamp = `${Date.now()}-${Math.random().toString(36).slice(2)}`; + const dataDir = mkdtempSync(path.join(tmpdir(), "rakazo-media-")); + + beforeAll(async () => { + const { createApp } = await import("../../../apps/api/src/app.ts"); + handles = await createApp({ + databaseUrl: process.env.DATABASE_URL!, + dataDir, + sandboxProvider: "fake", + agentRuntime: "scripted", + wakeupDriver: "memory", + signupsEnabled: "true", + composio: new ComposioEmulator(), + mediaOperations: fakeOperations(), + }); + app = handles.app; + }); + + afterAll(async () => { + await handles?.prisma.integrationProviderConfig.deleteMany({ where: { id: "media" } }); + await handles?.stop(); + rmSync(dataDir, { recursive: true, force: true }); + }); + + async function user(label: string) { + const cookie = await signup(app, `${label}-${stamp}@rakazo.test`, label); + const me = await rpc<{ userId: string; spaceId: string }>(app, cookie, "me"); + const bot = await rpc<{ id: string }>(app, cookie, "bots/create", { + name: `${label} bot`, + title: "", + description: "", + instructions: "", + notifyOnFinish: false, + }); + const thread = await handles.prisma.thread.findUniqueOrThrow({ where: { botId: bot.id } }); + const card = await createThreadMessage(handles.prisma, { + threadId: thread.id, + role: "bot", + botId: bot.id, + blocks: [{ kind: "text", text: "Here is your clip." }], + }); + const artifact = await rpc<{ id: string }>(app, cookie, "artifacts/create", { + botId: bot.id, + name: "clip.png", + mimeType: "image/png", + contentBase64: PNG_BASE64, + }); + return { cookie, me, bot, thread, card, artifact }; + } + + async function waitForStatus(jobId: string, status: string) { + const start = Date.now(); + while (Date.now() - start < 20_000) { + const job = await handles.prisma.mediaJob.findUnique({ where: { id: jobId } }); + if (job?.status === status) return job; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`media job ${jobId} never reached ${status}`); + } + + it("lets only the deployment owner read and change media settings, and never returns keys", async () => { + const owner = await user("media-owner"); + const other = await user("media-other"); + await handles.prisma.deploymentSettings.upsert({ + where: { id: "default" }, + create: { id: "default", ownerUserId: owner.me.userId }, + update: { ownerUserId: owner.me.userId }, + }); + + const update = { + comfyui: { + baseUrl: "http://127.0.0.1:8188/", + apiKey: "comfy-test-key", + imageModel: "sd_xl_base_1.0.safetensors", + imageWorkflow: '{"3":{"class_type":"KSampler","inputs":{}}}', + videoWorkflow: "", + }, + transcription: { baseUrl: "http://127.0.0.1:8080", model: "whisper-1", apiKey: "" }, + }; + expect((await raw(app, other.cookie, "media/settings/get", {})).status).toBe(403); + expect((await raw(app, other.cookie, "media/settings/update", update)).status).toBe(403); + expect(await handles.prisma.integrationProviderConfig.count({ where: { id: "media" } })).toBe( + 0, + ); + + const saved = await rpc(app, owner.cookie, "media/settings/update", update); + expect(saved).toEqual({ + comfyui: { + baseUrl: "http://127.0.0.1:8188", + hasApiKey: true, + imageModel: "sd_xl_base_1.0.safetensors", + imageWorkflow: update.comfyui.imageWorkflow, + }, + transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "whisper-1", hasApiKey: false }, + }); + expect(JSON.stringify(saved)).not.toContain("comfy-test-key"); + expect(await rpc(app, owner.cookie, "media/settings/get")).toEqual(saved); + const row = await handles.prisma.integrationProviderConfig.findUniqueOrThrow({ + where: { id: "media" }, + }); + expect(row.ciphertext).not.toContain("comfy-test-key"); + expect(row.ciphertext).not.toContain("127.0.0.1"); + + // An omitted key stays with the same server, never follows the settings elsewhere. + const { apiKey: _key, ...withoutKey } = update.comfyui; + const kept = await rpc(app, owner.cookie, "media/settings/update", { + ...update, + comfyui: withoutKey, + }); + expect(kept.comfyui?.hasApiKey).toBe(true); + const moved = await rpc(app, owner.cookie, "media/settings/update", { + ...update, + comfyui: { ...withoutKey, baseUrl: "http://127.0.0.1:8189" }, + }); + expect(moved.comfyui?.hasApiKey).toBe(false); + + for (const baseUrl of ["http://169.254.169.254", "http://user:pass@127.0.0.1:8188"]) { + const refused = await raw(app, owner.cookie, "media/settings/update", { + ...update, + comfyui: { ...update.comfyui, baseUrl }, + }); + expect(refused.status, baseUrl).toBe(400); + } + const badWorkflow = await raw(app, owner.cookie, "media/settings/update", { + ...update, + comfyui: { ...update.comfyui, imageWorkflow: "[1, 2" }, + }); + expect(badWorkflow.status).toBe(400); + + const previous = process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC; + try { + delete process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC; + expect( + ( + await raw(app, owner.cookie, "media/settings/update", { + comfyui: null, + transcription: { baseUrl: "http://transcribe.example.com" }, + }) + ).status, + ).toBe(400); + process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC = "1"; + // A key is never sent in cleartext to a public host. + expect( + ( + await raw(app, owner.cookie, "media/settings/update", { + comfyui: null, + transcription: { baseUrl: "http://transcribe.example.com", apiKey: "secret" }, + }) + ).status, + ).toBe(400); + const https = await rpc(app, owner.cookie, "media/settings/update", { + comfyui: null, + transcription: { baseUrl: "https://transcribe.example.com", apiKey: "secret" }, + }); + expect(https.transcription).toMatchObject({ + baseUrl: "https://transcribe.example.com/v1", + hasApiKey: true, + }); + } finally { + if (previous === undefined) delete process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC; + else process.env.RAKAZO_OPENAI_COMPAT_ALLOW_PUBLIC = previous; + } + + expect( + await rpc(app, owner.cookie, "media/settings/update", { comfyui: null, transcription: null }), + ).toEqual({ comfyui: null, transcription: null }); + expect(await handles.prisma.integrationProviderConfig.count({ where: { id: "media" } })).toBe( + 0, + ); + }); + + it("starts, reads and cancels card jobs only for the actor's own threads and files", async () => { + const alice = await user("media-alice"); + const mallory = await user("media-mallory"); + + const started = await rpc<{ jobId: string; messageId: string }>( + app, + alice.cookie, + "media/jobs/start", + { + botId: alice.bot.id, + messageId: alice.card.id, + operation: "export_video", + input: { artifactId: alice.artifact.id }, + }, + ); + const job = await waitForStatus(started.jobId, "done"); + expect(job).toMatchObject({ + spaceId: alice.me.spaceId, + userId: alice.me.userId, + botId: alice.bot.id, + threadId: alice.thread.id, + runId: null, + continueAfter: false, + }); + const snapshot = await rpc<{ status: string; outputs: MediaJobBlock["outputs"] }>( + app, + alice.cookie, + "media/jobs/get", + { jobId: started.jobId }, + ); + expect(snapshot.status).toBe("done"); + expect(snapshot.outputs).toEqual([ + expect.objectContaining({ + name: "export.png", + parentArtifactId: alice.artifact.id, + preset: "square", + }), + ]); + const card = await handles.prisma.message.findUniqueOrThrow({ + where: { id: started.messageId }, + }); + expect(card).toMatchObject({ threadId: alice.thread.id, botId: alice.bot.id, role: "bot" }); + expect((card.blocks as MessageBlock[])[0]).toMatchObject({ kind: "media", status: "done" }); + + // Another user can neither read nor cancel the job, nor start one in Alice's conversation + // or on her files. + expect( + (await raw(app, mallory.cookie, "media/jobs/get", { jobId: started.jobId })).status, + ).toBe(404); + expect( + (await raw(app, mallory.cookie, "media/jobs/cancel", { jobId: started.jobId })).status, + ).toBe(404); + for (const body of [ + { + botId: alice.bot.id, + messageId: alice.card.id, + operation: "export_video", + input: { artifactId: mallory.artifact.id }, + }, + { + botId: mallory.bot.id, + messageId: alice.card.id, + operation: "export_video", + input: { artifactId: mallory.artifact.id }, + }, + { + botId: mallory.bot.id, + messageId: mallory.card.id, + operation: "export_video", + input: { artifactId: alice.artifact.id }, + }, + { + botId: mallory.bot.id, + messageId: mallory.card.id, + operation: "generate_image", + input: { prompt: "anything" }, + }, + ]) { + const response = await raw(app, mallory.cookie, "media/jobs/start", body); + expect(response.status, JSON.stringify(body)).toBeGreaterThanOrEqual(400); + } + expect(await handles.prisma.mediaJob.count({ where: { userId: mallory.me.userId } })).toBe(0); + + const slow = await rpc<{ jobId: string }>(app, alice.cookie, "media/jobs/start", { + botId: alice.bot.id, + messageId: alice.card.id, + operation: "transcribe", + input: { artifactId: alice.artifact.id }, + }); + await waitForStatus(slow.jobId, "running"); + const cancelled = await rpc<{ status: string }>(app, alice.cookie, "media/jobs/cancel", { + jobId: slow.jobId, + }); + expect(["running", "cancelled"]).toContain(cancelled.status); + await waitForStatus(slow.jobId, "cancelled"); + }); +}); + +async function signup(app: App, email: string, name: string) { + const res = await app.request("/api/auth/sign-up/email", { + method: "POST", + headers: { "content-type": "application/json", origin: "http://127.0.0.1:5173" }, + body: JSON.stringify({ email, password: "password12", name }), + }); + if (res.status >= 400) throw new Error(`signup failed ${res.status}: ${await res.text()}`); + return sessionCookieHeader(res); +} + +function raw(app: App, cookie: string, proc: string, body: unknown = {}) { + return app.request(`/rpc/${proc}`, { + method: "POST", + headers: { "content-type": "application/json", cookie, origin: "http://127.0.0.1:5173" }, + body: JSON.stringify({ json: body }), + }); +} + +async function rpc(app: App, cookie: string, proc: string, body: unknown = {}): Promise { + const res = await raw(app, cookie, proc, body); + const text = await res.text(); + const parsed = JSON.parse(text) as { json?: T; error?: { message?: string } }; + if (res.status >= 400 || parsed.error) { + throw new Error(`${proc} ${res.status}: ${parsed.error?.message ?? text}`); + } + return parsed.json as T; +}