cynarAI/evOo-local
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(media): use snake_case keys in every media tool input

Browse Source 
The model saw edit_video and export_video with artifact_id next to
generate_image, generate_video, edit_image and transcribe with artifactId,
sourceArtifactId and negativePrompt. Mixed conventions in one tool list invite
wrong calls, so all media inputs now use snake_case like continue_after and the
other builtin tools: artifact_id, source_artifact_id, negative_prompt,
duration_seconds. Handlers, artifact checks, the media card's transcribe
action and the tests follow. Nothing has shipped, so stored job inputs need no
migration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude 2026-10-03 14:39:36 +00:00
parent 0c2b6dd666
commit 6a450bb694
No known key found for this signature in database
12 changed files with 80 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/web/e2e/media-card.spec.ts
View File

@ -206,6 +206,6 @@ test("renders a media job card while it runs and when it is done", async ({ page
botId, botId,
messageId: expect.stringMatching(/^media-message-\d+$/), messageId: expect.stringMatching(/^media-message-\d+$/),
operation: "transcribe", operation: "transcribe",
input: { artifactId: voice.id, title: "voice-note.wav" }, input: { artifact_id: voice.id, title: "voice-note.wav" },
}); });
}); });

2
apps/web/src/components/media/MediaCard.tsx
View File

@ -378,7 +378,7 @@ export function MediaCard({
: // The new card is titled with the file it transcribes. : // The new card is titled with the file it transcribes.
{ {
operation: "transcribe", operation: "transcribe",
input: { artifactId: action.artifactId, title: action.name }, input: { artifact_id: action.artifactId, title: action.name },
}), }),
}); });
}; };

6
packages/adapters/src/media/image-operations.postgres.test.ts
View File

@ -120,7 +120,7 @@ describePostgres("ComfyUI operations in the media runtime (PostgreSQL + ComfyUI
const edited = await runtime.start({ const edited = await runtime.start({
...scope, ...scope,
operation: "edit_image", operation: "edit_image",
input: { artifactId: source, resize: { aspect: "vertical", fit: "blur" }, format: "jpeg" }, input: { artifact_id: source, resize: { aspect: "vertical", fit: "blur" }, format: "jpeg" },
}); });
await runtime.runJob(edited.jobId); await runtime.runJob(edited.jobId);
const edit = await runtime.get(edited.jobId); const edit = await runtime.get(edited.jobId);
@ -145,14 +145,14 @@ describePostgres("ComfyUI operations in the media runtime (PostgreSQL + ComfyUI
runtime.start({ runtime.start({
...scope, ...scope,
operation: "generate_video", operation: "generate_video",
input: { prompt: "animate it", sourceArtifactId: foreign }, input: { prompt: "animate it", source_artifact_id: foreign },
}), }),
).rejects.toThrow(MediaJobRejectedError); ).rejects.toThrow(MediaJobRejectedError);
await expect( await expect(
runtime.start({ runtime.start({
...scope, ...scope,
operation: "edit_image", operation: "edit_image",
input: { artifactId: foreign, rotate: 90 }, input: { artifact_id: foreign, rotate: 90 },
}), }),
).rejects.toThrow(MediaJobRejectedError); ).rejects.toThrow(MediaJobRejectedError);
}); });

27
packages/adapters/src/media/image-operations.test.ts
View File

@ -165,7 +165,7 @@ describe("image operations", () => {
const { outputs } = await run( const { outputs } = await run(
operation, operation,
"generate_image", "generate_image",
{ prompt: "night", sourceArtifactId: "photo-1" }, { prompt: "night", source_artifact_id: "photo-1" },
{ "photo-1": photo }, { "photo-1": photo },
); );
const [upload] = emulator.uploads.values(); const [upload] = emulator.uploads.values();
@ -173,7 +173,7 @@ describe("image operations", () => {
expect(outputs[0]).toMatchObject({ width: 1280, height: 832, parentArtifactId: "photo-1" }); expect(outputs[0]).toMatchObject({ width: 1280, height: 832, parentArtifactId: "photo-1" });
const clip = await fixture("clip.mp4", 8, 8); const clip = await fixture("clip.mp4", 8, 8);
await expect( await expect(
run(operation, "generate_image", { prompt: "x", sourceArtifactId: "c" }, { c: clip }), run(operation, "generate_image", { prompt: "x", source_artifact_id: "c" }, { c: clip }),
).rejects.toThrow("The source must be an image"); ).rejects.toThrow("The source must be an image");
}); });
@ -203,7 +203,7 @@ describe("image operations", () => {
operation, operation,
"edit_image", "edit_image",
{ {
artifactId: "photo-1", artifact_id: "photo-1",
crop: { left: 0, top: 0, width: 300, height: 300 }, crop: { left: 0, top: 0, width: 300, height: 300 },
rotate: 90, rotate: 90,
resize: { aspect: "vertical", fit: "blur" }, resize: { aspect: "vertical", fit: "blur" },
@ -226,12 +226,17 @@ describe("image operations", () => {
const titled = await run( const titled = await run(
operation, operation,
"edit_image", "edit_image",
{ artifactId: "photo-1", title: "Titelbild: Sommer", rotate: 180 }, { artifact_id: "photo-1", title: "Titelbild: Sommer", rotate: 180 },
{ "photo-1": photo }, { "photo-1": photo },
); );
expect(titled.outputs[0]?.name).toBe("Titelbild Sommer.jpg"); expect(titled.outputs[0]?.name).toBe("Titelbild Sommer.jpg");
await expect( await expect(
run(operation, "edit_image", { artifactId: "photo-1", prompt: "snow" }, { "photo-1": photo }), run(
operation,
"edit_image",
{ artifact_id: "photo-1", prompt: "snow" },
{ "photo-1": photo },
),
).rejects.toThrow("ComfyUI is not configured"); ).rejects.toThrow("ComfyUI is not configured");
}); });
@ -239,7 +244,7 @@ describe("image operations", () => {
const { operation, emulator } = setup(); const { operation, emulator } = setup();
const photo = await fixture("photo.png", 300, 200); const photo = await fixture("photo.png", 300, 200);
const input = { const input = {
artifactId: "photo-1", artifact_id: "photo-1",
crop: { left: 0, top: 0, width: 250, height: 200 }, crop: { left: 0, top: 0, width: 250, height: 200 },
prompt: "snow", prompt: "snow",
}; };
@ -268,7 +273,7 @@ describe("helpers", () => {
expect(fileStem("夜の海", "video")).toBe("video"); expect(fileStem("夜の海", "video")).toBe("video");
expect( expect(
editImageTitle({ editImageTitle({
artifactId: "a", artifact_id: "a",
crop: { left: 0, top: 0, width: 1, height: 1 }, crop: { left: 0, top: 0, width: 1, height: 1 },
rotate: 90, rotate: 90,
resize: { aspect: "vertical", fit: "crop" }, resize: { aspect: "vertical", fit: "crop" },
@ -277,15 +282,15 @@ describe("helpers", () => {
).toBe("1×1 · 90° · 9:16 · PNG"); ).toBe("1×1 · 90° · 9:16 · PNG");
expect( expect(
editImageTitle({ editImageTitle({
artifactId: "a", artifact_id: "a",
flip: "vertical", flip: "vertical",
extend: { top: 4, right: 0, bottom: 0, left: 0 }, extend: { top: 4, right: 0, bottom: 0, left: 0 },
}), }),
).toBe("↕"); ).toBe("↕");
expect( expect(
editImageTitle({ artifactId: "a", title: "Titelbild", prompt: "snow", rotate: 90 }), editImageTitle({ artifact_id: "a", title: "Titelbild", prompt: "snow", rotate: 90 }),
).toBe("Titelbild"); ).toBe("Titelbild");
expect(editImageTitle({ artifactId: "a", resize: { width: 800, fit: "crop" } })).toBe("800×…"); expect(editImageTitle({ artifact_id: "a", resize: { width: 800, fit: "crop" } })).toBe("800×…");
expect(editImageTitle({ artifactId: "a", prompt: "make it snow" })).toBe("make it snow"); expect(editImageTitle({ artifact_id: "a", prompt: "make it snow" })).toBe("make it snow");
}); });
}); });

24
packages/adapters/src/media/image-operations.ts
View File

@ -162,8 +162,8 @@ async function generateImage(
): Promise<MediaOperationResult> { ): Promise<MediaOperationResult> {
const input = GenerateImageInputSchema.parse(context.input); const input = GenerateImageInputSchema.parse(context.input);
const provider = await requireProvider(providers); const provider = await requireProvider(providers);
const source = input.sourceArtifactId const source = input.source_artifact_id
? await fetchImage(context, input.sourceArtifactId) ? await fetchImage(context, input.source_artifact_id)
: undefined; : undefined;
const aspect = input.aspect const aspect = input.aspect
? presetAspect(input.aspect) ? presetAspect(input.aspect)
@ -177,7 +177,7 @@ async function generateImage(
const seed = input.seed ?? randomInt(2 ** 48 - 1); const seed = input.seed ?? randomInt(2 ** 48 - 1);
const images = await provider.generateImage({ const images = await provider.generateImage({
prompt: input.prompt, prompt: input.prompt,
negativePrompt: input.negativePrompt, negativePrompt: input.negative_prompt,
...size, ...size,
count: input.count, count: input.count,
seed, seed,
@ -200,8 +200,8 @@ async function generateVideo(
): Promise<MediaOperationResult> { ): Promise<MediaOperationResult> {
const input = GenerateVideoInputSchema.parse(context.input); const input = GenerateVideoInputSchema.parse(context.input);
const provider = await requireProvider(providers); const provider = await requireProvider(providers);
const source = input.sourceArtifactId const source = input.source_artifact_id
? await fetchImage(context, input.sourceArtifactId) ? await fetchImage(context, input.source_artifact_id)
: undefined; : undefined;
const aspect = input.aspect const aspect = input.aspect
? presetAspect(input.aspect) ? presetAspect(input.aspect)
@ -212,9 +212,9 @@ async function generateVideo(
const seed = input.seed ?? randomInt(2 ** 48 - 1); const seed = input.seed ?? randomInt(2 ** 48 - 1);
const clips = await provider.generateVideo({ const clips = await provider.generateVideo({
prompt: input.prompt, prompt: input.prompt,
negativePrompt: input.negativePrompt, negativePrompt: input.negative_prompt,
...size, ...size,
durationSeconds: input.durationSeconds, durationSeconds: input.duration_seconds,
seed, seed,
source: source && { source: source && {
bytes: await coverImage(source.path, size.width, size.height), bytes: await coverImage(source.path, size.width, size.height),
@ -224,7 +224,7 @@ async function generateVideo(
onProgress: reportProgress(context), onProgress: reportProgress(context),
}); });
await addGeneratedOutputs(context, clips, fileStem(input.prompt, "video"), source?.artifactId); await addGeneratedOutputs(context, clips, fileStem(input.prompt, "video"), source?.artifactId);
return { summary: `Generated a ${input.durationSeconds} s video with seed ${seed}.` }; return { summary: `Generated a ${input.duration_seconds} s video with seed ${seed}.` };
} }
function defaultFormat(mimeType: string): ImageFormat { function defaultFormat(mimeType: string): ImageFormat {
@ -238,7 +238,7 @@ async function editImage(
providers: ProviderSource, providers: ProviderSource,
): Promise<MediaOperationResult> { ): Promise<MediaOperationResult> {
const input = EditImageInputSchema.parse(context.input); const input = EditImageInputSchema.parse(context.input);
const source = await fetchImage(context, input.artifactId); const source = await fetchImage(context, input.artifact_id);
let png: Uint8Array = await editImageGeometry(source.path, input); let png: Uint8Array = await editImageGeometry(source.path, input);
if (input.prompt) { if (input.prompt) {
const provider = await requireProvider(providers); const provider = await requireProvider(providers);
@ -289,8 +289,8 @@ export function createImageOperations(deps: ImageOperationsDeps): MediaOperation
}; };
const comfyUiConfigured = async () => Boolean((await deps.loadSettings()).comfyui); const comfyUiConfigured = async () => Boolean((await deps.loadSettings()).comfyui);
const status = async (signal?: AbortSignal) => (await providers())?.gpuStatus(signal); const status = async (signal?: AbortSignal) => (await providers())?.gpuStatus(signal);
const sourceIds = (input: { sourceArtifactId?: string }) => const sourceIds = (input: { source_artifact_id?: string }) =>
input.sourceArtifactId ? [input.sourceArtifactId] : []; input.source_artifact_id ? [input.source_artifact_id] : [];
const image: MediaOperationRegistration<GenerateImageInput> = { const image: MediaOperationRegistration<GenerateImageInput> = {
operation: "generate_image", operation: "generate_image",
inputSchema: GenerateImageInputSchema, inputSchema: GenerateImageInputSchema,
@ -315,7 +315,7 @@ export function createImageOperations(deps: ImageOperationsDeps): MediaOperation
title: editImageTitle, title: editImageTitle,
available: sharpAvailable, available: sharpAvailable,
handler: { operation: "edit_image", run: (context) => editImage(context, providers) }, handler: { operation: "edit_image", run: (context) => editImage(context, providers) },
artifactIds: (input) => [input.artifactId], artifactIds: (input) => [input.artifact_id],
}; };
return [image, video, edit] as MediaOperationRegistration[]; return [image, video, edit] as MediaOperationRegistration[];
} }

6
packages/adapters/src/media/transcription.postgres.test.ts
View File

@ -130,7 +130,7 @@ describePostgres("transcribe through the media runtime (PostgreSQL)", () => {
it("turns an audio artifact into subtitle files on the media card", async () => { it("turns an audio artifact into subtitle files on the media card", async () => {
const ctx = await setup(); const ctx = await setup();
expect((await ctx.runtime.capabilities(ctx.id)).transcribe).toBe(true); expect((await ctx.runtime.capabilities(ctx.id)).transcribe).toBe(true);
const started = await ctx.start({ artifactId: ctx.source.id }); const started = await ctx.start({ artifact_id: ctx.source.id });
expect(await ctx.card(started.jobId)).toMatchObject({ expect(await ctx.card(started.jobId)).toMatchObject({
operation: "transcribe", operation: "transcribe",
status: "queued", status: "queued",
@ -163,7 +163,9 @@ describePostgres("transcribe through the media runtime (PostgreSQL)", () => {
it("refuses an artifact of another user before a job exists", async () => { it("refuses an artifact of another user before a job exists", async () => {
const owner = await setup(); const owner = await setup();
const other = await setup(); const other = await setup();
await expect(other.start({ artifactId: owner.source.id })).rejects.toThrow("Unknown artifact."); await expect(other.start({ artifact_id: owner.source.id })).rejects.toThrow(
"Unknown artifact.",
);
expect(await prisma.mediaJob.count({ where: { userId: other.id } })).toBe(0); expect(await prisma.mediaJob.count({ where: { userId: other.id } })).toBe(0);
}); });
}); });

24
packages/adapters/src/media/transcription.test.ts
View File

@ -198,18 +198,18 @@ describe("transcribe operation", () => {
}), }),
).toBe(false); ).toBe(false);
const { registration } = operation(); const { registration } = operation();
const input = registration.inputSchema.parse({ artifactId: "audio-1", language: "de" }); const input = registration.inputSchema.parse({ artifact_id: "audio-1", language: "de" });
expect(registration.artifactIds?.(input)).toEqual(["audio-1"]); expect(registration.artifactIds?.(input)).toEqual(["audio-1"]);
expect(registration.title(input)).toBe("Transcript"); expect(registration.title(input)).toBe("Transcript");
expect(registration.title({ ...input, title: "voice-note.wav" })).toBe("voice-note.wav"); expect(registration.title({ ...input, title: "voice-note.wav" })).toBe("voice-note.wav");
expect( expect(
registration.inputSchema.safeParse({ artifactId: "audio-1", language: "German!" }).success, registration.inputSchema.safeParse({ artifact_id: "audio-1", language: "German!" }).success,
).toBe(false); ).toBe(false);
}); });
it("turns an audio artifact into SRT and VTT files next to it", async () => { it("turns an audio artifact into SRT and VTT files next to it", async () => {
const { registration, server } = operation(); const { registration, server } = operation();
const job = await jobContext(audioSource, { artifactId: "audio-1", language: "de" }); const job = await jobContext(audioSource, { artifact_id: "audio-1", language: "de" });
const result = await registration.handler.run(job.context); const result = await registration.handler.run(job.context);
expect(server.requests).toEqual([ expect(server.requests).toEqual([
expect.objectContaining({ expect.objectContaining({
@ -234,7 +234,7 @@ describe("transcribe operation", () => {
it("writes a text file when the server returns no timings", async () => { it("writes a text file when the server returns no timings", async () => {
const { registration } = operation(TRANSCRIPTION, { flavor: "text" }); const { registration } = operation(TRANSCRIPTION, { flavor: "text" });
const job = await jobContext(audioSource, { artifactId: "audio-1" }); const job = await jobContext(audioSource, { artifact_id: "audio-1" });
const result = await registration.handler.run(job.context); const result = await registration.handler.run(job.context);
expect(job.outputs.map((output) => [output.name, output.mimeType])).toEqual([ expect(job.outputs.map((output) => [output.name, output.mimeType])).toEqual([
["Interview take 2.txt", "text/plain"], ["Interview take 2.txt", "text/plain"],
@ -252,7 +252,7 @@ describe("transcribe operation", () => {
}; };
const { registration } = operation(); const { registration } = operation();
await expect( await expect(
registration.handler.run((await jobContext(video, { artifactId: "video-1" })).context), registration.handler.run((await jobContext(video, { artifact_id: "video-1" })).context),
).rejects.toThrow("Transcribing video needs ffmpeg on the server."); ).rejects.toThrow("Transcribing video needs ffmpeg on the server.");
const image = { const image = {
...audioSource, ...audioSource,
@ -261,12 +261,12 @@ describe("transcribe operation", () => {
kind: "image" as const, kind: "image" as const,
}; };
await expect( await expect(
registration.handler.run((await jobContext(image, { artifactId: "image-1" })).context), registration.handler.run((await jobContext(image, { artifact_id: "image-1" })).context),
).rejects.toThrow("Only audio and video can be transcribed."); ).rejects.toThrow("Only audio and video can be transcribed.");
const long = operation(TRANSCRIPTION, { maxBytes: 4 }); const long = operation(TRANSCRIPTION, { maxBytes: 4 });
await expect( await expect(
long.registration.handler.run( long.registration.handler.run(
(await jobContext(audioSource, { artifactId: "audio-1" })).context, (await jobContext(audioSource, { artifact_id: "audio-1" })).context,
), ),
).rejects.toThrow("too long to transcribe without ffmpeg"); ).rejects.toThrow("too long to transcribe without ffmpeg");
}); });
@ -294,7 +294,7 @@ describe("transcribe operation", () => {
mimeType: "video/mp4", mimeType: "video/mp4",
kind: "video" as const, kind: "video" as const,
}; };
const job = await jobContext(video, { artifactId: "video-1" }); const job = await jobContext(video, { artifact_id: "video-1" });
await registration.handler.run(job.context); await registration.handler.run(job.context);
expect(extracted).toEqual([ expect(extracted).toEqual([
{ startMs: 0, endMs: 62_000, mimeType: "video/mp4" }, { startMs: 0, endMs: 62_000, mimeType: "video/mp4" },
@ -312,7 +312,7 @@ describe("transcribe operation", () => {
it("waits for a transcription server that is down instead of failing", async () => { it("waits for a transcription server that is down instead of failing", async () => {
const { registration, server } = operation(); const { registration, server } = operation();
server.failNextStatus = 503; server.failNextStatus = 503;
const job = await jobContext(audioSource, { artifactId: "audio-1" }); const job = await jobContext(audioSource, { artifact_id: "audio-1" });
await expect(registration.handler.run(job.context)).rejects.toBeInstanceOf( await expect(registration.handler.run(job.context)).rejects.toBeInstanceOf(
MediaJobDeferredError, MediaJobDeferredError,
); );
@ -358,7 +358,7 @@ describe.skipIf(!ffmpeg)("transcribe with ffmpeg", () => {
it("transcribes a video's audio as 16 kHz mono WAV", async () => { it("transcribes a video's audio as 16 kHz mono WAV", async () => {
const { registration, server } = ffmpegOperation(); const { registration, server } = ffmpegOperation();
const job = await fakeMediaJobContext( const job = await fakeMediaJobContext(
{ artifactId: "video-1" }, { artifact_id: "video-1" },
{ "video-1": { path: clips.landscape, name: "landscape.mp4", mimeType: "video/mp4" } }, { "video-1": { path: clips.landscape, name: "landscape.mp4", mimeType: "video/mp4" } },
); );
dirs.push(job.tempDir); dirs.push(job.tempDir);
@ -374,7 +374,7 @@ describe.skipIf(!ffmpeg)("transcribe with ffmpeg", () => {
it("uploads a long recording in ranges below the server's limit", async () => { it("uploads a long recording in ranges below the server's limit", async () => {
const { registration, server } = ffmpegOperation(1024); const { registration, server } = ffmpegOperation(1024);
const job = await fakeMediaJobContext( const job = await fakeMediaJobContext(
{ artifactId: "audio-1" }, { artifact_id: "audio-1" },
{ "audio-1": { path: longTone, name: "long-tone.m4a", mimeType: "audio/mp4" } }, { "audio-1": { path: longTone, name: "long-tone.m4a", mimeType: "audio/mp4" } },
); );
dirs.push(job.tempDir); dirs.push(job.tempDir);
@ -387,7 +387,7 @@ describe.skipIf(!ffmpeg)("transcribe with ffmpeg", () => {
it("refuses a video without sound", async () => { it("refuses a video without sound", async () => {
const { registration, server } = ffmpegOperation(); const { registration, server } = ffmpegOperation();
const job = await fakeMediaJobContext( const job = await fakeMediaJobContext(
{ artifactId: "video-2" }, { artifact_id: "video-2" },
{ "video-2": { path: clips.silentPortrait, name: "silent.mp4", mimeType: "video/mp4" } }, { "video-2": { path: clips.silentPortrait, name: "silent.mp4", mimeType: "video/mp4" } },
); );
dirs.push(job.tempDir); dirs.push(job.tempDir);

4
packages/adapters/src/media/transcription.ts
View File

@ -148,7 +148,7 @@ export function createTranscriptionOperations(
inputSchema: TranscribeInputSchema, inputSchema: TranscribeInputSchema,
title: (input) => input.title ?? "Transcript", title: (input) => input.title ?? "Transcript",
available: async () => Boolean((await deps.loadSettings()).transcription), available: async () => Boolean((await deps.loadSettings()).transcription),
artifactIds: (input) => [input.artifactId], artifactIds: (input) => [input.artifact_id],
handler: { handler: {
operation: "transcribe", operation: "transcribe",
async run(context) { async run(context) {
@ -156,7 +156,7 @@ export function createTranscriptionOperations(
const settings = (await deps.loadSettings()).transcription; const settings = (await deps.loadSettings()).transcription;
if (!settings) throw new MediaJobRejectedError("Transcription is not set up."); if (!settings) throw new MediaJobRejectedError("Transcription is not set up.");
const provider = createProvider(settings); const provider = createProvider(settings);
const source = await context.fetchArtifact(input.artifactId); const source = await context.fetchArtifact(input.artifact_id);
if (source.kind !== "audio" && source.kind !== "video") { if (source.kind !== "audio" && source.kind !== "video") {
throw new MediaJobRejectedError("Only audio and video can be transcribed."); throw new MediaJobRejectedError("Only audio and video can be transcribed.");
} }

17
packages/contracts/src/media-generation.test.ts
View File

@ -13,7 +13,7 @@ describe("media generation inputs", () => {
}); });
expect(GenerateVideoInputSchema.parse({ prompt: "waves" })).toEqual({ expect(GenerateVideoInputSchema.parse({ prompt: "waves" })).toEqual({
prompt: "waves", prompt: "waves",
durationSeconds: 5, duration_seconds: 5,
}); });
}); });
@ -23,20 +23,21 @@ describe("media generation inputs", () => {
expect( expect(
GenerateImageInputSchema.safeParse({ prompt: "x", width: 512, height: 4096 }).success, GenerateImageInputSchema.safeParse({ prompt: "x", width: 512, height: 4096 }).success,
).toBe(false); ).toBe(false);
expect(GenerateVideoInputSchema.safeParse({ prompt: "x", durationSeconds: 11 }).success).toBe( expect(GenerateVideoInputSchema.safeParse({ prompt: "x", duration_seconds: 11 }).success).toBe(
false, false,
); );
}); });
it("needs at least one edit and valid edit values", () => { it("needs at least one edit and valid edit values", () => {
expect(EditImageInputSchema.safeParse({ artifactId: "a" }).success).toBe(false); expect(EditImageInputSchema.safeParse({ artifact_id: "a" }).success).toBe(false);
expect(EditImageInputSchema.safeParse({ artifactId: "a", rotate: 45 }).success).toBe(false); expect(EditImageInputSchema.safeParse({ artifact_id: "a", rotate: 45 }).success).toBe(false);
expect(EditImageInputSchema.safeParse({ artifactId: "a", resize: {} }).success).toBe(false); expect(EditImageInputSchema.safeParse({ artifact_id: "a", resize: {} }).success).toBe(false);
expect( expect(
EditImageInputSchema.safeParse({ artifactId: "a", extend: { top: 4, color: "red" } }).success, EditImageInputSchema.safeParse({ artifact_id: "a", extend: { top: 4, color: "red" } })
.success,
).toBe(false); ).toBe(false);
expect( expect(
EditImageInputSchema.parse({ artifactId: "a", resize: { aspect: "vertical" }, rotate: 90 }), EditImageInputSchema.parse({ artifact_id: "a", resize: { aspect: "vertical" }, rotate: 90 }),
).toEqual({ artifactId: "a", resize: { aspect: "vertical", fit: "crop" }, rotate: 90 }); ).toEqual({ artifact_id: "a", resize: { aspect: "vertical", fit: "crop" }, rotate: 90 });
}); });
}); });

12
packages/contracts/src/media-generation.ts
View File

@ -24,13 +24,13 @@ const Pixels = z.number().int().min(0).max(16_384);
export const GenerateImageInputSchema = z export const GenerateImageInputSchema = z
.object({ .object({
prompt: Prompt.describe("What the image shows."), prompt: Prompt.describe("What the image shows."),
negativePrompt: NegativePrompt, negative_prompt: NegativePrompt,
aspect: aspect("Default square, or the source image's aspect ratio.").optional(), aspect: aspect("Default square, or the source image's aspect ratio.").optional(),
width: Side.optional().describe("Exact width in pixels, with height; overrides aspect."), width: Side.optional().describe("Exact width in pixels, with height; overrides aspect."),
height: Side.optional(), height: Side.optional(),
count: z.number().int().min(1).max(MEDIA_IMAGE_MAX_COUNT).default(1), count: z.number().int().min(1).max(MEDIA_IMAGE_MAX_COUNT).default(1),
seed: Seed, seed: Seed,
sourceArtifactId: Id.optional().describe("Image artifact to start from (image-to-image)."), source_artifact_id: Id.optional().describe("Image artifact to start from (image-to-image)."),
strength: Strength.describe("With a source: how far to move away from it (default 0.6)."), strength: Strength.describe("With a source: how far to move away from it (default 0.6)."),
}) })
.refine((input) => (input.width === undefined) === (input.height === undefined), { .refine((input) => (input.width === undefined) === (input.height === undefined), {
@ -41,11 +41,11 @@ export type GenerateImageInput = z.infer<typeof GenerateImageInputSchema>;
export const GenerateVideoInputSchema = z.object({ export const GenerateVideoInputSchema = z.object({
prompt: Prompt.describe("What happens in the clip: subject, motion, camera."), prompt: Prompt.describe("What happens in the clip: subject, motion, camera."),
negativePrompt: NegativePrompt, negative_prompt: NegativePrompt,
aspect: aspect("Default landscape, or the start frame's aspect ratio.").optional(), aspect: aspect("Default landscape, or the start frame's aspect ratio.").optional(),
durationSeconds: z.number().min(1).max(MEDIA_VIDEO_MAX_SECONDS).default(5), duration_seconds: z.number().min(1).max(MEDIA_VIDEO_MAX_SECONDS).default(5),
seed: Seed, seed: Seed,
sourceArtifactId: Id.optional().describe("Image artifact to use as the first frame."), source_artifact_id: Id.optional().describe("Image artifact to use as the first frame."),
}); });
export type GenerateVideoInput = z.infer<typeof GenerateVideoInputSchema>; export type GenerateVideoInput = z.infer<typeof GenerateVideoInputSchema>;
@ -55,7 +55,7 @@ export type ImageFormat = z.infer<typeof ImageFormatSchema>;
/** Edits apply in this order: crop, rotate, flip, resize, extend, repaint, format. */ /** Edits apply in this order: crop, rotate, flip, resize, extend, repaint, format. */
export const EditImageInputSchema = z export const EditImageInputSchema = z
.object({ .object({
artifactId: Id.describe("Image artifact to edit."), artifact_id: Id.describe("Image artifact to edit."),
title: MediaTitleSchema.optional().describe( title: MediaTitleSchema.optional().describe(
"A few words in the user's language for the card and file name.", "A few words in the user's language for the card and file name.",
), ),

2
packages/contracts/src/media-transcription.ts
View File

@ -2,7 +2,7 @@ import * as z from "zod";
import { Id } from "./ids.js"; import { Id } from "./ids.js";
export const TranscribeInputSchema = z.object({ export const TranscribeInputSchema = z.object({
artifactId: Id.describe("Audio or video artifact to transcribe."), artifact_id: Id.describe("Audio or video artifact to transcribe."),
language: z language: z
.string() .string()
.trim() .trim()

26
packages/testkit/src/media.postgres.test.ts
View File

@ -5,7 +5,7 @@ import path from "node:path";
import type { MediaOperationRegistration } from "@rakazo/adapters"; import type { MediaOperationRegistration } from "@rakazo/adapters";
import { ComposioEmulator } from "@rakazo/adapters"; import { ComposioEmulator } from "@rakazo/adapters";
import type { MediaJobBlock, MediaSettingsView, MessageBlock } from "@rakazo/contracts"; import type { MediaJobBlock, MediaSettingsView, MessageBlock } from "@rakazo/contracts";
import { MediaOutputSchema } from "@rakazo/contracts"; import { TranscribeInputSchema } from "@rakazo/contracts";
import { createThreadMessage } from "@rakazo/db"; import { createThreadMessage } from "@rakazo/db";
import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { afterAll, beforeAll, describe, expect, it } from "vitest";
import type { createApp } from "../../../apps/api/src/app.ts"; import type { createApp } from "../../../apps/api/src/app.ts";
@ -26,21 +26,21 @@ const PNG_BASE64 = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
); );
// testkit has no zod of its own; the contracts' schemas carry it. // testkit has no zod of its own; the contracts' schemas carry it.
const SourceInputSchema = MediaOutputSchema.pick({ artifactId: true }); const SourceInputSchema = TranscribeInputSchema.pick({ artifact_id: true });
/** Fake card operations: an export that copies its source, and a transcription that waits. */ /** Fake card operations: an export that copies its source, and a transcription that waits. */
function fakeOperations(): MediaOperationRegistration[] { function fakeOperations(): MediaOperationRegistration[] {
const exportVideo: MediaOperationRegistration<{ artifactId: string }> = { const exportVideo: MediaOperationRegistration<{ artifact_id: string }> = {
operation: "export_video", operation: "export_video",
inputSchema: SourceInputSchema, inputSchema: SourceInputSchema,
title: () => "Export", title: () => "Export",
available: async () => true, available: async () => true,
artifactIds: (input) => [input.artifactId], artifactIds: (input) => [input.artifact_id],
handler: { handler: {
operation: "export_video", operation: "export_video",
run: async (context) => { run: async (context) => {
const { artifactId } = context.input as { artifactId: string }; const { artifact_id } = context.input as { artifact_id: string };
const source = await context.fetchArtifact(artifactId); const source = await context.fetchArtifact(artifact_id);
const out = path.join(context.tempDir, "export.png"); const out = path.join(context.tempDir, "export.png");
await writeFile(out, Buffer.from(PNG_BASE64, "base64")); await writeFile(out, Buffer.from(PNG_BASE64, "base64"));
await context.addOutput({ await context.addOutput({
@ -54,12 +54,12 @@ function fakeOperations(): MediaOperationRegistration[] {
}, },
}, },
}; };
const transcribe: MediaOperationRegistration<{ artifactId: string }> = { const transcribe: MediaOperationRegistration<{ artifact_id: string }> = {
operation: "transcribe", operation: "transcribe",
inputSchema: SourceInputSchema, inputSchema: SourceInputSchema,
title: () => "Transcript", title: () => "Transcript",
available: async () => true, available: async () => true,
artifactIds: (input) => [input.artifactId], artifactIds: (input) => [input.artifact_id],
handler: { handler: {
operation: "transcribe", operation: "transcribe",
run: async (context) => { run: async (context) => {
@ -260,7 +260,7 @@ describeWithDatabase("media RPCs (PostgreSQL)", () => {
botId: alice.bot.id, botId: alice.bot.id,
messageId: alice.card.id, messageId: alice.card.id,
operation: "export_video", operation: "export_video",
input: { artifactId: alice.artifact.id }, input: { artifact_id: alice.artifact.id },
}, },
); );
const job = await waitForStatus(started.jobId, "done"); const job = await waitForStatus(started.jobId, "done");
@ -305,19 +305,19 @@ describeWithDatabase("media RPCs (PostgreSQL)", () => {
botId: alice.bot.id, botId: alice.bot.id,
messageId: alice.card.id, messageId: alice.card.id,
operation: "export_video", operation: "export_video",
input: { artifactId: mallory.artifact.id }, input: { artifact_id: mallory.artifact.id },
}, },
{ {
botId: mallory.bot.id, botId: mallory.bot.id,
messageId: alice.card.id, messageId: alice.card.id,
operation: "export_video", operation: "export_video",
input: { artifactId: mallory.artifact.id }, input: { artifact_id: mallory.artifact.id },
}, },
{ {
botId: mallory.bot.id, botId: mallory.bot.id,
messageId: mallory.card.id, messageId: mallory.card.id,
operation: "export_video", operation: "export_video",
input: { artifactId: alice.artifact.id }, input: { artifact_id: alice.artifact.id },
}, },
{ {
botId: mallory.bot.id, botId: mallory.bot.id,
@ -335,7 +335,7 @@ describeWithDatabase("media RPCs (PostgreSQL)", () => {
botId: alice.bot.id, botId: alice.bot.id,
messageId: alice.card.id, messageId: alice.card.id,
operation: "transcribe", operation: "transcribe",
input: { artifactId: alice.artifact.id }, input: { artifact_id: alice.artifact.id },
}); });
await waitForStatus(slow.jobId, "running"); await waitForStatus(slow.jobId, "running");
const cancelled = await rpc<{ status: string }>(app, alice.cookie, "media/jobs/cancel", { const cancelled = await rpc<{ status: string }>(app, alice.cookie, "media/jobs/cancel", {