import { readFile, rm } from "node:fs/promises"; import { join } from "node:path"; import { MEDIA_PLATFORM_PROFILES, MediaPlatformIdSchema } from "@rakazo/contracts"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { builtinAgentTools } from "../builtin-tools.js"; import { createMediaOperations } from "../media-operations.js"; import { MEDIA_CONTINUE_AFTER_PROPERTY } from "../media-tools.js"; import { parametersFor } from "../pi-runtime.js"; import { detectFfmpeg, measureLoudness, probeMedia, runFfmpeg } from "./ffmpeg.js"; import type { TestArtifact, TestClips } from "./media-test-support.js"; import { createTestClips, fakeMediaJobContext } from "./media-test-support.js"; import { LOUDNESS_TOLERANCE_LU } from "./platform-export.js"; import { cleanFileName, createVideoOperations, subtitleFormat } from "./video-operations.js"; const ffmpeg = await detectFfmpeg(); function operation(name: "edit_video" | "export_video", deps = {}) { const registration = createVideoOperations(deps).find((entry) => entry.operation === name); if (!registration) throw new Error(`missing ${name}`); return registration; } describe("video operation registrations", () => { it("are registered for the runtime and unavailable without ffmpeg", async () => { const names = createMediaOperations({ loadSettings: async () => ({}) as never }).map( (entry) => entry.operation, ); expect(names).toEqual(expect.arrayContaining(["edit_video", "export_video"])); const edit = operation("edit_video", { detect: async () => null }); expect(await edit.available({ spaceId: "space-1" })).toBe(false); const { context } = await fakeMediaJobContext({}, {}); await expect(edit.handler.run(context)).rejects.toThrow( "ffmpeg and ffprobe are not installed on this server.", ); }); it("names every artifact the input reads so start() can authorize them", () => { const edit = operation("edit_video"); const input = edit.inputSchema.parse({ artifact_id: "src", title: "Reel", steps: [ { op: "concat", artifact_ids: ["clip-1", "clip-2"] }, { op: "subtitles", artifact_id: "subs" }, { op: "overlay", artifact_id: "logo" }, ], }); expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs", "logo"]); const exported = operation("export_video"); const exportInput = exported.inputSchema.parse({ artifact_id: "src", presets: ["vertical"] }); expect(exported.artifactIds?.(exportInput)).toEqual(["src"]); }); it("titles exports by platform and aspect ratio unless the bot gave a title", () => { const exported = operation("export_video"); expect( exported.title( exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical", "square"] }), ), ).toBe("9:16 · 1:1"); expect( exported.title( exported.inputSchema.parse({ artifact_id: "a", platforms: ["tiktok", "youtube-shorts"], presets: ["square"], }), ), ).toBe("TikTok · YouTube Shorts · 1:1"); expect( exported.title( exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical"], title: "Reel" }), ), ).toBe("Reel"); }); it("defines tools named like the operations, with continue_after, that the runtime can encode", () => { for (const name of ["edit_video", "export_video"]) { const tool = builtinAgentTools.find((entry) => entry.name === name); expect(tool).toBeDefined(); expect(tool?.inputSchema.properties).toMatchObject(MEDIA_CONTINUE_AFTER_PROPERTY); expect(() => parametersFor(tool!)).not.toThrow(); } }); it("tells the model every platform id with its longest video", () => { const tool = builtinAgentTools.find((entry) => entry.name === "export_video")!; const properties = tool.inputSchema.properties as Record< string, { items?: { enum?: string[] } } >; expect(properties.platforms?.items?.enum).toEqual(MediaPlatformIdSchema.options); for (const profile of Object.values(MEDIA_PLATFORM_PROFILES)) { expect(tool.description).toContain(`${profile.id} ${profile.duration.hardMaxMs / 1000} s`); } }); }); describe("subtitle and file name helpers", () => { it("recognizes SRT and VTT by content only", () => { expect(subtitleFormat("WEBVTT\n\n00:00.000 --> 00:01.000\nHi")).toBe("vtt"); expect(subtitleFormat("1\r\n00:00:01,000 --> 00:00:02,000\r\nHi")).toBe("srt"); expect(subtitleFormat("00:00:01.000 --> 00:00:02.000\nHi")).toBe("srt"); expect(subtitleFormat("#EXTM3U\n#EXTINF:1,\nfile:/etc/passwd")).toBeNull(); expect(subtitleFormat("WEBVTTX")).toBeNull(); }); it("cleans file names without losing the user's language", () => { expect(cleanFileName("Reel: Sommer/Urlaub 2026", "video")).toBe("Reel Sommer Urlaub 2026"); expect(cleanFileName("../..\u0000", "video")).toBe("video"); expect(cleanFileName("x".repeat(200), "video")).toHaveLength(80); }); }); // CI runners may not have ffmpeg; these run wherever it is installed. describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => { let clips: TestClips; let artifacts: Record; beforeAll(async () => { clips = await createTestClips(); artifacts = { land: { path: clips.landscape, name: "Landscape clip.mp4", mimeType: "video/mp4" }, silent: { path: clips.silentPortrait, name: "silent.mp4", mimeType: "video/mp4" }, srt: { path: clips.srt, name: "captions.srt", mimeType: "application/x-subrip" }, vtt: { path: clips.vtt, name: "captions.vtt", mimeType: "text/vtt" }, notes: { path: clips.landscape, name: "notes.txt", mimeType: "text/plain" }, logo: { path: clips.logo, name: "logo.png", mimeType: "image/png" }, }; }); afterAll(async () => { if (clips) await rm(clips.dir, { recursive: true, force: true }); }); async function run(name: "edit_video" | "export_video", input: Record) { const registration = operation(name); const job = await fakeMediaJobContext(registration.inputSchema.parse(input), artifacts); try { const result = await registration.handler.run(job.context); return { ...job, result }; } finally { await rm(job.tempDir, { recursive: true, force: true }); } } it("cuts, reframes to 9:16 with a blurred fill, burns in subtitles and speeds up", async () => { const { outputs, progress, result } = await run("edit_video", { artifact_id: "land", title: "Sommer-Reel", steps: [ { op: "trim", start: 0.5, end: 2.5 }, { op: "reframe", preset: "vertical", fit: "blur" }, { op: "subtitles", artifact_id: "srt" }, { op: "speed", factor: 2 }, { op: "poster", at: 0.5 }, ], }); expect(outputs).toHaveLength(1); const [output] = outputs; expect(output).toMatchObject({ name: "Sommer-Reel.mp4", mimeType: "video/mp4", width: 1080, height: 1920, parentArtifactId: "land", }); expect(output!.posterSize).toBeGreaterThan(0); expect(Math.abs(output!.durationMs! - 1000)).toBeLessThan(150); expect(progress.at(-1)).toBe(1); expect(progress).toEqual([...progress].sort((a, b) => a - b)); expect(result.summary).toContain("Sommer-Reel.mp4 (1080×1920, "); }); it("keeps segments, appends a silent clip with generated silence, and changes volume", async () => { const registration = operation("edit_video"); const job = await fakeMediaJobContext( registration.inputSchema.parse({ artifact_id: "land", title: "Cut", steps: [ { op: "keep", segments: [ { start: 0, end: 0.5 }, { start: 2, end: 2.8 }, ], }, { op: "concat", artifact_ids: ["silent"] }, { op: "volume", gain: 0.5 }, { op: "subtitles", artifact_id: "vtt" }, ], }), artifacts, ); await registration.handler.run(job.context); const [output] = job.outputs; const probe = await probeMedia(output!.path, { mimeType: "video/mp4" }); expect(probe.hasAudio).toBe(true); expect(probe.video).toMatchObject({ width: 320, height: 180, rotation: 0 }); expect(Math.abs(probe.durationMs - 3300)).toBeLessThan(200); await rm(job.tempDir, { recursive: true, force: true }); }); it("plays rotated phone video upright without rotating it twice", async (context) => { const rotated = join(clips.dir, "rotated.mp4"); try { await runFfmpeg([ ...["-display_rotation", "90", "-i", `file:${clips.landscape}`], ...["-c", "copy", `file:${rotated}`], ]); } catch { context.skip("this ffmpeg cannot write rotation metadata"); } const registration = operation("edit_video"); const job = await fakeMediaJobContext( registration.inputSchema.parse({ artifact_id: "rotated", title: "Upright", steps: [{ op: "trim", start: 0, end: 1 }], }), { rotated: { path: rotated, name: "rotated.mp4", mimeType: "video/mp4" } }, ); await registration.handler.run(job.context); const probe = await probeMedia(job.outputs[0]!.path, { mimeType: "video/mp4" }); expect(probe.video).toMatchObject({ width: 180, height: 320, rotation: 0 }); await rm(job.tempDir, { recursive: true, force: true }); }); it("puts an intro first and a translucent logo on the main part", async () => { const red = join(clips.dir, "red.mp4"); const green = join(clips.dir, "green.mp4"); for (const [path, color] of [ [red, "red"], [green, "green"], ] as const) { await runFfmpeg([ ...["-f", "lavfi", "-i", `color=c=${color}:size=320x180:rate=25:duration=1`], ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", `file:${path}`], ]); } const registration = operation("edit_video"); const job = await fakeMediaJobContext( registration.inputSchema.parse({ artifact_id: "red", title: "Branded", steps: [ { op: "overlay", artifact_id: "logo", position: "top_left", width: 0.25, opacity: 0.5 }, { op: "concat", artifact_ids: ["green"], at: "start" }, ], }), { ...artifacts, red: { path: red, name: "red.mp4", mimeType: "video/mp4" }, green: { path: green, name: "green.mp4", mimeType: "video/mp4" }, }, ); try { await registration.handler.run(job.context); const output = job.outputs[0]!.path; expect((await probeMedia(output, { mimeType: "video/mp4" })).durationMs).toBeCloseTo( 2000, -2, ); // The logo is 80×40 at 13, 7: half blue over red there, plain red elsewhere, none on the intro. const [introR, introG] = await pixel(output, 0.5, 30, 20); expect(introG).toBeGreaterThan(100); expect(introR).toBeLessThan(40); const [r, g, b] = await pixel(output, 1.5, 50, 25); expect(r).toBeGreaterThan(90); expect(r).toBeLessThan(170); expect(b).toBeGreaterThan(90); expect(b).toBeLessThan(170); expect(g).toBeLessThan(40); const [plainR, , plainB] = await pixel(output, 1.5, 200, 120); expect(plainR).toBeGreaterThan(200); expect(plainB).toBeLessThan(40); } finally { await rm(job.tempDir, { recursive: true, force: true }); } }); async function pixel(path: string, at: number, x: number, y: number): Promise { const out = join(clips.dir, `pixel-${at}-${x}-${y}.rgb`); await runFfmpeg([ ...["-ss", String(at), "-i", `file:${path}`, "-frames:v", "1"], ...["-vf", `format=rgb24,crop=1:1:${x}:${y}`, "-f", "rawvideo", `file:${out}`], ]); return [...(await readFile(out))]; } it("extracts the audio as M4A", async () => { const { outputs } = await run("edit_video", { artifact_id: "land", title: "Ton", steps: [{ op: "trim", start: 1, end: 2 }, { op: "extract_audio" }], }); expect(outputs).toEqual([ expect.objectContaining({ name: "Ton.m4a", mimeType: "audio/mp4", parentArtifactId: "land" }), ]); expect(outputs[0]?.posterPath).toBeUndefined(); expect(Math.abs(outputs[0]!.durationMs! - 1000)).toBeLessThan(100); }); it("saves a single frame when poster is the only step", async () => { const { outputs } = await run("edit_video", { artifact_id: "land", title: "Cover", steps: [{ op: "poster", at: 1 }], }); expect(outputs).toEqual([ expect.objectContaining({ name: "Cover.jpg", mimeType: "image/jpeg", width: 320, height: 180, }), ]); }); it("exports one output per distinct preset with posters and batch progress", async () => { const { outputs, progress } = await run("export_video", { artifact_id: "land", presets: ["square", "portrait", "square"], fit: "pad", }); expect( outputs.map((output) => [output.name, output.preset, output.width, output.height]), ).toEqual([ ["Landscape clip-1x1.mp4", "square", 1080, 1080], ["Landscape clip-4x5.mp4", "portrait", 1080, 1350], ]); for (const output of outputs) { expect(output.parentArtifactId).toBe("land"); expect(output.posterSize).toBeGreaterThan(0); } expect(progress).toContain(0.5); expect(progress.at(-1)).toBe(1); }); it("exports for a platform at its frame, frame rate and loudness, and skips one it is too short for", async () => { const fast = join(clips.dir, "fast.mp4"); await runFfmpeg([ ...["-f", "lavfi", "-i", "testsrc=size=320x180:rate=60:duration=4"], ...["-f", "lavfi", "-i", "sine=frequency=330:sample_rate=48000:duration=4"], ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"], ...["-c:a", "aac", "-shortest", `file:${fast}`], ]); const registration = operation("export_video"); const job = await fakeMediaJobContext( registration.inputSchema.parse({ artifact_id: "fast", platforms: ["tiktok", "snapchat-spotlight"], }), { fast: { path: fast, name: "Fast clip.mp4", mimeType: "video/mp4" } }, ); try { const result = await registration.handler.run(job.context); expect(job.skipped).toEqual([ { platform: "snapchat-spotlight", issues: [{ code: "too_short", minMs: 5000, actualMs: expect.any(Number) }], }, ]); expect(job.outputs).toHaveLength(1); const [output] = job.outputs; expect(output).toMatchObject({ name: "Fast clip-tiktok.mp4", platform: "tiktok", width: 1080, height: 1920, parentArtifactId: "fast", check: { status: "pass", issues: [] }, }); expect(output!.posterSize).toBeGreaterThan(0); const probe = await probeMedia(output!.path, { mimeType: "video/mp4" }); expect(probe.video).toMatchObject({ codec: "h264", width: 1080, height: 1920, fps: 30 }); expect(Math.abs(probe.durationMs - 4000)).toBeLessThan(100); const loudness = await measureLoudness( output!.path, { integratedLufs: -14, truePeakDb: -1 }, { mimeType: "video/mp4" }, ); expect(Math.abs(loudness.integratedLufs + 14)).toBeLessThanOrEqual(LOUDNESS_TOLERANCE_LU); expect(result.summary).toContain("Fast clip-tiktok.mp4 for TikTok (1080×1920, 4 s)"); expect(result.summary).toContain( "Not exported for Snapchat Spotlight: 4 s is shorter than its 5 s minimum; use a longer video.", ); expect(job.progress.at(-1)).toBe(1); } finally { await rm(job.tempDir, { recursive: true, force: true }); } }); it("fails without rendering when the video fits none of the platforms", async () => { const registration = operation("export_video"); const job = await fakeMediaJobContext( registration.inputSchema.parse({ artifact_id: "land", platforms: ["snapchat-spotlight"] }), artifacts, ); await expect(registration.handler.run(job.context)).rejects.toThrow( "Not exported for Snapchat Spotlight: 3 s is shorter than its 5 s minimum; use a longer video.", ); expect(job.skipped.map((target) => target.platform)).toEqual(["snapchat-spotlight"]); expect(job.outputs).toEqual([]); await rm(job.tempDir, { recursive: true, force: true }); }); it("fails clearly on ranges past the end, non-subtitle files and non-videos", async () => { await expect( run("edit_video", { artifact_id: "land", title: "x", steps: [{ op: "trim", start: 5, end: 6 }], }), ).rejects.toThrow("trim 5–6 s is outside the 3 s video."); await expect( run("edit_video", { artifact_id: "land", title: "x", steps: [{ op: "subtitles", artifact_id: "land" }], }), ).rejects.toThrow("Landscape clip.mp4 is not an SRT or VTT file."); await expect( run("edit_video", { artifact_id: "land", title: "x", steps: [{ op: "overlay", artifact_id: "srt" }], }), ).rejects.toThrow("captions.srt is not a PNG, JPEG or WebP image."); await expect( run("export_video", { artifact_id: "notes", presets: ["square"] }), ).rejects.toThrow("notes.txt is not a video."); }); it("stops ffmpeg and stores nothing when the job is cancelled", async () => { const registration = operation("export_video"); const job = await fakeMediaJobContext( registration.inputSchema.parse({ artifact_id: "land", presets: ["cinema"], fit: "blur" }), artifacts, ); const progress = job.context.progress; job.context.progress = async (fraction) => { await progress(fraction); job.controller.abort(new Error("cancelled")); }; await expect(registration.handler.run(job.context)).rejects.toThrow("cancelled"); expect(job.outputs).toEqual([]); await rm(job.tempDir, { recursive: true, force: true }); }); });