diff --git a/infra/compose/Dockerfile b/infra/compose/Dockerfile index fa6b508..e40d6f5 100644 --- a/infra/compose/Dockerfile +++ b/infra/compose/Dockerfile @@ -1,4 +1,8 @@ FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a +# ffmpeg/ffprobe cut, reframe and export videos; DejaVu is the burned-in subtitle font. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* RUN corepack enable WORKDIR /app # Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index 3229a7d..da32e33 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -10,24 +10,15 @@ import { USER_FACT_CATEGORIES, } from "@rakazo/contracts"; import { z } from "zod"; +import { mediaToolInputSchema } from "./media-tools.js"; export const SUBAGENT_MODEL_TIER_DESCRIPTION = "Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set."; -/** A media tool's arguments: its operation's input plus whether to wake the bot when it ends. */ -function mediaToolSchema(input: z.ZodType): Record { - const schema = z.toJSONSchema(input, { io: "input" }); - return { - ...schema, - properties: { - ...schema.properties, - continue_after: { - type: "boolean", - description: - "true to be woken with the result when the job ends, to continue a multi-step task. Default false: the card alone shows the result.", - }, - }, - }; +/** A media tool's JSON schema, generated from its operation's input schema. */ +function mediaToolSchemaOf(input: z.ZodType): ConnectorTool["inputSchema"] { + const { properties = {}, required } = z.toJSONSchema(input, { io: "input" }); + return mediaToolInputSchema({ properties, required }); } const VIDEO_PRESETS_AND_FITS = @@ -350,12 +341,12 @@ export const builtinAgentTools: ConnectorTool[] = [ { name: "edit_video", description: `Edit a video artifact in a background job; a media card shows progress and the result, a new version of the source. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4 with a poster. title: a few words in the user's language for the card and file name.`, - inputSchema: mediaToolSchema(EditVideoInputSchema), + inputSchema: mediaToolSchemaOf(EditVideoInputSchema), }, { name: "export_video", description: `Export a video artifact to one or more presets in a background job: one H.264/AAC MP4 with a poster per preset, each a new version of the source, shown on a media card. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`, - inputSchema: mediaToolSchema(ExportVideoInputSchema), + inputSchema: mediaToolSchemaOf(ExportVideoInputSchema), }, { name: "add_mcp_server", diff --git a/packages/adapters/src/media-operations.ts b/packages/adapters/src/media-operations.ts index 98dbf7e..b5e0710 100644 --- a/packages/adapters/src/media-operations.ts +++ b/packages/adapters/src/media-operations.ts @@ -1,4 +1,5 @@ import type { MediaSettings } from "@rakazo/contracts"; +import { createVideoOperations } from "./media/video-operations.js"; import type { MediaOperationRegistration } from "./media-runtime.js"; export interface MediaOperationDeps { @@ -11,5 +12,5 @@ export interface MediaOperationDeps { * from this one list, so both processes run the same operations. */ export function createMediaOperations(_deps: MediaOperationDeps): MediaOperationRegistration[] { - return []; + return [...createVideoOperations()]; } diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts index b34e722..e2ebc77 100644 --- a/packages/adapters/src/media/ffmpeg.ts +++ b/packages/adapters/src/media/ffmpeg.ts @@ -52,7 +52,6 @@ export interface RunFfmpegOptions { outputs?: readonly string[]; /** Kills ffmpeg when it reports no progress for this long. */ stallTimeoutMs?: number; - ffmpegPath?: string; } export class FfmpegError extends Error { @@ -162,9 +161,9 @@ export function ffmpegInputArgs( export async function probeMedia( path: string, - options: { signal?: AbortSignal; mimeType?: string; ffprobePath?: string } = {}, + options: { signal?: AbortSignal; mimeType?: string } = {}, ): Promise { - const ffprobePath = options.ffprobePath ?? (await requireFfmpeg()).ffprobePath; + const { ffprobePath } = await requireFfmpeg(); const input = ffmpegInputArgs(path, { mimeType: options.mimeType }); const result = await capture( ffprobePath, @@ -325,7 +324,7 @@ export async function runFfmpeg( options: RunFfmpegOptions = {}, ): Promise { options.signal?.throwIfAborted(); - const ffmpegPath = options.ffmpegPath ?? (await requireFfmpeg()).ffmpegPath; + const { ffmpegPath } = await requireFfmpeg(); try { await spawnFfmpeg(ffmpegPath, args, options); } catch (error) { @@ -469,12 +468,7 @@ export async function extractAudio( onProgress?: (fraction: number) => void; } = {}, ): Promise { - const tools = await requireFfmpeg(); - const probe = await probeMedia(input, { - signal: options.signal, - mimeType: options.mimeType, - ffprobePath: tools.ffprobePath, - }); + const probe = await probeMedia(input, { signal: options.signal, mimeType: options.mimeType }); if (!probe.hasAudio) throw new Error("The file has no audio track."); const startMs = Math.max(0, options.startMs ?? 0); const endMs = Math.min(probe.durationMs, options.endMs ?? probe.durationMs); @@ -505,7 +499,6 @@ export async function extractAudio( durationMs: endMs - startMs, onProgress: options.onProgress, outputs: [output], - ffmpegPath: tools.ffmpegPath, }, ); } @@ -516,7 +509,6 @@ export async function renderPoster( output: string, options: { signal?: AbortSignal; mimeType?: string; atMs?: number } = {}, ): Promise { - const ffmpegPath = (await requireFfmpeg()).ffmpegPath; const render = (atMs: number) => runFfmpeg( [ @@ -537,7 +529,7 @@ export async function renderPoster( "1", `file:${output}`, ], - { signal: options.signal, outputs: [output], ffmpegPath }, + { signal: options.signal, outputs: [output] }, ); const atMs = Math.max(0, options.atMs ?? 0); await render(atMs); diff --git a/packages/adapters/src/media/video-operations.test.ts b/packages/adapters/src/media/video-operations.test.ts new file mode 100644 index 0000000..1d8d6f9 --- /dev/null +++ b/packages/adapters/src/media/video-operations.test.ts @@ -0,0 +1,289 @@ +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { builtinAgentTools } from "../builtin-tools.js"; +import { createMediaOperations } from "../media-operations.js"; +import { MEDIA_CONTINUE_AFTER_PROPERTY } from "../media-tools.js"; +import { parametersFor } from "../pi-runtime.js"; +import { detectFfmpeg, probeMedia, runFfmpeg } from "./ffmpeg.js"; +import type { TestArtifact, TestClips } from "./media-test-support.js"; +import { createTestClips, fakeMediaJobContext } from "./media-test-support.js"; +import { cleanFileName, createVideoOperations, subtitleFormat } from "./video-operations.js"; + +const ffmpeg = await detectFfmpeg(); + +function operation(name: "edit_video" | "export_video", deps = {}) { + const registration = createVideoOperations(deps).find((entry) => entry.operation === name); + if (!registration) throw new Error(`missing ${name}`); + return registration; +} + +describe("video operation registrations", () => { + it("are registered for the runtime and unavailable without ffmpeg", async () => { + const names = createMediaOperations({ loadSettings: async () => ({}) as never }).map( + (entry) => entry.operation, + ); + expect(names).toEqual(expect.arrayContaining(["edit_video", "export_video"])); + const edit = operation("edit_video", { detect: async () => null }); + expect(await edit.available({ spaceId: "space-1" })).toBe(false); + const { context } = await fakeMediaJobContext({}, {}); + await expect(edit.handler.run(context)).rejects.toThrow( + "ffmpeg and ffprobe are not installed on this server.", + ); + }); + + it("names every artifact the input reads so start() can authorize them", () => { + const edit = operation("edit_video"); + const input = edit.inputSchema.parse({ + artifact_id: "src", + title: "Reel", + steps: [ + { op: "concat", artifact_ids: ["clip-1", "clip-2"] }, + { op: "subtitles", artifact_id: "subs" }, + ], + }); + expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs"]); + const exported = operation("export_video"); + const exportInput = exported.inputSchema.parse({ artifact_id: "src", presets: ["vertical"] }); + expect(exported.artifactIds?.(exportInput)).toEqual(["src"]); + }); + + it("titles exports by aspect ratio unless the bot gave a title", () => { + const exported = operation("export_video"); + expect( + exported.title( + exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical", "square"] }), + ), + ).toBe("9:16 · 1:1"); + expect( + exported.title( + exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical"], title: "Reel" }), + ), + ).toBe("Reel"); + }); + + it("defines tools named like the operations, with continue_after, that the runtime can encode", () => { + for (const name of ["edit_video", "export_video"]) { + const tool = builtinAgentTools.find((entry) => entry.name === name); + expect(tool).toBeDefined(); + expect(tool?.inputSchema.properties).toMatchObject(MEDIA_CONTINUE_AFTER_PROPERTY); + expect(() => parametersFor(tool!)).not.toThrow(); + } + }); +}); + +describe("subtitle and file name helpers", () => { + it("recognizes SRT and VTT by content only", () => { + expect(subtitleFormat("WEBVTT\n\n00:00.000 --> 00:01.000\nHi")).toBe("vtt"); + expect(subtitleFormat("1\r\n00:00:01,000 --> 00:00:02,000\r\nHi")).toBe("srt"); + expect(subtitleFormat("00:00:01.000 --> 00:00:02.000\nHi")).toBe("srt"); + expect(subtitleFormat("#EXTM3U\n#EXTINF:1,\nfile:/etc/passwd")).toBeNull(); + expect(subtitleFormat("WEBVTTX")).toBeNull(); + }); + + it("cleans file names without losing the user's language", () => { + expect(cleanFileName("Reel: Sommer/Urlaub 2026", "video")).toBe("Reel Sommer Urlaub 2026"); + expect(cleanFileName("../..\u0000", "video")).toBe("video"); + expect(cleanFileName("x".repeat(200), "video")).toHaveLength(80); + }); +}); + +// CI runners may not have ffmpeg; these run wherever it is installed. +describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => { + let clips: TestClips; + let artifacts: Record; + + beforeAll(async () => { + clips = await createTestClips(); + artifacts = { + land: { path: clips.landscape, name: "Landscape clip.mp4", mimeType: "video/mp4" }, + silent: { path: clips.silentPortrait, name: "silent.mp4", mimeType: "video/mp4" }, + srt: { path: clips.srt, name: "captions.srt", mimeType: "application/x-subrip" }, + vtt: { path: clips.vtt, name: "captions.vtt", mimeType: "text/vtt" }, + notes: { path: clips.landscape, name: "notes.txt", mimeType: "text/plain" }, + }; + }); + + afterAll(async () => { + if (clips) await rm(clips.dir, { recursive: true, force: true }); + }); + + async function run(name: "edit_video" | "export_video", input: Record) { + const registration = operation(name); + const job = await fakeMediaJobContext(registration.inputSchema.parse(input), artifacts); + try { + const result = await registration.handler.run(job.context); + return { ...job, result }; + } finally { + await rm(job.tempDir, { recursive: true, force: true }); + } + } + + it("cuts, reframes to 9:16 with a blurred fill, burns in subtitles and speeds up", async () => { + const { outputs, progress, result } = await run("edit_video", { + artifact_id: "land", + title: "Sommer-Reel", + steps: [ + { op: "trim", start: 0.5, end: 2.5 }, + { op: "reframe", preset: "vertical", fit: "blur" }, + { op: "subtitles", artifact_id: "srt" }, + { op: "speed", factor: 2 }, + { op: "poster", at: 0.5 }, + ], + }); + expect(outputs).toHaveLength(1); + const [output] = outputs; + expect(output).toMatchObject({ + name: "Sommer-Reel.mp4", + mimeType: "video/mp4", + width: 1080, + height: 1920, + parentArtifactId: "land", + }); + expect(output!.posterSize).toBeGreaterThan(0); + expect(Math.abs(output!.durationMs! - 1000)).toBeLessThan(150); + expect(progress.at(-1)).toBe(1); + expect(progress).toEqual([...progress].sort((a, b) => a - b)); + expect(result.summary).toContain("Sommer-Reel.mp4 (1080×1920, "); + }); + + it("keeps segments, appends a silent clip with generated silence, and changes volume", async () => { + const registration = operation("edit_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ + artifact_id: "land", + title: "Cut", + steps: [ + { + op: "keep", + segments: [ + { start: 0, end: 0.5 }, + { start: 2, end: 2.8 }, + ], + }, + { op: "concat", artifact_ids: ["silent"] }, + { op: "volume", gain: 0.5 }, + { op: "subtitles", artifact_id: "vtt" }, + ], + }), + artifacts, + ); + await registration.handler.run(job.context); + const [output] = job.outputs; + const probe = await probeMedia(output!.path, { mimeType: "video/mp4" }); + expect(probe.hasAudio).toBe(true); + expect(probe.video).toMatchObject({ width: 320, height: 180, rotation: 0 }); + expect(Math.abs(probe.durationMs - 3300)).toBeLessThan(200); + await rm(job.tempDir, { recursive: true, force: true }); + }); + + it("plays rotated phone video upright without rotating it twice", async (context) => { + const rotated = join(clips.dir, "rotated.mp4"); + try { + await runFfmpeg([ + ...["-display_rotation", "90", "-i", `file:${clips.landscape}`], + ...["-c", "copy", `file:${rotated}`], + ]); + } catch { + context.skip("this ffmpeg cannot write rotation metadata"); + } + const registration = operation("edit_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ + artifact_id: "rotated", + title: "Upright", + steps: [{ op: "trim", start: 0, end: 1 }], + }), + { rotated: { path: rotated, name: "rotated.mp4", mimeType: "video/mp4" } }, + ); + await registration.handler.run(job.context); + const probe = await probeMedia(job.outputs[0]!.path, { mimeType: "video/mp4" }); + expect(probe.video).toMatchObject({ width: 180, height: 320, rotation: 0 }); + await rm(job.tempDir, { recursive: true, force: true }); + }); + + it("extracts the audio as M4A", async () => { + const { outputs } = await run("edit_video", { + artifact_id: "land", + title: "Ton", + steps: [{ op: "trim", start: 1, end: 2 }, { op: "extract_audio" }], + }); + expect(outputs).toEqual([ + expect.objectContaining({ name: "Ton.m4a", mimeType: "audio/mp4", parentArtifactId: "land" }), + ]); + expect(outputs[0]?.posterPath).toBeUndefined(); + expect(Math.abs(outputs[0]!.durationMs! - 1000)).toBeLessThan(100); + }); + + it("saves a single frame when poster is the only step", async () => { + const { outputs } = await run("edit_video", { + artifact_id: "land", + title: "Cover", + steps: [{ op: "poster", at: 1 }], + }); + expect(outputs).toEqual([ + expect.objectContaining({ + name: "Cover.jpg", + mimeType: "image/jpeg", + width: 320, + height: 180, + }), + ]); + }); + + it("exports one output per distinct preset with posters and batch progress", async () => { + const { outputs, progress } = await run("export_video", { + artifact_id: "land", + presets: ["square", "portrait", "square"], + fit: "pad", + }); + expect( + outputs.map((output) => [output.name, output.preset, output.width, output.height]), + ).toEqual([ + ["Landscape clip-1x1.mp4", "square", 1080, 1080], + ["Landscape clip-4x5.mp4", "portrait", 1080, 1350], + ]); + for (const output of outputs) { + expect(output.parentArtifactId).toBe("land"); + expect(output.posterSize).toBeGreaterThan(0); + } + expect(progress).toContain(0.5); + expect(progress.at(-1)).toBe(1); + }); + + it("fails clearly on ranges past the end, non-subtitle files and non-videos", async () => { + await expect( + run("edit_video", { + artifact_id: "land", + title: "x", + steps: [{ op: "trim", start: 5, end: 6 }], + }), + ).rejects.toThrow("trim 5–6 s is outside the 3 s video."); + await expect( + run("edit_video", { + artifact_id: "land", + title: "x", + steps: [{ op: "subtitles", artifact_id: "land" }], + }), + ).rejects.toThrow("Landscape clip.mp4 is not an SRT or VTT file."); + await expect( + run("export_video", { artifact_id: "notes", presets: ["square"] }), + ).rejects.toThrow("notes.txt is not a video."); + }); + + it("stops ffmpeg and stores nothing when the job is cancelled", async () => { + const registration = operation("export_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ artifact_id: "land", presets: ["cinema"], fit: "blur" }), + artifacts, + ); + const progress = job.context.progress; + job.context.progress = async (fraction) => { + await progress(fraction); + job.controller.abort(new Error("cancelled")); + }; + await expect(registration.handler.run(job.context)).rejects.toThrow("cancelled"); + expect(job.outputs).toEqual([]); + await rm(job.tempDir, { recursive: true, force: true }); + }); +}); diff --git a/packages/adapters/src/media/video-operations.ts b/packages/adapters/src/media/video-operations.ts index b21d9df..e4e1e6d 100644 --- a/packages/adapters/src/media/video-operations.ts +++ b/packages/adapters/src/media/video-operations.ts @@ -1,11 +1,6 @@ import { copyFile, mkdtemp, open, stat } from "node:fs/promises"; import { join } from "node:path"; -import type { - MediaJobContext, - MediaJobInputFile, - MediaOperationHandler, - MediaOperationResult, -} from "@rakazo/adapter-kit"; +import type { MediaJobContext, MediaJobInputFile, MediaOperationResult } from "@rakazo/adapter-kit"; import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts"; import { ARTIFACT_MAX_BYTES_BY_KIND, @@ -13,6 +8,7 @@ import { ExportVideoInputSchema, MEDIA_PRESETS, } from "@rakazo/contracts"; +import type { MediaOperationRegistration } from "../media-runtime.js"; import type { FfmpegInfo } from "./ffmpeg.js"; import { detectFfmpeg, @@ -29,59 +25,71 @@ export interface VideoOperationsDeps { detect?: () => Promise; } -const PROGRESS_INTERVAL_MS = 1_000; /** Share of each output's progress spent encoding; the rest is probing and the poster. */ const ENCODE_SHARE = 0.95; const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024; /** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */ -export function createVideoOperations(deps: VideoOperationsDeps = {}) { +export function createVideoOperations( + deps: VideoOperationsDeps = {}, +): MediaOperationRegistration[] { const detect = deps.detect ?? detectFfmpeg; const available = async () => (await detect()) !== null; - const tools = async () => { - const info = await detect(); - if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server."); - return info; + const requireTools = async () => { + if (!(await available())) + throw new Error("ffmpeg and ffprobe are not installed on this server."); }; - const editVideo: MediaOperationHandler = { + const editVideo: MediaOperationRegistration = { operation: "edit_video", - run: async (context) => runEditVideo(context, await tools()), + inputSchema: EditVideoInputSchema, + title: (input) => input.title, + available, + artifactIds: (input) => [ + input.artifact_id, + ...input.steps.flatMap((step) => + step.op === "concat" + ? step.artifact_ids + : step.op === "subtitles" + ? [step.artifact_id] + : [], + ), + ], + handler: { + operation: "edit_video", + run: async (context) => { + await requireTools(); + return runEditVideo(context); + }, + }, }; - const exportVideo: MediaOperationHandler = { + const exportVideo: MediaOperationRegistration = { operation: "export_video", - run: async (context) => runExportVideo(context, await tools()), + inputSchema: ExportVideoInputSchema, + title: (input) => + input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "), + available, + artifactIds: (input) => [input.artifact_id], + handler: { + operation: "export_video", + run: async (context) => { + await requireTools(); + return runExportVideo(context); + }, + }, }; - return [ - { - operation: "edit_video" as const, - inputSchema: EditVideoInputSchema, - title: (input: EditVideoInput) => input.title, - available, - handler: editVideo, - }, - { - operation: "export_video" as const, - inputSchema: ExportVideoInputSchema, - title: (input: ExportVideoInput) => - input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "), - available, - handler: exportVideo, - }, - ]; + return [editVideo, exportVideo]; } -async function runEditVideo( - context: MediaJobContext, - tools: FfmpegInfo, -): Promise { +async function runEditVideo(context: MediaJobContext): Promise { const input = EditVideoInputSchema.parse(context.input); const workDir = await mkdtemp(join(context.tempDir, "edit-")); const progress = createProgressReporter(context); - const source = await loadVideo(context, tools, input.artifact_id); + const source = await loadVideo(context, input.artifact_id); const name = cleanFileName(input.title, cleanFileName(withoutExtension(source.name), "video")); - if (input.steps.every((step) => step.op === "poster")) { - const at = input.steps[0]?.op === "poster" ? input.steps[0].at : 0; + const [first] = input.steps; + if (input.steps.length === 1 && first?.op === "poster") { + const at = first.at; if (at * 1000 >= source.probe.durationMs) { throw new Error( `poster at ${at} s is past the end of the ${source.probe.durationMs / 1000} s video.`, @@ -110,7 +118,7 @@ async function runEditVideo( for (const step of input.steps) { if (step.op === "concat") { for (const id of step.artifact_ids) { - if (!clips.has(id)) clips.set(id, await loadVideo(context, tools, id)); + if (!clips.has(id)) clips.set(id, await loadVideo(context, id)); } } else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) { subtitleFiles.set( @@ -122,7 +130,7 @@ async function runEditVideo( const audioOnly = input.steps.at(-1)?.op === "extract_audio"; const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4"); const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath }); - const output = await encode(context, tools, plan, outputPath, workDir, progress.range(0, 1), { + const output = await encode(context, plan, outputPath, workDir, progress.range(0, 1), { name: `${name}${audioOnly ? ".m4a" : ".mp4"}`, parentArtifactId: source.artifactId, }); @@ -132,15 +140,12 @@ async function runEditVideo( }; } -async function runExportVideo( - context: MediaJobContext, - tools: FfmpegInfo, -): Promise { +async function runExportVideo(context: MediaJobContext): Promise { const input = ExportVideoInputSchema.parse(context.input); const presets = [...new Set(input.presets)]; const workDir = await mkdtemp(join(context.tempDir, "export-")); const progress = createProgressReporter(context); - const source = await loadVideo(context, tools, input.artifact_id); + const source = await loadVideo(context, input.artifact_id); const base = cleanFileName(withoutExtension(source.name), "video"); const outputs: MediaOutput[] = []; for (const [index, preset] of presets.entries()) { @@ -154,7 +159,6 @@ async function runExportVideo( outputs.push( await encode( context, - tools, plan, outputPath, workDir, @@ -174,7 +178,6 @@ async function runExportVideo( /** Runs a plan, checks the result, renders its poster and stores it as an output. */ async function encode( context: MediaJobContext, - tools: FfmpegInfo, plan: VideoEditPlan, outputPath: string, workDir: string, @@ -189,13 +192,11 @@ async function encode( onProgress: (fraction) => report(fraction * ENCODE_SHARE), maxOutputBytes: maxBytes, outputs: [outputPath], - ffmpegPath: tools.ffmpegPath, }); if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes)); const probe = await probeMedia(outputPath, { signal: context.signal, mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4", - ffprobePath: tools.ffprobePath, }); if (plan.kind === "audio") { const output = await context.addOutput({ @@ -228,7 +229,6 @@ async function encode( async function loadVideo( context: MediaJobContext, - tools: FfmpegInfo, artifactId: string, ): Promise { const file = await context.fetchArtifact(artifactId); @@ -236,7 +236,6 @@ async function loadVideo( const probe = await probeMedia(file.path, { signal: context.signal, mimeType: file.mimeType, - ffprobePath: tools.ffprobePath, }); if (!probe.video) throw new Error(`${file.name} has no video.`); if (probe.durationMs <= 0) throw new Error(`Could not read the length of ${file.name}.`); @@ -295,33 +294,27 @@ function withoutExtension(name: string): string { return name.replace(/\.[A-Za-z0-9]{1,5}$/, ""); } +/** "name (1080×1920, 12.5 s)" for the bot's summary; the runtime adds the artifact ids. */ function describe(output: MediaOutput): string { - const size = output.width && output.height ? ` ${output.width}×${output.height}` : ""; - const length = - output.durationMs !== undefined ? ` ${Number((output.durationMs / 1000).toFixed(1))} s` : ""; - return `${output.name}${size}${length} (artifact ${output.artifactId})`; + const details = [ + output.width && output.height ? `${output.width}×${output.height}` : "", + output.durationMs !== undefined ? `${Number((output.durationMs / 1000).toFixed(1))} s` : "", + ].filter(Boolean); + return details.length ? `${output.name} (${details.join(", ")})` : output.name; } -/** - * Maps ffmpeg progress onto the card: monotonic, at most once a second, written in order and - * never failing the job when a write fails. - */ +/** Maps ffmpeg progress onto the card: monotonic, written in order, never failing the job. */ function createProgressReporter(context: MediaJobContext) { let reported = 0; - let reportedAt = 0; let writes = Promise.resolve(); const write = (fraction: number) => { reported = fraction; - reportedAt = Date.now(); writes = writes.then(() => context.progress(fraction)).catch(() => undefined); }; return { range: (from: number, to: number) => (fraction: number) => { const value = Math.min(1, from + (to - from) * Math.min(1, Math.max(0, fraction))); - const boundary = fraction >= 1; - if (value <= reported) return; - if (!boundary && Date.now() - reportedAt < PROGRESS_INTERVAL_MS) return; - write(value); + if (value > reported) write(value); }, done: async () => { if (reported < 1) write(1); diff --git a/packages/contracts/src/media-video.test.ts b/packages/contracts/src/media-video.test.ts new file mode 100644 index 0000000..eb3fc17 --- /dev/null +++ b/packages/contracts/src/media-video.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { EditVideoInputSchema, ExportVideoInputSchema } from "./media-video.js"; + +const edit = (steps: unknown[]) => + EditVideoInputSchema.safeParse({ artifact_id: "a", title: "Reel", steps }); + +describe("edit_video input", () => { + it("accepts ordered steps and fills defaults", () => { + const parsed = EditVideoInputSchema.parse({ + artifact_id: "a", + title: " Reel ", + steps: [ + { op: "trim", start: 1, end: 5 }, + { op: "reframe", preset: "vertical" }, + { op: "extract_audio" }, + ], + }); + expect(parsed.title).toBe("Reel"); + expect(parsed.steps[1]).toEqual({ op: "reframe", preset: "vertical", fit: "crop" }); + // The job stores the parsed input and parses it again when it runs. + expect(EditVideoInputSchema.parse(parsed)).toEqual(parsed); + }); + + it("rejects malformed step lists", () => { + expect(edit([]).success).toBe(false); + expect(edit([{ op: "trim", start: 5, end: 5 }]).success).toBe(false); + expect(edit([{ op: "keep", segments: [{ start: 3, end: 1 }] }]).success).toBe(false); + expect(edit([{ op: "speed", factor: 8 }]).success).toBe(false); + expect(edit([{ op: "extract_audio" }, { op: "mute" }]).success).toBe(false); + expect(edit([{ op: "poster", at: 1 }, { op: "extract_audio" }]).success).toBe(false); + expect( + edit([ + { op: "poster", at: 1 }, + { op: "poster", at: 2 }, + ]).success, + ).toBe(false); + expect(edit([{ op: "blur" }]).success).toBe(false); + expect( + EditVideoInputSchema.safeParse({ artifact_id: "a", steps: [{ op: "mute" }] }).success, + ).toBe(false); + }); +}); + +describe("export_video input", () => { + it("needs at least one preset and defaults to cropping", () => { + expect(ExportVideoInputSchema.parse({ artifact_id: "a", presets: ["square"] })).toEqual({ + artifact_id: "a", + presets: ["square"], + fit: "crop", + }); + expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: [] }).success).toBe(false); + expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: ["imax"] }).success).toBe( + false, + ); + }); +});