feat(media): register video operations with the media runtime
edit_video and export_video now run as media jobs: both registrations join the shared operation list the API and the worker build their runtime from, name every artifact they read (source, appended clips, subtitle files) so starting a job refuses artifacts the user cannot read, and their tool schemas use the runtime's continue_after helper. Container deployments get ffmpeg, ffprobe and the subtitle font in the API/worker image. Tests cover the handlers against a fake job context: cuts, blurred 9:16 reframes with subtitles, appended silent clips, audio extraction, single frames, batch exports with posters and progress, rotated phone video, clear input errors and cancellation. They skip where ffmpeg is not installed; the planner and parser tests always run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
parent
d8c324a7af
commit
44702e3770
@ -1,4 +1,8 @@
|
||||
FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a
|
||||
# ffmpeg/ffprobe cut, reframe and export videos; DejaVu is the burned-in subtitle font.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ffmpeg fonts-dejavu-core \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN corepack enable
|
||||
WORKDIR /app
|
||||
# Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise
|
||||
|
||||
@ -10,24 +10,15 @@ import {
|
||||
USER_FACT_CATEGORIES,
|
||||
} from "@rakazo/contracts";
|
||||
import { z } from "zod";
|
||||
import { mediaToolInputSchema } from "./media-tools.js";
|
||||
|
||||
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
|
||||
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
|
||||
|
||||
/** A media tool's arguments: its operation's input plus whether to wake the bot when it ends. */
|
||||
function mediaToolSchema(input: z.ZodType): Record<string, unknown> {
|
||||
const schema = z.toJSONSchema(input, { io: "input" });
|
||||
return {
|
||||
...schema,
|
||||
properties: {
|
||||
...schema.properties,
|
||||
continue_after: {
|
||||
type: "boolean",
|
||||
description:
|
||||
"true to be woken with the result when the job ends, to continue a multi-step task. Default false: the card alone shows the result.",
|
||||
},
|
||||
},
|
||||
};
|
||||
/** A media tool's JSON schema, generated from its operation's input schema. */
|
||||
function mediaToolSchemaOf(input: z.ZodType): ConnectorTool["inputSchema"] {
|
||||
const { properties = {}, required } = z.toJSONSchema(input, { io: "input" });
|
||||
return mediaToolInputSchema({ properties, required });
|
||||
}
|
||||
|
||||
const VIDEO_PRESETS_AND_FITS =
|
||||
@ -350,12 +341,12 @@ export const builtinAgentTools: ConnectorTool[] = [
|
||||
{
|
||||
name: "edit_video",
|
||||
description: `Edit a video artifact in a background job; a media card shows progress and the result, a new version of the source. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4 with a poster. title: a few words in the user's language for the card and file name.`,
|
||||
inputSchema: mediaToolSchema(EditVideoInputSchema),
|
||||
inputSchema: mediaToolSchemaOf(EditVideoInputSchema),
|
||||
},
|
||||
{
|
||||
name: "export_video",
|
||||
description: `Export a video artifact to one or more presets in a background job: one H.264/AAC MP4 with a poster per preset, each a new version of the source, shown on a media card. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`,
|
||||
inputSchema: mediaToolSchema(ExportVideoInputSchema),
|
||||
inputSchema: mediaToolSchemaOf(ExportVideoInputSchema),
|
||||
},
|
||||
{
|
||||
name: "add_mcp_server",
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import type { MediaSettings } from "@rakazo/contracts";
|
||||
import { createVideoOperations } from "./media/video-operations.js";
|
||||
import type { MediaOperationRegistration } from "./media-runtime.js";
|
||||
|
||||
export interface MediaOperationDeps {
|
||||
@ -11,5 +12,5 @@ export interface MediaOperationDeps {
|
||||
* from this one list, so both processes run the same operations.
|
||||
*/
|
||||
export function createMediaOperations(_deps: MediaOperationDeps): MediaOperationRegistration[] {
|
||||
return [];
|
||||
return [...createVideoOperations()];
|
||||
}
|
||||
|
||||
@ -52,7 +52,6 @@ export interface RunFfmpegOptions {
|
||||
outputs?: readonly string[];
|
||||
/** Kills ffmpeg when it reports no progress for this long. */
|
||||
stallTimeoutMs?: number;
|
||||
ffmpegPath?: string;
|
||||
}
|
||||
|
||||
export class FfmpegError extends Error {
|
||||
@ -162,9 +161,9 @@ export function ffmpegInputArgs(
|
||||
|
||||
export async function probeMedia(
|
||||
path: string,
|
||||
options: { signal?: AbortSignal; mimeType?: string; ffprobePath?: string } = {},
|
||||
options: { signal?: AbortSignal; mimeType?: string } = {},
|
||||
): Promise<MediaProbe> {
|
||||
const ffprobePath = options.ffprobePath ?? (await requireFfmpeg()).ffprobePath;
|
||||
const { ffprobePath } = await requireFfmpeg();
|
||||
const input = ffmpegInputArgs(path, { mimeType: options.mimeType });
|
||||
const result = await capture(
|
||||
ffprobePath,
|
||||
@ -325,7 +324,7 @@ export async function runFfmpeg(
|
||||
options: RunFfmpegOptions = {},
|
||||
): Promise<void> {
|
||||
options.signal?.throwIfAborted();
|
||||
const ffmpegPath = options.ffmpegPath ?? (await requireFfmpeg()).ffmpegPath;
|
||||
const { ffmpegPath } = await requireFfmpeg();
|
||||
try {
|
||||
await spawnFfmpeg(ffmpegPath, args, options);
|
||||
} catch (error) {
|
||||
@ -469,12 +468,7 @@ export async function extractAudio(
|
||||
onProgress?: (fraction: number) => void;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
const tools = await requireFfmpeg();
|
||||
const probe = await probeMedia(input, {
|
||||
signal: options.signal,
|
||||
mimeType: options.mimeType,
|
||||
ffprobePath: tools.ffprobePath,
|
||||
});
|
||||
const probe = await probeMedia(input, { signal: options.signal, mimeType: options.mimeType });
|
||||
if (!probe.hasAudio) throw new Error("The file has no audio track.");
|
||||
const startMs = Math.max(0, options.startMs ?? 0);
|
||||
const endMs = Math.min(probe.durationMs, options.endMs ?? probe.durationMs);
|
||||
@ -505,7 +499,6 @@ export async function extractAudio(
|
||||
durationMs: endMs - startMs,
|
||||
onProgress: options.onProgress,
|
||||
outputs: [output],
|
||||
ffmpegPath: tools.ffmpegPath,
|
||||
},
|
||||
);
|
||||
}
|
||||
@ -516,7 +509,6 @@ export async function renderPoster(
|
||||
output: string,
|
||||
options: { signal?: AbortSignal; mimeType?: string; atMs?: number } = {},
|
||||
): Promise<void> {
|
||||
const ffmpegPath = (await requireFfmpeg()).ffmpegPath;
|
||||
const render = (atMs: number) =>
|
||||
runFfmpeg(
|
||||
[
|
||||
@ -537,7 +529,7 @@ export async function renderPoster(
|
||||
"1",
|
||||
`file:${output}`,
|
||||
],
|
||||
{ signal: options.signal, outputs: [output], ffmpegPath },
|
||||
{ signal: options.signal, outputs: [output] },
|
||||
);
|
||||
const atMs = Math.max(0, options.atMs ?? 0);
|
||||
await render(atMs);
|
||||
|
||||
289
packages/adapters/src/media/video-operations.test.ts
Normal file
289
packages/adapters/src/media/video-operations.test.ts
Normal file
@ -0,0 +1,289 @@
|
||||
import { rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { builtinAgentTools } from "../builtin-tools.js";
|
||||
import { createMediaOperations } from "../media-operations.js";
|
||||
import { MEDIA_CONTINUE_AFTER_PROPERTY } from "../media-tools.js";
|
||||
import { parametersFor } from "../pi-runtime.js";
|
||||
import { detectFfmpeg, probeMedia, runFfmpeg } from "./ffmpeg.js";
|
||||
import type { TestArtifact, TestClips } from "./media-test-support.js";
|
||||
import { createTestClips, fakeMediaJobContext } from "./media-test-support.js";
|
||||
import { cleanFileName, createVideoOperations, subtitleFormat } from "./video-operations.js";
|
||||
|
||||
const ffmpeg = await detectFfmpeg();
|
||||
|
||||
function operation(name: "edit_video" | "export_video", deps = {}) {
|
||||
const registration = createVideoOperations(deps).find((entry) => entry.operation === name);
|
||||
if (!registration) throw new Error(`missing ${name}`);
|
||||
return registration;
|
||||
}
|
||||
|
||||
describe("video operation registrations", () => {
|
||||
it("are registered for the runtime and unavailable without ffmpeg", async () => {
|
||||
const names = createMediaOperations({ loadSettings: async () => ({}) as never }).map(
|
||||
(entry) => entry.operation,
|
||||
);
|
||||
expect(names).toEqual(expect.arrayContaining(["edit_video", "export_video"]));
|
||||
const edit = operation("edit_video", { detect: async () => null });
|
||||
expect(await edit.available({ spaceId: "space-1" })).toBe(false);
|
||||
const { context } = await fakeMediaJobContext({}, {});
|
||||
await expect(edit.handler.run(context)).rejects.toThrow(
|
||||
"ffmpeg and ffprobe are not installed on this server.",
|
||||
);
|
||||
});
|
||||
|
||||
it("names every artifact the input reads so start() can authorize them", () => {
|
||||
const edit = operation("edit_video");
|
||||
const input = edit.inputSchema.parse({
|
||||
artifact_id: "src",
|
||||
title: "Reel",
|
||||
steps: [
|
||||
{ op: "concat", artifact_ids: ["clip-1", "clip-2"] },
|
||||
{ op: "subtitles", artifact_id: "subs" },
|
||||
],
|
||||
});
|
||||
expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs"]);
|
||||
const exported = operation("export_video");
|
||||
const exportInput = exported.inputSchema.parse({ artifact_id: "src", presets: ["vertical"] });
|
||||
expect(exported.artifactIds?.(exportInput)).toEqual(["src"]);
|
||||
});
|
||||
|
||||
it("titles exports by aspect ratio unless the bot gave a title", () => {
|
||||
const exported = operation("export_video");
|
||||
expect(
|
||||
exported.title(
|
||||
exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical", "square"] }),
|
||||
),
|
||||
).toBe("9:16 · 1:1");
|
||||
expect(
|
||||
exported.title(
|
||||
exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical"], title: "Reel" }),
|
||||
),
|
||||
).toBe("Reel");
|
||||
});
|
||||
|
||||
it("defines tools named like the operations, with continue_after, that the runtime can encode", () => {
|
||||
for (const name of ["edit_video", "export_video"]) {
|
||||
const tool = builtinAgentTools.find((entry) => entry.name === name);
|
||||
expect(tool).toBeDefined();
|
||||
expect(tool?.inputSchema.properties).toMatchObject(MEDIA_CONTINUE_AFTER_PROPERTY);
|
||||
expect(() => parametersFor(tool!)).not.toThrow();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("subtitle and file name helpers", () => {
|
||||
it("recognizes SRT and VTT by content only", () => {
|
||||
expect(subtitleFormat("WEBVTT\n\n00:00.000 --> 00:01.000\nHi")).toBe("vtt");
|
||||
expect(subtitleFormat("1\r\n00:00:01,000 --> 00:00:02,000\r\nHi")).toBe("srt");
|
||||
expect(subtitleFormat("00:00:01.000 --> 00:00:02.000\nHi")).toBe("srt");
|
||||
expect(subtitleFormat("#EXTM3U\n#EXTINF:1,\nfile:/etc/passwd")).toBeNull();
|
||||
expect(subtitleFormat("WEBVTTX")).toBeNull();
|
||||
});
|
||||
|
||||
it("cleans file names without losing the user's language", () => {
|
||||
expect(cleanFileName("Reel: Sommer/Urlaub 2026", "video")).toBe("Reel Sommer Urlaub 2026");
|
||||
expect(cleanFileName("../..\u0000", "video")).toBe("video");
|
||||
expect(cleanFileName("x".repeat(200), "video")).toHaveLength(80);
|
||||
});
|
||||
});
|
||||
|
||||
// CI runners may not have ffmpeg; these run wherever it is installed.
|
||||
describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => {
|
||||
let clips: TestClips;
|
||||
let artifacts: Record<string, TestArtifact>;
|
||||
|
||||
beforeAll(async () => {
|
||||
clips = await createTestClips();
|
||||
artifacts = {
|
||||
land: { path: clips.landscape, name: "Landscape clip.mp4", mimeType: "video/mp4" },
|
||||
silent: { path: clips.silentPortrait, name: "silent.mp4", mimeType: "video/mp4" },
|
||||
srt: { path: clips.srt, name: "captions.srt", mimeType: "application/x-subrip" },
|
||||
vtt: { path: clips.vtt, name: "captions.vtt", mimeType: "text/vtt" },
|
||||
notes: { path: clips.landscape, name: "notes.txt", mimeType: "text/plain" },
|
||||
};
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (clips) await rm(clips.dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function run(name: "edit_video" | "export_video", input: Record<string, unknown>) {
|
||||
const registration = operation(name);
|
||||
const job = await fakeMediaJobContext(registration.inputSchema.parse(input), artifacts);
|
||||
try {
|
||||
const result = await registration.handler.run(job.context);
|
||||
return { ...job, result };
|
||||
} finally {
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it("cuts, reframes to 9:16 with a blurred fill, burns in subtitles and speeds up", async () => {
|
||||
const { outputs, progress, result } = await run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "Sommer-Reel",
|
||||
steps: [
|
||||
{ op: "trim", start: 0.5, end: 2.5 },
|
||||
{ op: "reframe", preset: "vertical", fit: "blur" },
|
||||
{ op: "subtitles", artifact_id: "srt" },
|
||||
{ op: "speed", factor: 2 },
|
||||
{ op: "poster", at: 0.5 },
|
||||
],
|
||||
});
|
||||
expect(outputs).toHaveLength(1);
|
||||
const [output] = outputs;
|
||||
expect(output).toMatchObject({
|
||||
name: "Sommer-Reel.mp4",
|
||||
mimeType: "video/mp4",
|
||||
width: 1080,
|
||||
height: 1920,
|
||||
parentArtifactId: "land",
|
||||
});
|
||||
expect(output!.posterSize).toBeGreaterThan(0);
|
||||
expect(Math.abs(output!.durationMs! - 1000)).toBeLessThan(150);
|
||||
expect(progress.at(-1)).toBe(1);
|
||||
expect(progress).toEqual([...progress].sort((a, b) => a - b));
|
||||
expect(result.summary).toContain("Sommer-Reel.mp4 (1080×1920, ");
|
||||
});
|
||||
|
||||
it("keeps segments, appends a silent clip with generated silence, and changes volume", async () => {
|
||||
const registration = operation("edit_video");
|
||||
const job = await fakeMediaJobContext(
|
||||
registration.inputSchema.parse({
|
||||
artifact_id: "land",
|
||||
title: "Cut",
|
||||
steps: [
|
||||
{
|
||||
op: "keep",
|
||||
segments: [
|
||||
{ start: 0, end: 0.5 },
|
||||
{ start: 2, end: 2.8 },
|
||||
],
|
||||
},
|
||||
{ op: "concat", artifact_ids: ["silent"] },
|
||||
{ op: "volume", gain: 0.5 },
|
||||
{ op: "subtitles", artifact_id: "vtt" },
|
||||
],
|
||||
}),
|
||||
artifacts,
|
||||
);
|
||||
await registration.handler.run(job.context);
|
||||
const [output] = job.outputs;
|
||||
const probe = await probeMedia(output!.path, { mimeType: "video/mp4" });
|
||||
expect(probe.hasAudio).toBe(true);
|
||||
expect(probe.video).toMatchObject({ width: 320, height: 180, rotation: 0 });
|
||||
expect(Math.abs(probe.durationMs - 3300)).toBeLessThan(200);
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("plays rotated phone video upright without rotating it twice", async (context) => {
|
||||
const rotated = join(clips.dir, "rotated.mp4");
|
||||
try {
|
||||
await runFfmpeg([
|
||||
...["-display_rotation", "90", "-i", `file:${clips.landscape}`],
|
||||
...["-c", "copy", `file:${rotated}`],
|
||||
]);
|
||||
} catch {
|
||||
context.skip("this ffmpeg cannot write rotation metadata");
|
||||
}
|
||||
const registration = operation("edit_video");
|
||||
const job = await fakeMediaJobContext(
|
||||
registration.inputSchema.parse({
|
||||
artifact_id: "rotated",
|
||||
title: "Upright",
|
||||
steps: [{ op: "trim", start: 0, end: 1 }],
|
||||
}),
|
||||
{ rotated: { path: rotated, name: "rotated.mp4", mimeType: "video/mp4" } },
|
||||
);
|
||||
await registration.handler.run(job.context);
|
||||
const probe = await probeMedia(job.outputs[0]!.path, { mimeType: "video/mp4" });
|
||||
expect(probe.video).toMatchObject({ width: 180, height: 320, rotation: 0 });
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("extracts the audio as M4A", async () => {
|
||||
const { outputs } = await run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "Ton",
|
||||
steps: [{ op: "trim", start: 1, end: 2 }, { op: "extract_audio" }],
|
||||
});
|
||||
expect(outputs).toEqual([
|
||||
expect.objectContaining({ name: "Ton.m4a", mimeType: "audio/mp4", parentArtifactId: "land" }),
|
||||
]);
|
||||
expect(outputs[0]?.posterPath).toBeUndefined();
|
||||
expect(Math.abs(outputs[0]!.durationMs! - 1000)).toBeLessThan(100);
|
||||
});
|
||||
|
||||
it("saves a single frame when poster is the only step", async () => {
|
||||
const { outputs } = await run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "Cover",
|
||||
steps: [{ op: "poster", at: 1 }],
|
||||
});
|
||||
expect(outputs).toEqual([
|
||||
expect.objectContaining({
|
||||
name: "Cover.jpg",
|
||||
mimeType: "image/jpeg",
|
||||
width: 320,
|
||||
height: 180,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("exports one output per distinct preset with posters and batch progress", async () => {
|
||||
const { outputs, progress } = await run("export_video", {
|
||||
artifact_id: "land",
|
||||
presets: ["square", "portrait", "square"],
|
||||
fit: "pad",
|
||||
});
|
||||
expect(
|
||||
outputs.map((output) => [output.name, output.preset, output.width, output.height]),
|
||||
).toEqual([
|
||||
["Landscape clip-1x1.mp4", "square", 1080, 1080],
|
||||
["Landscape clip-4x5.mp4", "portrait", 1080, 1350],
|
||||
]);
|
||||
for (const output of outputs) {
|
||||
expect(output.parentArtifactId).toBe("land");
|
||||
expect(output.posterSize).toBeGreaterThan(0);
|
||||
}
|
||||
expect(progress).toContain(0.5);
|
||||
expect(progress.at(-1)).toBe(1);
|
||||
});
|
||||
|
||||
it("fails clearly on ranges past the end, non-subtitle files and non-videos", async () => {
|
||||
await expect(
|
||||
run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "x",
|
||||
steps: [{ op: "trim", start: 5, end: 6 }],
|
||||
}),
|
||||
).rejects.toThrow("trim 5–6 s is outside the 3 s video.");
|
||||
await expect(
|
||||
run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "x",
|
||||
steps: [{ op: "subtitles", artifact_id: "land" }],
|
||||
}),
|
||||
).rejects.toThrow("Landscape clip.mp4 is not an SRT or VTT file.");
|
||||
await expect(
|
||||
run("export_video", { artifact_id: "notes", presets: ["square"] }),
|
||||
).rejects.toThrow("notes.txt is not a video.");
|
||||
});
|
||||
|
||||
it("stops ffmpeg and stores nothing when the job is cancelled", async () => {
|
||||
const registration = operation("export_video");
|
||||
const job = await fakeMediaJobContext(
|
||||
registration.inputSchema.parse({ artifact_id: "land", presets: ["cinema"], fit: "blur" }),
|
||||
artifacts,
|
||||
);
|
||||
const progress = job.context.progress;
|
||||
job.context.progress = async (fraction) => {
|
||||
await progress(fraction);
|
||||
job.controller.abort(new Error("cancelled"));
|
||||
};
|
||||
await expect(registration.handler.run(job.context)).rejects.toThrow("cancelled");
|
||||
expect(job.outputs).toEqual([]);
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
@ -1,11 +1,6 @@
|
||||
import { copyFile, mkdtemp, open, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type {
|
||||
MediaJobContext,
|
||||
MediaJobInputFile,
|
||||
MediaOperationHandler,
|
||||
MediaOperationResult,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { MediaJobContext, MediaJobInputFile, MediaOperationResult } from "@rakazo/adapter-kit";
|
||||
import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts";
|
||||
import {
|
||||
ARTIFACT_MAX_BYTES_BY_KIND,
|
||||
@ -13,6 +8,7 @@ import {
|
||||
ExportVideoInputSchema,
|
||||
MEDIA_PRESETS,
|
||||
} from "@rakazo/contracts";
|
||||
import type { MediaOperationRegistration } from "../media-runtime.js";
|
||||
import type { FfmpegInfo } from "./ffmpeg.js";
|
||||
import {
|
||||
detectFfmpeg,
|
||||
@ -29,59 +25,71 @@ export interface VideoOperationsDeps {
|
||||
detect?: () => Promise<FfmpegInfo | null>;
|
||||
}
|
||||
|
||||
const PROGRESS_INTERVAL_MS = 1_000;
|
||||
/** Share of each output's progress spent encoding; the rest is probing and the poster. */
|
||||
const ENCODE_SHARE = 0.95;
|
||||
const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
/** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */
|
||||
export function createVideoOperations(deps: VideoOperationsDeps = {}) {
|
||||
export function createVideoOperations(
|
||||
deps: VideoOperationsDeps = {},
|
||||
): MediaOperationRegistration[] {
|
||||
const detect = deps.detect ?? detectFfmpeg;
|
||||
const available = async () => (await detect()) !== null;
|
||||
const tools = async () => {
|
||||
const info = await detect();
|
||||
if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server.");
|
||||
return info;
|
||||
const requireTools = async () => {
|
||||
if (!(await available()))
|
||||
throw new Error("ffmpeg and ffprobe are not installed on this server.");
|
||||
};
|
||||
const editVideo: MediaOperationHandler = {
|
||||
const editVideo: MediaOperationRegistration<EditVideoInput> = {
|
||||
operation: "edit_video",
|
||||
run: async (context) => runEditVideo(context, await tools()),
|
||||
inputSchema: EditVideoInputSchema,
|
||||
title: (input) => input.title,
|
||||
available,
|
||||
artifactIds: (input) => [
|
||||
input.artifact_id,
|
||||
...input.steps.flatMap((step) =>
|
||||
step.op === "concat"
|
||||
? step.artifact_ids
|
||||
: step.op === "subtitles"
|
||||
? [step.artifact_id]
|
||||
: [],
|
||||
),
|
||||
],
|
||||
handler: {
|
||||
operation: "edit_video",
|
||||
run: async (context) => {
|
||||
await requireTools();
|
||||
return runEditVideo(context);
|
||||
},
|
||||
},
|
||||
};
|
||||
const exportVideo: MediaOperationHandler = {
|
||||
const exportVideo: MediaOperationRegistration<ExportVideoInput> = {
|
||||
operation: "export_video",
|
||||
run: async (context) => runExportVideo(context, await tools()),
|
||||
inputSchema: ExportVideoInputSchema,
|
||||
title: (input) =>
|
||||
input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "),
|
||||
available,
|
||||
artifactIds: (input) => [input.artifact_id],
|
||||
handler: {
|
||||
operation: "export_video",
|
||||
run: async (context) => {
|
||||
await requireTools();
|
||||
return runExportVideo(context);
|
||||
},
|
||||
},
|
||||
};
|
||||
return [
|
||||
{
|
||||
operation: "edit_video" as const,
|
||||
inputSchema: EditVideoInputSchema,
|
||||
title: (input: EditVideoInput) => input.title,
|
||||
available,
|
||||
handler: editVideo,
|
||||
},
|
||||
{
|
||||
operation: "export_video" as const,
|
||||
inputSchema: ExportVideoInputSchema,
|
||||
title: (input: ExportVideoInput) =>
|
||||
input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "),
|
||||
available,
|
||||
handler: exportVideo,
|
||||
},
|
||||
];
|
||||
return [editVideo, exportVideo];
|
||||
}
|
||||
|
||||
async function runEditVideo(
|
||||
context: MediaJobContext,
|
||||
tools: FfmpegInfo,
|
||||
): Promise<MediaOperationResult> {
|
||||
async function runEditVideo(context: MediaJobContext): Promise<MediaOperationResult> {
|
||||
const input = EditVideoInputSchema.parse(context.input);
|
||||
const workDir = await mkdtemp(join(context.tempDir, "edit-"));
|
||||
const progress = createProgressReporter(context);
|
||||
const source = await loadVideo(context, tools, input.artifact_id);
|
||||
const source = await loadVideo(context, input.artifact_id);
|
||||
const name = cleanFileName(input.title, cleanFileName(withoutExtension(source.name), "video"));
|
||||
|
||||
if (input.steps.every((step) => step.op === "poster")) {
|
||||
const at = input.steps[0]?.op === "poster" ? input.steps[0].at : 0;
|
||||
const [first] = input.steps;
|
||||
if (input.steps.length === 1 && first?.op === "poster") {
|
||||
const at = first.at;
|
||||
if (at * 1000 >= source.probe.durationMs) {
|
||||
throw new Error(
|
||||
`poster at ${at} s is past the end of the ${source.probe.durationMs / 1000} s video.`,
|
||||
@ -110,7 +118,7 @@ async function runEditVideo(
|
||||
for (const step of input.steps) {
|
||||
if (step.op === "concat") {
|
||||
for (const id of step.artifact_ids) {
|
||||
if (!clips.has(id)) clips.set(id, await loadVideo(context, tools, id));
|
||||
if (!clips.has(id)) clips.set(id, await loadVideo(context, id));
|
||||
}
|
||||
} else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) {
|
||||
subtitleFiles.set(
|
||||
@ -122,7 +130,7 @@ async function runEditVideo(
|
||||
const audioOnly = input.steps.at(-1)?.op === "extract_audio";
|
||||
const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4");
|
||||
const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath });
|
||||
const output = await encode(context, tools, plan, outputPath, workDir, progress.range(0, 1), {
|
||||
const output = await encode(context, plan, outputPath, workDir, progress.range(0, 1), {
|
||||
name: `${name}${audioOnly ? ".m4a" : ".mp4"}`,
|
||||
parentArtifactId: source.artifactId,
|
||||
});
|
||||
@ -132,15 +140,12 @@ async function runEditVideo(
|
||||
};
|
||||
}
|
||||
|
||||
async function runExportVideo(
|
||||
context: MediaJobContext,
|
||||
tools: FfmpegInfo,
|
||||
): Promise<MediaOperationResult> {
|
||||
async function runExportVideo(context: MediaJobContext): Promise<MediaOperationResult> {
|
||||
const input = ExportVideoInputSchema.parse(context.input);
|
||||
const presets = [...new Set(input.presets)];
|
||||
const workDir = await mkdtemp(join(context.tempDir, "export-"));
|
||||
const progress = createProgressReporter(context);
|
||||
const source = await loadVideo(context, tools, input.artifact_id);
|
||||
const source = await loadVideo(context, input.artifact_id);
|
||||
const base = cleanFileName(withoutExtension(source.name), "video");
|
||||
const outputs: MediaOutput[] = [];
|
||||
for (const [index, preset] of presets.entries()) {
|
||||
@ -154,7 +159,6 @@ async function runExportVideo(
|
||||
outputs.push(
|
||||
await encode(
|
||||
context,
|
||||
tools,
|
||||
plan,
|
||||
outputPath,
|
||||
workDir,
|
||||
@ -174,7 +178,6 @@ async function runExportVideo(
|
||||
/** Runs a plan, checks the result, renders its poster and stores it as an output. */
|
||||
async function encode(
|
||||
context: MediaJobContext,
|
||||
tools: FfmpegInfo,
|
||||
plan: VideoEditPlan,
|
||||
outputPath: string,
|
||||
workDir: string,
|
||||
@ -189,13 +192,11 @@ async function encode(
|
||||
onProgress: (fraction) => report(fraction * ENCODE_SHARE),
|
||||
maxOutputBytes: maxBytes,
|
||||
outputs: [outputPath],
|
||||
ffmpegPath: tools.ffmpegPath,
|
||||
});
|
||||
if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes));
|
||||
const probe = await probeMedia(outputPath, {
|
||||
signal: context.signal,
|
||||
mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4",
|
||||
ffprobePath: tools.ffprobePath,
|
||||
});
|
||||
if (plan.kind === "audio") {
|
||||
const output = await context.addOutput({
|
||||
@ -228,7 +229,6 @@ async function encode(
|
||||
|
||||
async function loadVideo(
|
||||
context: MediaJobContext,
|
||||
tools: FfmpegInfo,
|
||||
artifactId: string,
|
||||
): Promise<PlanInput & MediaJobInputFile> {
|
||||
const file = await context.fetchArtifact(artifactId);
|
||||
@ -236,7 +236,6 @@ async function loadVideo(
|
||||
const probe = await probeMedia(file.path, {
|
||||
signal: context.signal,
|
||||
mimeType: file.mimeType,
|
||||
ffprobePath: tools.ffprobePath,
|
||||
});
|
||||
if (!probe.video) throw new Error(`${file.name} has no video.`);
|
||||
if (probe.durationMs <= 0) throw new Error(`Could not read the length of ${file.name}.`);
|
||||
@ -295,33 +294,27 @@ function withoutExtension(name: string): string {
|
||||
return name.replace(/\.[A-Za-z0-9]{1,5}$/, "");
|
||||
}
|
||||
|
||||
/** "name (1080×1920, 12.5 s)" for the bot's summary; the runtime adds the artifact ids. */
|
||||
function describe(output: MediaOutput): string {
|
||||
const size = output.width && output.height ? ` ${output.width}×${output.height}` : "";
|
||||
const length =
|
||||
output.durationMs !== undefined ? ` ${Number((output.durationMs / 1000).toFixed(1))} s` : "";
|
||||
return `${output.name}${size}${length} (artifact ${output.artifactId})`;
|
||||
const details = [
|
||||
output.width && output.height ? `${output.width}×${output.height}` : "",
|
||||
output.durationMs !== undefined ? `${Number((output.durationMs / 1000).toFixed(1))} s` : "",
|
||||
].filter(Boolean);
|
||||
return details.length ? `${output.name} (${details.join(", ")})` : output.name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps ffmpeg progress onto the card: monotonic, at most once a second, written in order and
|
||||
* never failing the job when a write fails.
|
||||
*/
|
||||
/** Maps ffmpeg progress onto the card: monotonic, written in order, never failing the job. */
|
||||
function createProgressReporter(context: MediaJobContext) {
|
||||
let reported = 0;
|
||||
let reportedAt = 0;
|
||||
let writes = Promise.resolve();
|
||||
const write = (fraction: number) => {
|
||||
reported = fraction;
|
||||
reportedAt = Date.now();
|
||||
writes = writes.then(() => context.progress(fraction)).catch(() => undefined);
|
||||
};
|
||||
return {
|
||||
range: (from: number, to: number) => (fraction: number) => {
|
||||
const value = Math.min(1, from + (to - from) * Math.min(1, Math.max(0, fraction)));
|
||||
const boundary = fraction >= 1;
|
||||
if (value <= reported) return;
|
||||
if (!boundary && Date.now() - reportedAt < PROGRESS_INTERVAL_MS) return;
|
||||
write(value);
|
||||
if (value > reported) write(value);
|
||||
},
|
||||
done: async () => {
|
||||
if (reported < 1) write(1);
|
||||
|
||||
56
packages/contracts/src/media-video.test.ts
Normal file
56
packages/contracts/src/media-video.test.ts
Normal file
@ -0,0 +1,56 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { EditVideoInputSchema, ExportVideoInputSchema } from "./media-video.js";
|
||||
|
||||
const edit = (steps: unknown[]) =>
|
||||
EditVideoInputSchema.safeParse({ artifact_id: "a", title: "Reel", steps });
|
||||
|
||||
describe("edit_video input", () => {
|
||||
it("accepts ordered steps and fills defaults", () => {
|
||||
const parsed = EditVideoInputSchema.parse({
|
||||
artifact_id: "a",
|
||||
title: " Reel ",
|
||||
steps: [
|
||||
{ op: "trim", start: 1, end: 5 },
|
||||
{ op: "reframe", preset: "vertical" },
|
||||
{ op: "extract_audio" },
|
||||
],
|
||||
});
|
||||
expect(parsed.title).toBe("Reel");
|
||||
expect(parsed.steps[1]).toEqual({ op: "reframe", preset: "vertical", fit: "crop" });
|
||||
// The job stores the parsed input and parses it again when it runs.
|
||||
expect(EditVideoInputSchema.parse(parsed)).toEqual(parsed);
|
||||
});
|
||||
|
||||
it("rejects malformed step lists", () => {
|
||||
expect(edit([]).success).toBe(false);
|
||||
expect(edit([{ op: "trim", start: 5, end: 5 }]).success).toBe(false);
|
||||
expect(edit([{ op: "keep", segments: [{ start: 3, end: 1 }] }]).success).toBe(false);
|
||||
expect(edit([{ op: "speed", factor: 8 }]).success).toBe(false);
|
||||
expect(edit([{ op: "extract_audio" }, { op: "mute" }]).success).toBe(false);
|
||||
expect(edit([{ op: "poster", at: 1 }, { op: "extract_audio" }]).success).toBe(false);
|
||||
expect(
|
||||
edit([
|
||||
{ op: "poster", at: 1 },
|
||||
{ op: "poster", at: 2 },
|
||||
]).success,
|
||||
).toBe(false);
|
||||
expect(edit([{ op: "blur" }]).success).toBe(false);
|
||||
expect(
|
||||
EditVideoInputSchema.safeParse({ artifact_id: "a", steps: [{ op: "mute" }] }).success,
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("export_video input", () => {
|
||||
it("needs at least one preset and defaults to cropping", () => {
|
||||
expect(ExportVideoInputSchema.parse({ artifact_id: "a", presets: ["square"] })).toEqual({
|
||||
artifact_id: "a",
|
||||
presets: ["square"],
|
||||
fit: "crop",
|
||||
});
|
||||
expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: [] }).success).toBe(false);
|
||||
expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: ["imax"] }).success).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user