From 485090c391b72dfaba7a96e3eff4e633850a5ced Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:35:34 +0000 Subject: [PATCH] feat(media): edit and export video with ffmpeg Bots can now cut, reframe and export videos for every common format without a hosted service: ffmpeg and ffprobe from PATH do the work on the server. - ffmpeg wrapper: PATH detection, ffprobe JSON probing (rotation, pixel aspect, frame rate, audio, packet scan when the container has no duration), runs with -progress parsing, abort and stall kills, an output size cap, partial-output cleanup and short errors without temp paths. Inputs go through the file protocol with the demuxer forced from the MIME type, so a crafted upload cannot be probed as a playlist that reads other files or URLs. extractAudio (16 kHz mono WAV) and renderPoster for transcription and media cards. - A pure planner turns edit steps (trim, keep, concat, reframe, speed, volume, mute, subtitles, extract_audio, poster) into one filter_complex pass: leading cuts seek the input, appended clips are normalized to the frame, fps and 48 kHz stereo (silence where a clip has none), reframes crop, pad or blur to the presets, speed chains atempo, subtitles burn in from generated file names with a readable style that stays clear of the controls on 9:16 frames. H.264/AAC MP4 with yuv420p and faststart; location metadata is stripped. - edit_video and export_video operations and tool definitions; the inputs are shared contracts so the media card can start exports. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- packages/adapters/src/builtin-tools.ts | 31 + packages/adapters/src/index.ts | 2 + packages/adapters/src/media/ffmpeg.test.ts | 256 ++++++++ packages/adapters/src/media/ffmpeg.ts | 603 ++++++++++++++++++ .../adapters/src/media/media-test-support.ts | 106 +++ .../src/media/video-edit-plan.test.ts | 301 +++++++++ .../adapters/src/media/video-edit-plan.ts | 571 +++++++++++++++++ .../adapters/src/media/video-operations.ts | 331 ++++++++++ packages/contracts/src/index.ts | 1 + packages/contracts/src/media-video.ts | 80 +++ 10 files changed, 2282 insertions(+) create mode 100644 packages/adapters/src/media/ffmpeg.test.ts create mode 100644 packages/adapters/src/media/ffmpeg.ts create mode 100644 packages/adapters/src/media/media-test-support.ts create mode 100644 packages/adapters/src/media/video-edit-plan.test.ts create mode 100644 packages/adapters/src/media/video-edit-plan.ts create mode 100644 packages/adapters/src/media/video-operations.ts create mode 100644 packages/contracts/src/media-video.ts diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index 0ff52fa..3229a7d 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -2,6 +2,8 @@ import type { ConnectorTool } from "@rakazo/adapter-kit"; import { BotSecretDestination, BotSecretName, + EditVideoInputSchema, + ExportVideoInputSchema, MODEL_TIERS, SecretAskPurpose, SecretHttpRequest, @@ -12,6 +14,25 @@ import { z } from "zod"; export const SUBAGENT_MODEL_TIER_DESCRIPTION = "Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set."; +/** A media tool's arguments: its operation's input plus whether to wake the bot when it ends. */ +function mediaToolSchema(input: z.ZodType): Record { + const schema = z.toJSONSchema(input, { io: "input" }); + return { + ...schema, + properties: { + ...schema.properties, + continue_after: { + type: "boolean", + description: + "true to be woken with the result when the job ends, to continue a multi-step task. Default false: the card alone shows the result.", + }, + }, + }; +} + +const VIDEO_PRESETS_AND_FITS = + "Presets: vertical 9:16, landscape 16:9, square 1:1, portrait 4:5, classic 4:3, cinema 21:9. fit: crop (default) fills the frame and cuts the edges, pad adds bars, blur fills the bars with a blurred copy."; + export const DELEGATION_TOOL_NAMES = new Set([ "run_subagent", "spawn_bot", @@ -326,6 +347,16 @@ export const builtinAgentTools: ConnectorTool[] = [ }, }, }, + { + name: "edit_video", + description: `Edit a video artifact in a background job; a media card shows progress and the result, a new version of the source. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4 with a poster. title: a few words in the user's language for the card and file name.`, + inputSchema: mediaToolSchema(EditVideoInputSchema), + }, + { + name: "export_video", + description: `Export a video artifact to one or more presets in a background job: one H.264/AAC MP4 with a poster per preset, each a new version of the source, shown on a media card. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`, + inputSchema: mediaToolSchema(ExportVideoInputSchema), + }, { name: "add_mcp_server", description: diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 7b4ecd0..9e2c16d 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -76,6 +76,8 @@ export * from "./mcp-emulator.js"; export * from "./mcp-oauth.js"; export * from "./mcp-server-tool.js"; export * from "./mcp-transport.js"; +export * from "./media/ffmpeg.js"; +export * from "./media/video-operations.js"; export * from "./memory-provider-factory.js"; export * from "./messaging-context.js"; export * from "./messaging-delivery.js"; diff --git a/packages/adapters/src/media/ffmpeg.test.ts b/packages/adapters/src/media/ffmpeg.test.ts new file mode 100644 index 0000000..ab846da --- /dev/null +++ b/packages/adapters/src/media/ffmpeg.test.ts @@ -0,0 +1,256 @@ +import { existsSync } from "node:fs"; +import { rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { FfmpegProgress } from "./ffmpeg.js"; +import { + createFfmpegProgressParser, + detectFfmpeg, + extractAudio, + FfmpegError, + ffmpegInputArgs, + ffmpegSeconds, + packetDurationMs, + parseProbeOutput, + probeMedia, + renderPoster, + runFfmpeg, +} from "./ffmpeg.js"; +import type { TestClips } from "./media-test-support.js"; +import { createTestClips } from "./media-test-support.js"; + +const ffmpeg = await detectFfmpeg(); + +describe("ffmpeg progress", () => { + it("reports each block, tolerates split chunks and N/A times, and marks the end", () => { + const updates: FfmpegProgress[] = []; + const feed = createFfmpegProgressParser((update) => updates.push(update)); + feed("frame=0\nout_time_us=N/A\ntotal_size=48\nprogress=cont"); + feed("inue\nframe=75\nout_time_us=2995"); + feed("374\nout_time_ms=2995374\ntotal_size=42300\nprogress=end\n"); + expect(updates).toEqual([ + { totalSize: 48, done: false }, + { outTimeUs: 2_995_374, totalSize: 42_300, done: true }, + ]); + }); +}); + +describe("probe parsing", () => { + const stream = (fields: Record) => ({ + codec_type: "video", + codec_name: "h264", + ...fields, + }); + + it("swaps the size of rotated video and reads frame rate and audio", () => { + const probe = parseProbeOutput( + JSON.stringify({ + format: { duration: "12.345", format_name: "mov,mp4,m4a,3gp,3g2,mj2" }, + streams: [ + stream({ + width: 1920, + height: 1080, + avg_frame_rate: "30000/1001", + sample_aspect_ratio: "1:1", + side_data_list: [{ side_data_type: "Display Matrix", rotation: -90 }], + }), + { codec_type: "audio", codec_name: "aac", sample_rate: "48000", channels: 2 }, + ], + }), + ); + expect(probe).toEqual({ + durationMs: 12_345, + formatName: "mov,mp4,m4a,3gp,3g2,mj2", + video: { + codec: "h264", + width: 1080, + height: 1920, + rotation: 270, + fps: 30000 / 1001, + sampleAspectRatio: 1, + }, + audio: { codec: "aac", sampleRate: 48_000, channels: 2 }, + hasAudio: true, + }); + }); + + it("reads legacy rotate tags, skips cover art and falls back to stream durations", () => { + const probe = parseProbeOutput( + JSON.stringify({ + format: { format_name: "mov" }, + streams: [ + stream({ width: 600, height: 600, disposition: { attached_pic: 1 } }), + stream({ + width: 640, + height: 480, + avg_frame_rate: "0/0", + r_frame_rate: "90000/1", + sample_aspect_ratio: "4:3", + tags: { rotate: "90" }, + duration: "4.5", + }), + ], + }), + ); + expect(probe.durationMs).toBe(4500); + expect(probe.hasAudio).toBe(false); + expect(probe.video).toMatchObject({ + width: 480, + height: 640, + rotation: 90, + fps: 0, + sampleAspectRatio: 3 / 4, + }); + }); + + it("reads the end of the last packet when the container has no duration", () => { + expect(packetDurationMs("0.000000,0.040000\n2.960000,0.040000\nN/A,N/A\n")).toBe(3000); + expect(packetDurationMs("")).toBe(0); + }); +}); + +describe("ffmpeg arguments", () => { + it("opens inputs through the file protocol with the demuxer of their MIME type", () => { + expect( + ffmpegInputArgs("/a/b.webm", { + mimeType: "video/webm", + startSeconds: 2, + durationSeconds: 1.5, + }), + ).toEqual([ + ...["-ss", "2", "-t", "1.5", "-protocol_whitelist", "file", "-f", "matroska"], + ...["-i", "file:/a/b.webm"], + ]); + expect(ffmpegInputArgs("/a/b", { mimeType: "application/octet-stream" })).toEqual([ + ...["-protocol_whitelist", "file", "-i", "file:/a/b"], + ]); + expect(ffmpegSeconds(1e-7)).toBe("0"); + expect(ffmpegSeconds(12.34567)).toBe("12.346"); + }); +}); + +// CI runners may not have ffmpeg; these run wherever it is installed. +describe.skipIf(!ffmpeg)("ffmpeg integration", () => { + let clips: TestClips; + + beforeAll(async () => { + clips = await createTestClips(); + }); + + afterAll(async () => { + if (clips) await rm(clips.dir, { recursive: true, force: true }); + }); + + it("probes duration, size, frame rate and audio", async () => { + const probe = await probeMedia(clips.landscape, { mimeType: "video/mp4" }); + expect(probe.durationMs).toBeGreaterThanOrEqual(2900); + expect(probe.durationMs).toBeLessThanOrEqual(3100); + expect(probe.video).toMatchObject({ codec: "h264", width: 320, height: 180, fps: 25 }); + expect(probe.hasAudio).toBe(true); + const silent = await probeMedia(clips.silentPortrait, { mimeType: "video/mp4" }); + expect(silent.hasAudio).toBe(false); + expect(silent.video).toMatchObject({ width: 180, height: 320 }); + }); + + it("refuses a playlist disguised as an MP4 instead of following its URLs", async () => { + const playlist = join(clips.dir, "playlist.mp4"); + await writeFile( + playlist, + `#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:1.0,\nfile:${clips.srt}\n#EXT-X-ENDLIST\n`, + ); + await expect(probeMedia(playlist, { mimeType: "video/mp4" })).rejects.toThrow(FfmpegError); + }); + + it("reports progress up to 1 for a run", async () => { + const output = join(clips.dir, "progress.mp4"); + const fractions: number[] = []; + await runFfmpeg( + [ + ...ffmpegInputArgs(clips.landscape, { mimeType: "video/mp4" }), + ...[ + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-c:a", + "aac", + "-f", + "mp4", + `file:${output}`, + ], + ], + { durationMs: 3000, onProgress: (fraction) => fractions.push(fraction) }, + ); + expect(fractions.at(-1)).toBe(1); + expect(fractions).toEqual([...fractions].sort((a, b) => a - b)); + expect(existsSync(output)).toBe(true); + }); + + it("fails with the last stderr line, without directory paths, and removes outputs", async () => { + const output = join(clips.dir, "broken.mp4"); + await writeFile(output, "partial"); + const failure = runFfmpeg( + [ + ...ffmpegInputArgs(join(clips.dir, "missing.mp4"), { mimeType: "video/mp4" }), + `file:${output}`, + ], + { outputs: [output] }, + ); + const error = await failure.catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(FfmpegError); + expect((error as Error).message).toMatch(/^ffmpeg failed: .*missing\.mp4.*No such file/); + expect((error as Error).message).not.toContain(clips.dir); + expect(existsSync(output)).toBe(false); + }); + + it("kills ffmpeg on abort and removes the partial output", async () => { + const output = join(clips.dir, "aborted.mp4"); + const controller = new AbortController(); + const run = runFfmpeg( + [ + ...["-re", "-f", "lavfi", "-i", "testsrc=size=320x180:rate=25"], + ...["-t", "60", "-c:v", "libx264", "-preset", "ultrafast", "-f", "mp4", `file:${output}`], + ], + { + signal: controller.signal, + durationMs: 60_000, + onProgress: () => controller.abort(new Error("cancelled")), + outputs: [output], + }, + ); + await expect(run).rejects.toThrow("cancelled"); + expect(existsSync(output)).toBe(false); + }); + + it("stops when the output outgrows its cap", async () => { + const output = join(clips.dir, "too-large.mp4"); + const run = runFfmpeg( + [ + ...["-f", "lavfi", "-i", "testsrc=size=640x360:rate=25:duration=20"], + ...["-c:v", "libx264", "-preset", "ultrafast", "-crf", "0", "-f", "mp4", `file:${output}`], + ], + { maxOutputBytes: 64 * 1024, outputs: [output] }, + ); + await expect(run).rejects.toThrow("The result would exceed 0 MB."); + expect(existsSync(output)).toBe(false); + }); + + it("extracts 16 kHz mono WAV for transcription, optionally a range", async () => { + const output = join(clips.dir, "speech.wav"); + await extractAudio(clips.landscape, output, { mimeType: "video/mp4", startMs: 1000 }); + const probe = await probeMedia(output, { mimeType: "audio/wav" }); + expect(probe.audio).toEqual({ codec: "pcm_s16le", sampleRate: 16_000, channels: 1 }); + expect(probe.video).toBeUndefined(); + expect(Math.abs(probe.durationMs - 2000)).toBeLessThan(100); + await expect( + extractAudio(clips.silentPortrait, join(clips.dir, "none.wav"), { mimeType: "video/mp4" }), + ).rejects.toThrow("The file has no audio track."); + }); + + it("renders a poster frame, falling back to the first frame past the end", async () => { + const poster = join(clips.dir, "poster.jpg"); + await renderPoster(clips.landscape, poster, { mimeType: "video/mp4", atMs: 60_000 }); + const probe = await probeMedia(poster); + expect(probe.video).toMatchObject({ codec: "mjpeg", width: 320, height: 180 }); + }); +}); diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts new file mode 100644 index 0000000..b34e722 --- /dev/null +++ b/packages/adapters/src/media/ffmpeg.ts @@ -0,0 +1,603 @@ +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { access, rm, stat } from "node:fs/promises"; +import { setPriority } from "node:os"; +import { delimiter, join } from "node:path"; + +/** + * ffmpeg and ffprobe from PATH. Every call passes an argument array (never a shell), opens + * inputs through the `file` protocol only, and forces the demuxer from the artifact's MIME type + * so a crafted upload cannot make ffmpeg probe it as a playlist that reads other files or URLs. + */ + +export interface FfmpegInfo { + ffmpegPath: string; + ffprobePath: string; + version: string; +} + +export interface MediaProbe { + durationMs: number; + formatName: string; + /** Width and height as displayed: swapped when the stream is rotated by 90 or 270 degrees. */ + video?: { + codec: string; + width: number; + height: number; + rotation: 0 | 90 | 180 | 270; + /** Frames per second; 0 when unknown or implausible (variable frame rate files). */ + fps: number; + /** Pixel aspect ratio of the displayed frame; 1 for square pixels. */ + sampleAspectRatio: number; + }; + audio?: { codec: string; sampleRate: number; channels: number }; + hasAudio: boolean; +} + +export interface FfmpegProgress { + outTimeUs?: number; + totalSize?: number; + done: boolean; +} + +export interface RunFfmpegOptions { + signal?: AbortSignal; + /** Expected output duration; progress is reported as out_time / durationMs. */ + durationMs?: number; + onProgress?: (fraction: number) => void; + cwd?: string; + /** Kills ffmpeg once the output grows past this size. */ + maxOutputBytes?: number; + /** Files removed when the run fails or is aborted. */ + outputs?: readonly string[]; + /** Kills ffmpeg when it reports no progress for this long. */ + stallTimeoutMs?: number; + ffmpegPath?: string; +} + +export class FfmpegError extends Error { + constructor( + message: string, + readonly exitCode: number | null, + readonly stderrTail: string, + ) { + super(message); + this.name = "FfmpegError"; + } +} + +const DEFAULT_STALL_TIMEOUT_MS = 2 * 60_000; +const PROBE_TIMEOUT_MS = 60_000; +const STDERR_TAIL_BYTES = 8 * 1024; +const KILL_GRACE_MS = 5_000; +const MAX_PLAUSIBLE_FPS = 240; + +const DEMUXERS: Record = { + "video/mp4": "mov", + "video/quicktime": "mov", + "audio/mp4": "mov", + "video/webm": "matroska", + "audio/webm": "matroska", + "audio/mpeg": "mp3", + "audio/wav": "wav", + "audio/ogg": "ogg", +}; + +let detection: Promise | undefined; + +/** Finds ffmpeg and ffprobe on PATH once per process; null when either is missing or broken. */ +export function detectFfmpeg(): Promise { + detection ??= locateFfmpeg(); + return detection; +} + +async function locateFfmpeg(): Promise { + const [ffmpegPath, ffprobePath] = await Promise.all([ + findExecutable("ffmpeg"), + findExecutable("ffprobe"), + ]); + if (!ffmpegPath || !ffprobePath) return null; + try { + const [ffmpeg, ffprobe] = await Promise.all([ + capture(ffmpegPath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }), + capture(ffprobePath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }), + ]); + if (ffmpeg.code !== 0 || ffprobe.code !== 0) return null; + const version = /version (\S+)/.exec(ffmpeg.stdout)?.[1] ?? "unknown"; + return { ffmpegPath, ffprobePath, version }; + } catch { + return null; + } +} + +async function findExecutable(name: string): Promise { + const extensions = + process.platform === "win32" + ? (process.env.PATHEXT ?? ".EXE").split(";").filter(Boolean) + : [""]; + for (const directory of (process.env.PATH ?? "").split(delimiter)) { + if (!directory) continue; + for (const extension of extensions) { + const candidate = join(directory, `${name}${extension}`); + try { + await access(candidate, constants.X_OK); + if ((await stat(candidate)).isFile()) return candidate; + } catch { + // Not in this directory. + } + } + } + return null; +} + +export async function requireFfmpeg(): Promise { + const info = await detectFfmpeg(); + if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server."); + return info; +} + +/** ffmpeg time argument with millisecond precision and no exponent notation. */ +export function ffmpegSeconds(seconds: number): string { + return String(Number(Math.max(0, seconds).toFixed(3))); +} + +/** Input arguments for a local file: optional seek, file protocol only, forced demuxer. */ +export function ffmpegInputArgs( + path: string, + options: { mimeType?: string; startSeconds?: number; durationSeconds?: number } = {}, +): string[] { + const demuxer = options.mimeType ? DEMUXERS[options.mimeType] : undefined; + return [ + ...(options.startSeconds ? ["-ss", ffmpegSeconds(options.startSeconds)] : []), + ...(options.durationSeconds !== undefined + ? ["-t", ffmpegSeconds(options.durationSeconds)] + : []), + "-protocol_whitelist", + "file", + ...(demuxer ? ["-f", demuxer] : []), + "-i", + `file:${path}`, + ]; +} + +export async function probeMedia( + path: string, + options: { signal?: AbortSignal; mimeType?: string; ffprobePath?: string } = {}, +): Promise { + const ffprobePath = options.ffprobePath ?? (await requireFfmpeg()).ffprobePath; + const input = ffmpegInputArgs(path, { mimeType: options.mimeType }); + const result = await capture( + ffprobePath, + ["-v", "error", ...input, "-print_format", "json", "-show_format", "-show_streams"], + { signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS }, + ); + if (result.code !== 0) throw ffmpegFailure("ffprobe", result.code, result.stderr); + const probe = parseProbeOutput(result.stdout); + if (probe.durationMs > 0 || (!probe.video && !probe.audio)) return probe; + // Recordings streamed by browsers often carry no duration header; read it from the packets. + const packets = await capture( + ffprobePath, + [ + "-v", + "error", + ...input, + "-select_streams", + probe.video ? "v:0" : "a:0", + "-show_entries", + "packet=pts_time,duration_time", + "-of", + "csv=p=0", + ], + { signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS * 5 }, + ); + if (packets.code !== 0) throw ffmpegFailure("ffprobe", packets.code, packets.stderr); + return { ...probe, durationMs: packetDurationMs(packets.stdout) }; +} + +/** End of the last packet, from ffprobe `packet=pts_time,duration_time` CSV lines. */ +export function packetDurationMs(csv: string): number { + let end = 0; + for (const line of csv.split("\n")) { + const [pts, duration] = line.split(",").map(Number); + if (pts === undefined || !Number.isFinite(pts)) continue; + end = Math.max(end, pts + (duration !== undefined && Number.isFinite(duration) ? duration : 0)); + } + return Math.round(end * 1000); +} + +interface ProbeStream { + codec_type?: string; + codec_name?: string; + width?: number; + height?: number; + avg_frame_rate?: string; + r_frame_rate?: string; + sample_aspect_ratio?: string; + sample_rate?: string; + channels?: number; + duration?: string; + disposition?: { attached_pic?: number }; + tags?: { rotate?: string }; + side_data_list?: { rotation?: number }[]; +} + +/** Parses `ffprobe -print_format json -show_format -show_streams` output. */ +export function parseProbeOutput(json: string): MediaProbe { + const data = JSON.parse(json) as { + format?: { duration?: string; format_name?: string }; + streams?: ProbeStream[]; + }; + const streams = data.streams ?? []; + // Cover art in audio files is a single attached picture, not a video track. + const video = streams.find( + (stream) => stream.codec_type === "video" && stream.disposition?.attached_pic !== 1, + ); + const audio = streams.find((stream) => stream.codec_type === "audio"); + const seconds = + positive(Number(data.format?.duration)) ?? + Math.max(0, ...streams.map((stream) => positive(Number(stream.duration)) ?? 0)); + return { + durationMs: Math.round(seconds * 1000), + formatName: data.format?.format_name ?? "", + ...(video?.width && video.height + ? { video: probeVideo(video, video.width, video.height) } + : {}), + ...(audio + ? { + audio: { + codec: audio.codec_name ?? "", + sampleRate: Number(audio.sample_rate) || 0, + channels: audio.channels ?? 0, + }, + } + : {}), + hasAudio: Boolean(audio), + }; +} + +function probeVideo(stream: ProbeStream, width: number, height: number) { + const rotation = normalizeRotation( + stream.side_data_list?.find((entry) => typeof entry.rotation === "number")?.rotation ?? + Number(stream.tags?.rotate ?? 0), + ); + const sideways = rotation === 90 || rotation === 270; + const sar = ratio(stream.sample_aspect_ratio, ":") || 1; + const fps = ratio(stream.avg_frame_rate, "/") || ratio(stream.r_frame_rate, "/"); + return { + codec: stream.codec_name ?? "", + width: sideways ? height : width, + height: sideways ? width : height, + rotation, + fps: fps > 0 && fps <= MAX_PLAUSIBLE_FPS ? fps : 0, + sampleAspectRatio: sideways ? 1 / sar : sar, + }; +} + +function normalizeRotation(degrees: number): 0 | 90 | 180 | 270 { + if (!Number.isFinite(degrees)) return 0; + const quarter = (((Math.round(degrees / 90) % 4) + 4) % 4) as 0 | 1 | 2 | 3; + return ([0, 90, 180, 270] as const)[quarter]; +} + +function ratio(value: string | undefined, separator: string): number { + if (!value) return 0; + const [numerator, denominator] = value.split(separator).map(Number); + if (!numerator || !denominator || !Number.isFinite(numerator / denominator)) return 0; + return numerator / denominator; +} + +function positive(value: number): number | undefined { + return Number.isFinite(value) && value > 0 ? value : undefined; +} + +/** Feeds ffmpeg `-progress` output (key=value lines) and reports each completed block. */ +export function createFfmpegProgressParser( + onUpdate: (progress: FfmpegProgress) => void, +): (chunk: string) => void { + let buffer = ""; + let current: Omit = {}; + return (chunk) => { + buffer += chunk; + for (let newline = buffer.indexOf("\n"); newline >= 0; newline = buffer.indexOf("\n")) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + const separator = line.indexOf("="); + if (separator < 0) continue; + const key = line.slice(0, separator); + const value = line.slice(separator + 1).trim(); + const number = Number(value); + // out_time_ms is microseconds too (a long-standing ffmpeg naming bug). + if ((key === "out_time_us" || key === "out_time_ms") && value && number >= 0) { + current.outTimeUs = number; + } else if (key === "total_size" && value && number >= 0) { + current.totalSize = number; + } else if (key === "progress") { + onUpdate({ ...current, done: value === "end" }); + current = {}; + } + } + }; +} + +/** Runs ffmpeg with progress on stdout; rejects with the stderr tail when it fails. */ +export async function runFfmpeg( + args: readonly string[], + options: RunFfmpegOptions = {}, +): Promise { + options.signal?.throwIfAborted(); + const ffmpegPath = options.ffmpegPath ?? (await requireFfmpeg()).ffmpegPath; + try { + await spawnFfmpeg(ffmpegPath, args, options); + } catch (error) { + await Promise.all((options.outputs ?? []).map((path) => rm(path, { force: true }))); + throw error; + } +} + +function spawnFfmpeg( + ffmpegPath: string, + args: readonly string[], + options: RunFfmpegOptions, +): Promise { + const { signal, durationMs, onProgress, maxOutputBytes } = options; + return new Promise((resolve, reject) => { + const child = spawn( + ffmpegPath, + [ + "-hide_banner", + "-nostdin", + "-nostats", + "-loglevel", + "error", + "-progress", + "pipe:1", + "-y", + ...args, + ], + { cwd: options.cwd, stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + if (child.pid) { + try { + // Long encodes must not starve the API process that shares this machine. + setPriority(child.pid, 10); + } catch { + // Lowering priority is best effort. + } + } + let stderr = ""; + let settled = false; + let reported = 0; + let failure: unknown; + let stallTimer: NodeJS.Timeout | undefined; + const finish = (error?: unknown) => { + if (settled) return; + settled = true; + clearTimeout(stallTimer); + signal?.removeEventListener("abort", abort); + if (error === undefined) resolve(); + else reject(error); + }; + // Settles once the process is gone, so callers can delete its files right away. + const kill = (error: unknown) => { + if (failure !== undefined || settled) return; + failure = error; + clearTimeout(stallTimer); + child.kill("SIGKILL"); + setTimeout(() => finish(error), KILL_GRACE_MS).unref?.(); + }; + const abort = () => kill(signal?.reason ?? new Error("aborted")); + const stallTimeoutMs = options.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS; + const armStallTimer = () => { + clearTimeout(stallTimer); + stallTimer = setTimeout( + () => kill(new FfmpegError("ffmpeg stopped making progress.", null, tail(stderr))), + stallTimeoutMs, + ); + stallTimer.unref?.(); + }; + const parse = createFfmpegProgressParser((progress) => { + if (failure !== undefined) return; + if (maxOutputBytes !== undefined && (progress.totalSize ?? 0) > maxOutputBytes) { + kill(new Error(outputTooLargeMessage(maxOutputBytes))); + return; + } + if (!onProgress) return; + const fraction = progress.done + ? 1 + : durationMs && progress.outTimeUs !== undefined + ? Math.min(1, progress.outTimeUs / (durationMs * 1000)) + : 0; + if (fraction > reported) { + reported = fraction; + onProgress(fraction); + } + }); + signal?.addEventListener("abort", abort, { once: true }); + armStallTimer(); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + if (failure === undefined) armStallTimer(); + parse(chunk); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr = tail(stderr + chunk); + }); + child.on("error", (error) => finish(failure ?? error)); + child.on("close", (code) => { + if (failure !== undefined) finish(failure); + else if (code === 0) finish(); + else finish(ffmpegFailure("ffmpeg", code, stderr)); + }); + if (signal?.aborted) abort(); + }); +} + +export function outputTooLargeMessage(maxBytes: number): string { + return `The result would exceed ${Math.round(maxBytes / (1024 * 1024))} MB.`; +} + +function tail(text: string): string { + return text.length > STDERR_TAIL_BYTES ? text.slice(-STDERR_TAIL_BYTES) : text; +} + +function ffmpegFailure(tool: string, code: number | null, stderr: string): FfmpegError { + // The cause is usually the line before ffmpeg's generic summary ("Conversion failed!"). + const lines = stderr + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .slice(-2); + // Name files by their base name only; the job's temp paths mean nothing to the reader. + const detail = lines.join(" ").replaceAll(/(?:file:)?(?:[A-Za-z]:)?[\\/][^\s'"]*[\\/]/g, ""); + return new FfmpegError( + detail ? `${tool} failed: ${detail}` : `${tool} failed with exit code ${code}.`, + code, + tail(stderr), + ); +} + +/** Extracts the first audio track as 16 kHz mono 16-bit WAV, the format speech models expect. */ +export async function extractAudio( + input: string, + output: string, + options: { + signal?: AbortSignal; + mimeType?: string; + startMs?: number; + endMs?: number; + onProgress?: (fraction: number) => void; + } = {}, +): Promise { + const tools = await requireFfmpeg(); + const probe = await probeMedia(input, { + signal: options.signal, + mimeType: options.mimeType, + ffprobePath: tools.ffprobePath, + }); + if (!probe.hasAudio) throw new Error("The file has no audio track."); + const startMs = Math.max(0, options.startMs ?? 0); + const endMs = Math.min(probe.durationMs, options.endMs ?? probe.durationMs); + if (endMs <= startMs) throw new Error("The audio range is empty."); + await runFfmpeg( + [ + ...ffmpegInputArgs(input, { + mimeType: options.mimeType, + startSeconds: startMs / 1000, + durationSeconds: (endMs - startMs) / 1000, + }), + "-map", + "0:a:0", + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "pcm_s16le", + "-map_metadata", + "-1", + "-f", + "wav", + `file:${output}`, + ], + { + signal: options.signal, + durationMs: endMs - startMs, + onProgress: options.onProgress, + outputs: [output], + ffmpegPath: tools.ffmpegPath, + }, + ); +} + +/** Writes one JPEG frame at atMs (default 0), falling back to the first frame past the end. */ +export async function renderPoster( + input: string, + output: string, + options: { signal?: AbortSignal; mimeType?: string; atMs?: number } = {}, +): Promise { + const ffmpegPath = (await requireFfmpeg()).ffmpegPath; + const render = (atMs: number) => + runFfmpeg( + [ + ...ffmpegInputArgs(input, { mimeType: options.mimeType, startSeconds: atMs / 1000 }), + "-map", + "0:v:0", + "-frames:v", + "1", + "-c:v", + "mjpeg", + "-q:v", + "3", + "-map_metadata", + "-1", + "-f", + "image2", + "-update", + "1", + `file:${output}`, + ], + { signal: options.signal, outputs: [output], ffmpegPath }, + ); + const atMs = Math.max(0, options.atMs ?? 0); + await render(atMs); + // Seeking into the last frame's gap writes nothing; the first frame is better than no poster. + if (atMs > 0 && !(await nonEmptyFile(output))) await render(0); + if (!(await nonEmptyFile(output))) throw new Error("Could not render a poster frame."); +} + +async function nonEmptyFile(path: string): Promise { + try { + return (await stat(path)).size > 0; + } catch { + return false; + } +} + +interface CaptureResult { + code: number | null; + stdout: string; + stderr: string; +} + +function capture( + command: string, + args: readonly string[], + options: { signal?: AbortSignal; timeoutMs: number }, +): Promise { + options.signal?.throwIfAborted(); + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }); + const stdout: Buffer[] = []; + let stderr = ""; + let settled = false; + const finish = (error: unknown, result?: CaptureResult) => { + if (settled) return; + settled = true; + clearTimeout(timer); + options.signal?.removeEventListener("abort", abort); + if (result) resolve(result); + else reject(error); + }; + const kill = (error: unknown) => { + child.kill("SIGKILL"); + finish(error); + }; + const abort = () => kill(options.signal?.reason ?? new Error("aborted")); + const timer = setTimeout( + () => kill(new Error(`${command} timed out after ${options.timeoutMs} ms`)), + options.timeoutMs, + ); + timer.unref?.(); + options.signal?.addEventListener("abort", abort, { once: true }); + child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk)); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr = tail(stderr + chunk); + }); + child.on("error", (error) => finish(error)); + child.on("close", (code) => + finish(undefined, { code, stdout: Buffer.concat(stdout).toString("utf8"), stderr }), + ); + }); +} diff --git a/packages/adapters/src/media/media-test-support.ts b/packages/adapters/src/media/media-test-support.ts new file mode 100644 index 0000000..940419a --- /dev/null +++ b/packages/adapters/src/media/media-test-support.ts @@ -0,0 +1,106 @@ +import { copyFile, mkdtemp, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit"; +import type { MediaOutput, MediaOutputKind } from "@rakazo/contracts"; +import { attachmentKindOf } from "@rakazo/contracts"; +import { runFfmpeg } from "./ffmpeg.js"; + +export interface TestClips { + dir: string; + /** 320×180, 25 fps, 3 s, 440 Hz tone. */ + landscape: string; + /** 180×320, 30 fps, 2 s, no audio. */ + silentPortrait: string; + srt: string; + vtt: string; +} + +/** Tiny generated clips; nothing binary is committed. */ +export async function createTestClips(): Promise { + const dir = await mkdtemp(join(tmpdir(), "evoo-media-test-")); + const landscape = join(dir, "landscape.mp4"); + const silentPortrait = join(dir, "silent-portrait.mp4"); + const encode = ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"]; + await runFfmpeg([ + ...["-f", "lavfi", "-i", "testsrc=size=320x180:rate=25:duration=3"], + ...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=3"], + ...encode, + ...["-c:a", "aac", "-shortest", `file:${landscape}`], + ]); + await runFfmpeg([ + ...["-f", "lavfi", "-i", "testsrc2=size=180x320:rate=30:duration=2"], + ...encode, + `file:${silentPortrait}`, + ]); + const srt = join(dir, "captions.srt"); + await writeFile( + srt, + "1\n00:00:00,100 --> 00:00:01,500\nHello from the subtitles\n\n2\n00:00:01,500 --> 00:00:02,900\nZweite Zeile äöü\n", + ); + const vtt = join(dir, "captions.vtt"); + await writeFile(vtt, "WEBVTT\n\n00:00.100 --> 00:01.500\nHello from WebVTT\n"); + return { dir, landscape, silentPortrait, srt, vtt }; +} + +export interface TestArtifact { + path: string; + name: string; + mimeType: string; + kind?: MediaOutputKind; +} + +/** A MediaJobContext over local files that records progress and outputs. */ +export async function fakeMediaJobContext(input: unknown, artifacts: Record) { + const tempDir = await mkdtemp(join(tmpdir(), "evoo-media-job-")); + const controller = new AbortController(); + const progress: number[] = []; + const outputs: (MediaJobOutputFile & { size: number; posterSize?: number })[] = []; + const context: MediaJobContext = { + jobId: "job-1", + spaceId: "space-1", + userId: "user-1", + botId: null, + input, + tempDir, + signal: controller.signal, + progress: async (fraction) => { + progress.push(fraction); + }, + fetchArtifact: async (artifactId): Promise => { + const artifact = artifacts[artifactId]; + if (!artifact) throw new Error(`Artifact ${artifactId} not found.`); + const path = join(tempDir, `input-${artifactId}`); + await copyFile(artifact.path, path); + return { + artifactId, + path, + name: artifact.name, + mimeType: artifact.mimeType, + kind: artifact.kind ?? attachmentKindOf(artifact.mimeType), + size: (await stat(path)).size, + }; + }, + addOutput: async (file): Promise => { + // Files must exist when handed over; the runner stores them before the job ends. + const size = (await stat(file.path)).size; + const posterSize = file.posterPath ? (await stat(file.posterPath)).size : undefined; + outputs.push({ ...file, size, ...(posterSize !== undefined ? { posterSize } : {}) }); + const index = outputs.length; + return { + artifactId: `output-${index}`, + kind: attachmentKindOf(file.mimeType), + mimeType: file.mimeType, + name: file.name, + size, + ...(file.width ? { width: file.width } : {}), + ...(file.height ? { height: file.height } : {}), + ...(file.durationMs !== undefined ? { durationMs: file.durationMs } : {}), + ...(file.posterPath ? { posterArtifactId: `poster-${index}` } : {}), + ...(file.parentArtifactId ? { parentArtifactId: file.parentArtifactId } : {}), + ...(file.preset ? { preset: file.preset } : {}), + }; + }, + }; + return { context, controller, progress, outputs, tempDir }; +} diff --git a/packages/adapters/src/media/video-edit-plan.test.ts b/packages/adapters/src/media/video-edit-plan.test.ts new file mode 100644 index 0000000..ea37cd6 --- /dev/null +++ b/packages/adapters/src/media/video-edit-plan.test.ts @@ -0,0 +1,301 @@ +import type { EditVideoStep } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { MediaProbe } from "./ffmpeg.js"; +import type { PlanInput } from "./video-edit-plan.js"; +import { atempoFilters, planVideoEdit, subtitleStyle } from "./video-edit-plan.js"; + +function probe( + overrides: Partial> & { + durationMs?: number; + hasAudio?: boolean; + } = {}, +): MediaProbe { + const { durationMs = 10_000, hasAudio = true, ...video } = overrides; + return { + durationMs, + formatName: "mov,mp4,m4a,3gp,3g2,mj2", + video: { + codec: "h264", + width: 1920, + height: 1080, + rotation: 0, + fps: 25, + sampleAspectRatio: 1, + ...video, + }, + ...(hasAudio ? { audio: { codec: "aac", sampleRate: 44_100, channels: 2 } } : {}), + hasAudio, + }; +} + +const source: PlanInput = { + path: "/jobs/1/input-source", + mimeType: "video/mp4", + name: "holiday.mp4", + probe: probe(), +}; + +function plan(steps: EditVideoStep[], extra: Partial[0]> = {}) { + return planVideoEdit({ source, steps, outputPath: "/jobs/1/output.mp4", ...extra }); +} + +function graphOf(args: string[]): string { + return args[args.indexOf("-filter_complex") + 1]!; +} + +describe("planVideoEdit", () => { + it("seeks the input for a leading trim and encodes H.264/AAC MP4", () => { + const result = plan([{ op: "trim", start: 1.5, end: 4 }]); + expect(result.args).toEqual([ + ...["-ss", "1.5", "-t", "2.5", "-protocol_whitelist", "file", "-f", "mov"], + ...["-i", "file:/jobs/1/input-source"], + "-filter_complex", + "[0:v:0]format=yuv420p[v1];[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a2]", + ...["-map", "[v1]", "-map", "[a2]", "-c:a", "aac", "-b:a", "160k"], + ...["-c:v", "libx264", "-preset", "veryfast", "-crf", "21", "-pix_fmt", "yuv420p"], + ...["-movflags", "+faststart", "-map_metadata", "-1", "-map_chapters", "-1"], + ...["-f", "mp4", "file:/jobs/1/output.mp4"], + ]); + expect(result).toMatchObject({ kind: "video", durationMs: 2500, width: 1920, height: 1080 }); + }); + + it("keeps sorted, merged segments from one seek with split, trims and concat", () => { + const result = plan([ + { + op: "keep", + segments: [ + { start: 6, end: 8 }, + { start: 2, end: 3 }, + { start: 2.5, end: 4 }, + ], + }, + ]); + expect(result.args.slice(0, 4)).toEqual(["-ss", "2", "-t", "6"]); + const graph = graphOf(result.args); + expect(graph).toContain("[0:v:0]split=2[v1][v2]"); + expect(graph).toContain("[v1]trim=start=0:end=2,setpts=PTS-STARTPTS[v3]"); + expect(graph).toContain("[v2]trim=start=4:end=6,setpts=PTS-STARTPTS[v4]"); + expect(graph).toContain("[0:a:0]asplit=2[a5][a6]"); + expect(graph).toContain("[v3][a7][v4][a8]concat=n=2:v=1:a=1[v9][a10]"); + expect(result.durationMs).toBe(4000); + }); + + it("applies later cuts on the timeline left by earlier steps", () => { + const result = plan([ + { op: "speed", factor: 2 }, + { op: "trim", start: 1, end: 99 }, + ]); + expect(graphOf(result.args)).toContain( + "[0:v:0]setpts=PTS/2,fps=25,trim=start=1:end=5,setpts=PTS-STARTPTS,format=yuv420p[v1]", + ); + expect(result.durationMs).toBe(4000); + }); + + it("rejects ranges outside the video and too short ones", () => { + expect(() => plan([{ op: "trim", start: 12, end: 18 }])).toThrow( + "trim 12–18 s is outside the 10 s video.", + ); + expect(() => + plan([ + { op: "speed", factor: 2 }, + { + op: "keep", + segments: [ + { start: 1, end: 2 }, + { start: 6, end: 7 }, + ], + }, + ]), + ).toThrow("keep 6–7 s is outside the 5 s video."); + expect(() => plan([{ op: "trim", start: 9.95, end: 12 }])).toThrow("shorter than 0.1 s"); + }); + + it("appends clips fitted to the frame, with silence for clips without audio", () => { + const clip: PlanInput = { + path: "/jobs/1/input-clip", + mimeType: "video/webm", + name: "clip.webm", + probe: probe({ width: 1080, height: 1920, fps: 30, durationMs: 2000, hasAudio: false }), + }; + const result = plan([{ op: "concat", artifact_ids: ["clip"] }], { + clips: new Map([["clip", clip]]), + }); + expect(result.args).toContain("matroska"); + const graph = graphOf(result.args); + expect(graph).toContain("[0:v:0]setsar=1,format=yuv420p[v1]"); + expect(graph).toContain( + "[1:v:0]scale=1920:1080:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1920:1080:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=25,format=yuv420p[v3]", + ); + expect(graph).toContain( + "anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=2,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a4]", + ); + expect(graph).toContain("[v1][a2][v3][a4]concat=n=2:v=1:a=1"); + expect(result.durationMs).toBe(12_000); + }); + + it("adds silence to a silent source when an appended clip has audio", () => { + const silent = { ...source, probe: probe({ hasAudio: false }) }; + const result = planVideoEdit({ + source: silent, + steps: [{ op: "concat", artifact_ids: ["clip"] }], + clips: new Map([["clip", source]]), + outputPath: "/out.mp4", + }); + expect(graphOf(result.args)).toContain( + "anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=10", + ); + expect(result.args.filter((arg) => arg === "-map")).toHaveLength(2); + }); + + it("appends clips with the last reframe's fit", () => { + const result = plan( + [ + { op: "reframe", preset: "vertical", fit: "crop" }, + { op: "concat", artifact_ids: ["clip"] }, + ], + { clips: new Map([["clip", source]]) }, + ); + expect(graphOf(result.args)).toContain( + "[1:v:0]scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920,setsar=1,fps=25", + ); + }); + + it("reframes by crop, pad or blur, and only scales when the aspect already matches", () => { + expect(graphOf(plan([{ op: "reframe", preset: "square", fit: "crop" }]).args)).toContain( + "scale=1080:1080:force_original_aspect_ratio=increase,crop=1080:1080,setsar=1", + ); + expect(graphOf(plan([{ op: "reframe", preset: "portrait", fit: "pad" }]).args)).toContain( + "pad=1080:1350:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1", + ); + const blur = plan([{ op: "reframe", preset: "vertical", fit: "blur" }]); + expect(graphOf(blur.args)).toBe( + [ + "[0:v:0]split=2[v1][v2]", + "[v1]scale=270:480:force_original_aspect_ratio=increase,crop=270:480,boxblur=27:2,scale=1080:1920,setsar=1[v3]", + "[v2]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,setsar=1[v4]", + "[v3][v4]overlay=(W-w)/2:(H-h)/2[v5]", + "[v5]format=yuv420p[v6]", + "[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a7]", + ].join(";"), + ); + expect(blur).toMatchObject({ width: 1080, height: 1920 }); + expect(graphOf(plan([{ op: "reframe", preset: "landscape", fit: "blur" }]).args)).toContain( + "[0:v:0]scale=1920:1080,setsar=1,format=yuv420p", + ); + }); + + it("changes speed with atempo chains and keeps the source frame rate", () => { + const fast = plan([{ op: "speed", factor: 4 }]); + expect(graphOf(fast.args)).toContain("setpts=PTS/4,fps=25"); + expect(graphOf(fast.args)).toContain("atempo=2,atempo=2"); + expect(fast.durationMs).toBe(2500); + expect(atempoFilters(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]); + expect(atempoFilters(3)).toEqual(["atempo=2", "atempo=1.5"]); + expect(atempoFilters(0.8)).toEqual(["atempo=0.8"]); + }); + + it("changes volume, and mute drops the audio track", () => { + expect(graphOf(plan([{ op: "volume", gain: 0.5 }]).args)).toContain( + "[0:a:0]volume=0.5,aformat", + ); + const muted = plan([{ op: "volume", gain: 2 }, { op: "mute" }]); + expect(muted.args.filter((arg) => arg === "-map")).toHaveLength(1); + expect(muted.args).not.toContain("aac"); + }); + + it("burns in subtitles styled for the current frame from a generated file name", () => { + const result = plan( + [ + { op: "reframe", preset: "vertical", fit: "crop" }, + { op: "subtitles", artifact_id: "subs" }, + ], + { subtitleFiles: new Map([["subs", "subtitles-0.srt"]]) }, + ); + expect(graphOf(result.args)).toContain( + `subtitles=filename=subtitles-0.srt:force_style='${subtitleStyle(1080, 1920)}'`, + ); + expect(() => + plan([{ op: "subtitles", artifact_id: "subs" }], { + subtitleFiles: new Map([["subs", "x.srt:force_style='a'"]]), + }), + ).toThrow("Unsafe subtitle file name."); + }); + + it("styles subtitles above the controls of tall frames", () => { + expect(subtitleStyle(1080, 1920)).toContain("MarginV=384"); + expect(subtitleStyle(1080, 1920)).toContain("FontSize=65"); + expect(subtitleStyle(1920, 1080)).toContain("MarginV=65"); + expect(subtitleStyle(1920, 1080)).toContain("PlayResX=1920,PlayResY=1080"); + }); + + it("extracts audio without decoding video work", () => { + const result = planVideoEdit({ + source, + steps: [ + { op: "trim", start: 0, end: 5 }, + { op: "reframe", preset: "vertical", fit: "blur" }, + { op: "subtitles", artifact_id: "subs" }, + { op: "extract_audio" }, + ], + subtitleFiles: new Map([["subs", "subtitles-0.srt"]]), + outputPath: "/out.m4a", + }); + expect(graphOf(result.args)).toBe( + "[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a1]", + ); + expect(result.args).toEqual(expect.arrayContaining(["-map", "[a1]", "-b:a", "192k"])); + expect(result.args).not.toContain("libx264"); + expect(result).toMatchObject({ kind: "audio", durationMs: 5000 }); + expect(() => + planVideoEdit({ + source: { ...source, probe: probe({ hasAudio: false }) }, + steps: [{ op: "extract_audio" }], + outputPath: "/out.m4a", + }), + ).toThrow("holiday.mp4 has no audio."); + }); + + it("validates the poster time against the result", () => { + expect( + plan([ + { op: "trim", start: 0, end: 4 }, + { op: "poster", at: 3 }, + ]).posterAtMs, + ).toBe(3000); + expect(() => + plan([ + { op: "trim", start: 0, end: 4 }, + { op: "poster", at: 5 }, + ]), + ).toThrow("poster at 5 s is past the end of the 4 s result."); + }); + + it("keeps file paths out of the filter graph", () => { + const hostile = { ...source, path: "/jobs/1/a'b[c]:d;e,f.mp4", name: "x'];[0:v]drawtext=" }; + const result = planVideoEdit({ + source: hostile, + steps: [{ op: "reframe", preset: "square", fit: "blur" }], + outputPath: "/jobs/1/out.mp4", + }); + expect(graphOf(result.args)).not.toContain("a'b"); + expect(graphOf(result.args)).not.toContain("drawtext"); + expect(result.args).toContain("file:/jobs/1/a'b[c]:d;e,f.mp4"); + }); + + it("squares pixels and evens odd frame sizes", () => { + const anamorphic = { + ...source, + probe: probe({ width: 1440, height: 1080, sampleAspectRatio: 4 / 3 }), + }; + const result = planVideoEdit({ + source: anamorphic, + steps: [{ op: "mute" }], + outputPath: "/o.mp4", + }); + expect(graphOf(result.args)).toBe("[0:v:0]scale=1920:1080,setsar=1,format=yuv420p[v1]"); + expect(result).toMatchObject({ width: 1920, height: 1080 }); + const odd = { ...source, probe: probe({ width: 721, height: 405 }) }; + const evened = planVideoEdit({ source: odd, steps: [{ op: "mute" }], outputPath: "/o.mp4" }); + expect(graphOf(evened.args)).toBe("[0:v:0]crop=720:404:0:0,format=yuv420p[v1]"); + }); +}); diff --git a/packages/adapters/src/media/video-edit-plan.ts b/packages/adapters/src/media/video-edit-plan.ts new file mode 100644 index 0000000..f39f8d5 --- /dev/null +++ b/packages/adapters/src/media/video-edit-plan.ts @@ -0,0 +1,571 @@ +import type { EditVideoStep, MediaFit } from "@rakazo/contracts"; +import { MEDIA_PRESETS } from "@rakazo/contracts"; +import type { MediaProbe } from "./ffmpeg.js"; +import { ffmpegInputArgs, ffmpegSeconds } from "./ffmpeg.js"; + +/** + * Turns an ordered list of edit steps into one ffmpeg invocation with a single filter_complex + * pass. Pure: the caller probes the inputs and runs the arguments. + */ + +export interface PlanInput { + path: string; + mimeType: string; + name: string; + probe: MediaProbe; +} + +export interface VideoEditPlanRequest { + source: PlanInput; + steps: readonly EditVideoStep[]; + /** Videos named by concat steps, by artifact id. */ + clips?: ReadonlyMap; + /** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */ + subtitleFiles?: ReadonlyMap; + outputPath: string; +} + +export interface VideoEditPlan { + args: string[]; + kind: "video" | "audio"; + durationMs: number; + width?: number; + height?: number; + posterAtMs?: number; +} + +export const VIDEO_ENCODE_ARGS = [ + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "21", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", +]; +const AUDIO_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "160k"]; +const AUDIO_ONLY_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"]; +/** Location and other camera metadata stay private; chapters do not survive cuts anyway. */ +const STRIP_METADATA_ARGS = ["-map_metadata", "-1", "-map_chapters", "-1"]; +const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo"; +const SAFE_FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const MIN_SEGMENT_SECONDS = 0.1; +const FALLBACK_FPS = 30; + +interface Track { + label: string; + filters: string[]; +} + +interface Timeline { + /** Null when the result is audio only: video work is skipped. */ + video: Track | null; + audio: Track | null; + width: number; + height: number; + fps: number; + seconds: number; + /** How appended clips fill the frame: the last reframe's fit, otherwise bars. */ + fit: MediaFit; +} + +class FilterGraph { + private readonly chains: string[] = []; + private count = 0; + + label(kind: "v" | "a"): string { + this.count += 1; + return `${kind}${this.count}`; + } + + add(inputs: readonly string[], filters: readonly string[], outputs: readonly string[]) { + const pads = (labels: readonly string[]) => labels.map((label) => `[${label}]`).join(""); + this.chains.push(`${pads(inputs)}${filters.join(",")}${pads(outputs)}`); + } + + /** Ends a track's pending filters in a labelled pad other chains can consume. */ + flush(track: Track, kind: "v" | "a"): string { + if (track.filters.length === 0) return track.label; + const output = this.label(kind); + this.add([track.label], track.filters, [output]); + track.label = output; + track.filters = []; + return output; + } + + toString(): string { + return this.chains.join(";"); + } +} + +export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { + const { source } = request; + const video = source.probe.video; + if (!video) throw new Error(`${source.name} has no video.`); + const sourceSeconds = source.probe.durationMs / 1000; + const audioOnly = request.steps.at(-1)?.op === "extract_audio"; + const graph = new FilterGraph(); + const inputs: string[][] = []; + const addInput = (input: PlanInput, seek?: { start: number; end: number }) => { + inputs.push( + ffmpegInputArgs(input.path, { + mimeType: input.mimeType, + ...(seek ? { startSeconds: seek.start, durationSeconds: seek.end - seek.start } : {}), + }), + ); + return inputs.length - 1; + }; + + let steps = request.steps; + // A leading cut seeks the input instead of decoding everything before it. + const first = steps[0]; + let segments: Segment[] | undefined; + if (first?.op === "trim" || first?.op === "keep") { + segments = normalizeSegments( + first.op === "trim" ? [first] : first.segments, + sourceSeconds, + first.op, + ); + steps = steps.slice(1); + } + const seek = segments && { start: segments[0]!.start, end: segments.at(-1)!.end }; + const index = addInput(source, seek); + const size = displaySize(video); + const timeline: Timeline = { + video: audioOnly ? null : { label: `${index}:v:0`, filters: sourceFilters(video) }, + audio: source.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null, + width: size.width, + height: size.height, + fps: video.fps, + seconds: seek ? seek.end - seek.start : sourceSeconds, + fit: "pad", + }; + if (segments && seek && segments.length > 1) { + keepSegments( + graph, + timeline, + segments.map((segment) => ({ + start: segment.start - seek.start, + end: segment.end - seek.start, + })), + ); + } + + let posterAt: number | undefined; + for (const step of steps) { + switch (step.op) { + case "trim": + case "keep": + keepSegments( + graph, + timeline, + normalizeSegments(step.op === "trim" ? [step] : step.segments, timeline.seconds, step.op), + ); + break; + case "concat": + appendClips( + graph, + timeline, + step.artifact_ids.map((id) => { + const clip = request.clips?.get(id); + if (!clip) throw new Error(`Video ${id} is not available.`); + return { clip, index: addInput(clip) }; + }), + ); + break; + case "reframe": { + const preset = MEDIA_PRESETS[step.preset]; + if (timeline.video) { + timeline.video = fitFrame( + graph, + timeline.video, + timeline, + { width: preset.width, height: preset.height }, + step.fit, + ); + } + timeline.width = preset.width; + timeline.height = preset.height; + timeline.fit = step.fit; + break; + } + case "speed": + timeline.video?.filters.push( + `setpts=PTS/${number(step.factor)}`, + ...(timeline.fps ? [`fps=${number(timeline.fps)}`] : []), + ); + timeline.audio?.filters.push(...atempoFilters(step.factor)); + timeline.seconds /= step.factor; + break; + case "volume": + timeline.audio?.filters.push(`volume=${number(step.gain)}`); + break; + case "mute": + timeline.audio = null; + break; + case "subtitles": { + const file = request.subtitleFiles?.get(step.artifact_id); + if (!file) throw new Error(`Subtitles ${step.artifact_id} are not available.`); + // Only generated names reach the filter graph, so nothing here needs escaping. + if (!SAFE_FILE_NAME.test(file)) throw new Error("Unsafe subtitle file name."); + timeline.video?.filters.push( + `subtitles=filename=${file}:force_style='${subtitleStyle(timeline.width, timeline.height)}'`, + ); + break; + } + case "extract_audio": + break; + case "poster": + posterAt = step.at; + break; + } + } + + if (posterAt !== undefined && posterAt >= timeline.seconds) { + throw new Error( + `poster at ${seconds(posterAt)} s is past the end of the ${seconds(timeline.seconds)} s result.`, + ); + } + const durationMs = Math.round(timeline.seconds * 1000); + const output = `file:${request.outputPath}`; + if (audioOnly || !timeline.video) { + if (!timeline.audio) throw new Error(`${source.name} has no audio.`); + timeline.audio.filters.push(AUDIO_FORMAT); + const audio = graph.flush(timeline.audio, "a"); + return { + args: [ + ...inputs.flat(), + "-filter_complex", + graph.toString(), + "-map", + `[${audio}]`, + ...AUDIO_ONLY_ENCODE_ARGS, + ...STRIP_METADATA_ARGS, + "-f", + "mp4", + output, + ], + kind: "audio", + durationMs, + }; + } + timeline.video.filters.push("format=yuv420p"); + const videoOut = graph.flush(timeline.video, "v"); + let audioOut: string | undefined; + if (timeline.audio) { + timeline.audio.filters.push(AUDIO_FORMAT); + audioOut = graph.flush(timeline.audio, "a"); + } + return { + args: [ + ...inputs.flat(), + "-filter_complex", + graph.toString(), + "-map", + `[${videoOut}]`, + ...(audioOut ? ["-map", `[${audioOut}]`, ...AUDIO_ENCODE_ARGS] : []), + ...VIDEO_ENCODE_ARGS, + ...STRIP_METADATA_ARGS, + "-f", + "mp4", + output, + ], + kind: "video", + durationMs, + width: timeline.width, + height: timeline.height, + ...(posterAt !== undefined ? { posterAtMs: Math.round(posterAt * 1000) } : {}), + }; +} + +interface Segment { + start: number; + end: number; +} + +/** Validates ranges against the timeline, clamps ends to it, sorts and merges overlaps. */ +function normalizeSegments( + ranges: readonly Segment[], + duration: number, + op: "trim" | "keep", +): Segment[] { + const clamped = ranges.map((range) => { + if (range.start >= duration) { + throw new Error( + `${op} ${seconds(range.start)}–${seconds(range.end)} s is outside the ${seconds(duration)} s video.`, + ); + } + const end = Math.min(range.end, duration); + if (end - range.start < MIN_SEGMENT_SECONDS) { + throw new Error( + `${op} ${seconds(range.start)}–${seconds(end)} s is shorter than ${MIN_SEGMENT_SECONDS} s.`, + ); + } + return { start: range.start, end }; + }); + clamped.sort((a, b) => a.start - b.start); + const merged: Segment[] = []; + for (const segment of clamped) { + const last = merged.at(-1); + if (last && segment.start <= last.end) last.end = Math.max(last.end, segment.end); + else merged.push({ ...segment }); + } + return merged; +} + +/** Keeps sorted, disjoint segments of the timeline: one trim, or split + trims + concat. */ +function keepSegments(graph: FilterGraph, timeline: Timeline, segments: readonly Segment[]) { + const [only] = segments; + if (segments.length === 1 && only) { + const range = `start=${ffmpegSeconds(only.start)}:end=${ffmpegSeconds(only.end)}`; + timeline.video?.filters.push(`trim=${range}`, "setpts=PTS-STARTPTS"); + timeline.audio?.filters.push(`atrim=${range}`, "asetpts=PTS-STARTPTS"); + timeline.seconds = only.end - only.start; + return; + } + const count = segments.length; + const branch = (track: Track | null, kind: "v" | "a"): string[] => { + if (!track) return []; + const input = graph.flush(track, kind); + const splits = segments.map(() => graph.label(kind)); + graph.add([input], [`${kind === "v" ? "split" : "asplit"}=${count}`], splits); + return segments.map((segment, index) => { + const output = graph.label(kind); + const range = `start=${ffmpegSeconds(segment.start)}:end=${ffmpegSeconds(segment.end)}`; + graph.add( + [splits[index]!], + kind === "v" + ? [`trim=${range}`, "setpts=PTS-STARTPTS"] + : [`atrim=${range}`, "asetpts=PTS-STARTPTS"], + [output], + ); + return output; + }); + }; + const videoParts = branch(timeline.video, "v"); + const audioParts = branch(timeline.audio, "a"); + concatParts(graph, timeline, videoParts, audioParts, count); + timeline.seconds = segments.reduce((total, segment) => total + segment.end - segment.start, 0); +} + +/** Appends clips normalized to the timeline: frame size, fit, fps and 48 kHz stereo audio. */ +function appendClips( + graph: FilterGraph, + timeline: Timeline, + clips: readonly { clip: PlanInput; index: number }[], +) { + const fps = timeline.fps || FALLBACK_FPS; + const needsAudio = timeline.audio !== null || clips.some(({ clip }) => clip.probe.hasAudio); + const videoParts: string[] = []; + const audioParts: string[] = []; + if (timeline.video) { + timeline.video.filters.push("setsar=1", "format=yuv420p"); + videoParts.push(graph.flush(timeline.video, "v")); + } + if (needsAudio) audioParts.push(audioPart(graph, timeline.audio, timeline.seconds)); + let seconds = timeline.seconds; + for (const { clip, index } of clips) { + const video = clip.probe.video; + if (!video) throw new Error(`${clip.name} has no video.`); + const clipSeconds = clip.probe.durationMs / 1000; + if (clipSeconds <= 0) throw new Error(`Could not read the length of ${clip.name}.`); + if (timeline.video) { + const track = fitFrame( + graph, + { label: `${index}:v:0`, filters: sourceFilters(video) }, + displaySize(video), + timeline, + timeline.fit, + ); + track.filters.push(`fps=${number(fps)}`, "format=yuv420p"); + videoParts.push(graph.flush(track, "v")); + } + if (needsAudio) { + const track = clip.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null; + audioParts.push(audioPart(graph, track, clipSeconds)); + } + seconds += clipSeconds; + } + concatParts(graph, timeline, videoParts, audioParts, clips.length + 1); + timeline.seconds = seconds; +} + +/** A clip's audio in the common format, or silence as long as the clip when it has none. */ +function audioPart(graph: FilterGraph, track: Track | null, seconds: number): string { + if (track) { + track.filters.push(AUDIO_FORMAT); + return graph.flush(track, "a"); + } + const output = graph.label("a"); + graph.add( + [], + [ + "anullsrc=channel_layout=stereo:sample_rate=48000", + `atrim=end=${ffmpegSeconds(seconds)}`, + AUDIO_FORMAT, + ], + [output], + ); + return output; +} + +function concatParts( + graph: FilterGraph, + timeline: Timeline, + videoParts: readonly string[], + audioParts: readonly string[], + count: number, +) { + const inputs = Array.from({ length: count }, (_, index) => [ + ...(videoParts[index] ? [videoParts[index]] : []), + ...(audioParts[index] ? [audioParts[index]] : []), + ]).flat(); + const video = videoParts.length > 0 ? graph.label("v") : undefined; + const audio = audioParts.length > 0 ? graph.label("a") : undefined; + graph.add( + inputs, + [`concat=n=${count}:v=${video ? 1 : 0}:a=${audio ? 1 : 0}`], + [...(video ? [video] : []), ...(audio ? [audio] : [])], + ); + timeline.video = video ? { label: video, filters: [] } : null; + timeline.audio = audio ? { label: audio, filters: [] } : null; +} + +interface Size { + width: number; + height: number; +} + +/** Scales a frame of size `from` into `to`: center crop, bars, or a blurred copy behind it. */ +function fitFrame(graph: FilterGraph, track: Track, from: Size, to: Size, fit: MediaFit): Track { + const { width, height } = to; + if (Math.abs(from.width / from.height - width / height) < 0.01) { + track.filters.push(`scale=${width}:${height}`, "setsar=1"); + return track; + } + const contain = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`; + if (fit === "crop") { + track.filters.push( + `scale=${width}:${height}:force_original_aspect_ratio=increase`, + `crop=${width}:${height}`, + "setsar=1", + ); + return track; + } + if (fit === "pad") { + track.filters.push( + contain, + `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, + "setsar=1", + ); + return track; + } + // Blur a small copy and scale it up: as smooth as a large blur at a fraction of the cost. + const input = graph.flush(track, "v"); + const [background, foreground, blurred, contained, output] = [ + graph.label("v"), + graph.label("v"), + graph.label("v"), + graph.label("v"), + graph.label("v"), + ]; + const smallWidth = even(width / 4); + const smallHeight = even(height / 4); + const radius = Math.max(2, Math.floor(Math.min(smallWidth, smallHeight) / 10)); + graph.add([input], ["split=2"], [background, foreground]); + graph.add( + [background], + [ + `scale=${smallWidth}:${smallHeight}:force_original_aspect_ratio=increase`, + `crop=${smallWidth}:${smallHeight}`, + `boxblur=${radius}:2`, + `scale=${width}:${height}`, + "setsar=1", + ], + [blurred], + ); + graph.add([foreground], [contain, "setsar=1"], [contained]); + graph.add([blurred, contained], ["overlay=(W-w)/2:(H-h)/2"], [output]); + return { label: output, filters: [] }; +} + +/** Square pixels and even dimensions, which yuv420p H.264 requires. */ +function sourceFilters(video: NonNullable): string[] { + const size = displaySize(video); + if (Math.abs(video.sampleAspectRatio - 1) > 0.01) { + return [`scale=${size.width}:${size.height}`, "setsar=1"]; + } + if (size.width !== video.width || size.height !== video.height) { + return [`crop=${size.width}:${size.height}:0:0`]; + } + return []; +} + +function displaySize(video: NonNullable): Size { + if (Math.abs(video.sampleAspectRatio - 1) > 0.01) { + return { + width: even(Math.round(video.width * video.sampleAspectRatio)), + height: even(video.height), + }; + } + return { width: even(video.width), height: even(video.height) }; +} + +/** atempo takes 0.5–2 per instance on older ffmpeg builds, so larger changes chain it. */ +export function atempoFilters(factor: number): string[] { + const filters: string[] = []; + let rest = factor; + while (rest > 2) { + filters.push("atempo=2"); + rest /= 2; + } + while (rest < 0.5) { + filters.push("atempo=0.5"); + rest /= 0.5; + } + filters.push(`atempo=${number(rest)}`); + return filters; +} + +/** + * Readable burned-in subtitles: bold white text with a dark outline, sized to the frame, and + * lifted above the bottom fifth of tall frames where 9:16 apps place their own controls. + */ +export function subtitleStyle(width: number, height: number): string { + const tall = height >= width * 1.5; + const fontSize = Math.round(Math.min(width, height) * 0.06); + const marginX = Math.round(width * (tall ? 0.1 : 0.05)); + const marginY = Math.round(height * (tall ? 0.2 : 0.06)); + return [ + `PlayResX=${width}`, + `PlayResY=${height}`, + "FontName=DejaVu Sans", + `FontSize=${fontSize}`, + "Bold=1", + "PrimaryColour=&H00FFFFFF", + "OutlineColour=&H00000000", + "BackColour=&H80000000", + "BorderStyle=1", + `Outline=${Math.max(2, Math.round(fontSize / 16))}`, + "Shadow=1", + "Alignment=2", + `MarginL=${marginX}`, + `MarginR=${marginX}`, + `MarginV=${marginY}`, + ].join(","); +} + +function even(value: number): number { + return Math.max(2, 2 * Math.floor(value / 2)); +} + +function number(value: number): string { + return String(Number(value.toFixed(6))); +} + +function seconds(value: number): string { + return String(Number(value.toFixed(2))); +} diff --git a/packages/adapters/src/media/video-operations.ts b/packages/adapters/src/media/video-operations.ts new file mode 100644 index 0000000..b21d9df --- /dev/null +++ b/packages/adapters/src/media/video-operations.ts @@ -0,0 +1,331 @@ +import { copyFile, mkdtemp, open, stat } from "node:fs/promises"; +import { join } from "node:path"; +import type { + MediaJobContext, + MediaJobInputFile, + MediaOperationHandler, + MediaOperationResult, +} from "@rakazo/adapter-kit"; +import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts"; +import { + ARTIFACT_MAX_BYTES_BY_KIND, + EditVideoInputSchema, + ExportVideoInputSchema, + MEDIA_PRESETS, +} from "@rakazo/contracts"; +import type { FfmpegInfo } from "./ffmpeg.js"; +import { + detectFfmpeg, + outputTooLargeMessage, + probeMedia, + renderPoster, + runFfmpeg, +} from "./ffmpeg.js"; +import type { PlanInput, VideoEditPlan } from "./video-edit-plan.js"; +import { planVideoEdit } from "./video-edit-plan.js"; + +export interface VideoOperationsDeps { + /** Finds ffmpeg; defaults to PATH lookup. */ + detect?: () => Promise; +} + +const PROGRESS_INTERVAL_MS = 1_000; +/** Share of each output's progress spent encoding; the rest is probing and the poster. */ +const ENCODE_SHARE = 0.95; +const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024; + +/** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */ +export function createVideoOperations(deps: VideoOperationsDeps = {}) { + const detect = deps.detect ?? detectFfmpeg; + const available = async () => (await detect()) !== null; + const tools = async () => { + const info = await detect(); + if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server."); + return info; + }; + const editVideo: MediaOperationHandler = { + operation: "edit_video", + run: async (context) => runEditVideo(context, await tools()), + }; + const exportVideo: MediaOperationHandler = { + operation: "export_video", + run: async (context) => runExportVideo(context, await tools()), + }; + return [ + { + operation: "edit_video" as const, + inputSchema: EditVideoInputSchema, + title: (input: EditVideoInput) => input.title, + available, + handler: editVideo, + }, + { + operation: "export_video" as const, + inputSchema: ExportVideoInputSchema, + title: (input: ExportVideoInput) => + input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "), + available, + handler: exportVideo, + }, + ]; +} + +async function runEditVideo( + context: MediaJobContext, + tools: FfmpegInfo, +): Promise { + const input = EditVideoInputSchema.parse(context.input); + const workDir = await mkdtemp(join(context.tempDir, "edit-")); + const progress = createProgressReporter(context); + const source = await loadVideo(context, tools, input.artifact_id); + const name = cleanFileName(input.title, cleanFileName(withoutExtension(source.name), "video")); + + if (input.steps.every((step) => step.op === "poster")) { + const at = input.steps[0]?.op === "poster" ? input.steps[0].at : 0; + if (at * 1000 >= source.probe.durationMs) { + throw new Error( + `poster at ${at} s is past the end of the ${source.probe.durationMs / 1000} s video.`, + ); + } + const posterPath = join(workDir, "poster.jpg"); + await renderPoster(source.path, posterPath, { + signal: context.signal, + mimeType: source.mimeType, + atMs: at * 1000, + }); + const output = await context.addOutput({ + path: posterPath, + name: `${name}.jpg`, + mimeType: "image/jpeg", + width: source.probe.video?.width, + height: source.probe.video?.height, + parentArtifactId: source.artifactId, + }); + await progress.done(); + return { summary: `Saved the frame at ${at} s of ${source.name} as ${describe(output)}.` }; + } + + const clips = new Map(); + const subtitleFiles = new Map(); + for (const step of input.steps) { + if (step.op === "concat") { + for (const id of step.artifact_ids) { + if (!clips.has(id)) clips.set(id, await loadVideo(context, tools, id)); + } + } else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) { + subtitleFiles.set( + step.artifact_id, + await prepareSubtitles(context, step.artifact_id, workDir, subtitleFiles.size), + ); + } + } + const audioOnly = input.steps.at(-1)?.op === "extract_audio"; + const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4"); + const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath }); + const output = await encode(context, tools, plan, outputPath, workDir, progress.range(0, 1), { + name: `${name}${audioOnly ? ".m4a" : ".mp4"}`, + parentArtifactId: source.artifactId, + }); + await progress.done(); + return { + summary: `Edited ${source.name} into ${describe(output)}.`, + }; +} + +async function runExportVideo( + context: MediaJobContext, + tools: FfmpegInfo, +): Promise { + const input = ExportVideoInputSchema.parse(context.input); + const presets = [...new Set(input.presets)]; + const workDir = await mkdtemp(join(context.tempDir, "export-")); + const progress = createProgressReporter(context); + const source = await loadVideo(context, tools, input.artifact_id); + const base = cleanFileName(withoutExtension(source.name), "video"); + const outputs: MediaOutput[] = []; + for (const [index, preset] of presets.entries()) { + const outputPath = join(workDir, `${preset}.mp4`); + const plan = planVideoEdit({ + source, + steps: [{ op: "reframe", preset, fit: input.fit }], + outputPath, + }); + const share = 1 / presets.length; + outputs.push( + await encode( + context, + tools, + plan, + outputPath, + workDir, + progress.range(index * share, (index + 1) * share), + { + name: `${base}-${MEDIA_PRESETS[preset].aspect.replace(":", "x")}.mp4`, + parentArtifactId: source.artifactId, + preset, + }, + ), + ); + } + await progress.done(); + return { summary: `Exported ${source.name} as ${outputs.map(describe).join(", ")}.` }; +} + +/** Runs a plan, checks the result, renders its poster and stores it as an output. */ +async function encode( + context: MediaJobContext, + tools: FfmpegInfo, + plan: VideoEditPlan, + outputPath: string, + workDir: string, + report: (fraction: number) => void, + file: Pick[0], "name" | "parentArtifactId" | "preset">, +): Promise { + const maxBytes = ARTIFACT_MAX_BYTES_BY_KIND[plan.kind]; + await runFfmpeg(plan.args, { + signal: context.signal, + cwd: workDir, + durationMs: plan.durationMs, + onProgress: (fraction) => report(fraction * ENCODE_SHARE), + maxOutputBytes: maxBytes, + outputs: [outputPath], + ffmpegPath: tools.ffmpegPath, + }); + if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes)); + const probe = await probeMedia(outputPath, { + signal: context.signal, + mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4", + ffprobePath: tools.ffprobePath, + }); + if (plan.kind === "audio") { + const output = await context.addOutput({ + ...file, + path: outputPath, + mimeType: "audio/mp4", + durationMs: probe.durationMs, + }); + report(1); + return output; + } + const posterPath = outputPath.replace(/\.mp4$/, ".jpg"); + await renderPoster(outputPath, posterPath, { + signal: context.signal, + mimeType: "video/mp4", + atMs: plan.posterAtMs ?? Math.min(1_000, probe.durationMs / 4), + }); + const output = await context.addOutput({ + ...file, + path: outputPath, + mimeType: "video/mp4", + width: probe.video?.width ?? plan.width, + height: probe.video?.height ?? plan.height, + durationMs: probe.durationMs, + posterPath, + }); + report(1); + return output; +} + +async function loadVideo( + context: MediaJobContext, + tools: FfmpegInfo, + artifactId: string, +): Promise { + const file = await context.fetchArtifact(artifactId); + if (file.kind !== "video") throw new Error(`${file.name} is not a video.`); + const probe = await probeMedia(file.path, { + signal: context.signal, + mimeType: file.mimeType, + ffprobePath: tools.ffprobePath, + }); + if (!probe.video) throw new Error(`${file.name} has no video.`); + if (probe.durationMs <= 0) throw new Error(`Could not read the length of ${file.name}.`); + return { ...file, probe }; +} + +/** Copies an SRT/VTT artifact to a generated name, so no user text reaches the filter graph. */ +async function prepareSubtitles( + context: MediaJobContext, + artifactId: string, + workDir: string, + index: number, +): Promise { + const file = await context.fetchArtifact(artifactId); + const format = file.size <= SUBTITLE_MAX_BYTES ? subtitleFormat(await readHead(file.path)) : null; + if (!format) throw new Error(`${file.name} is not an SRT or VTT file.`); + const name = `subtitles-${index}.${format}`; + await copyFile(file.path, join(workDir, name)); + return name; +} + +async function readHead(path: string): Promise { + const handle = await open(path, "r"); + try { + const buffer = Buffer.alloc(1024); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + return buffer.subarray(0, bytesRead).toString("utf8"); + } finally { + await handle.close(); + } +} + +/** + * Recognizes WebVTT and SubRip by content. Anything else is refused, so ffmpeg never probes + * an arbitrary file (a playlist, say) handed in as subtitles. + */ +export function subtitleFormat(head: string): "srt" | "vtt" | null { + const text = head.replace(/^/, "").trimStart(); + if (/^WEBVTT(?:[ \t\r\n]|$)/.test(text)) return "vtt"; + if (/^(?:\d+[ \t]*\r?\n)?\d{1,2}:\d{2}:\d{2}[,.]\d{1,3}[ \t]*-->/.test(text)) return "srt"; + return null; +} + +/** A name for produced files: no path separators or control characters, at most 80 chars. */ +export function cleanFileName(text: string, fallback: string): string { + const name = text + .replaceAll(/[\p{Cc}\\/:*?"<>|]+/gu, " ") + .replaceAll(/\s+/g, " ") + .replace(/^[\s.]+/, "") + .slice(0, 80) + .trim(); + return name || fallback; +} + +function withoutExtension(name: string): string { + return name.replace(/\.[A-Za-z0-9]{1,5}$/, ""); +} + +function describe(output: MediaOutput): string { + const size = output.width && output.height ? ` ${output.width}×${output.height}` : ""; + const length = + output.durationMs !== undefined ? ` ${Number((output.durationMs / 1000).toFixed(1))} s` : ""; + return `${output.name}${size}${length} (artifact ${output.artifactId})`; +} + +/** + * Maps ffmpeg progress onto the card: monotonic, at most once a second, written in order and + * never failing the job when a write fails. + */ +function createProgressReporter(context: MediaJobContext) { + let reported = 0; + let reportedAt = 0; + let writes = Promise.resolve(); + const write = (fraction: number) => { + reported = fraction; + reportedAt = Date.now(); + writes = writes.then(() => context.progress(fraction)).catch(() => undefined); + }; + return { + range: (from: number, to: number) => (fraction: number) => { + const value = Math.min(1, from + (to - from) * Math.min(1, Math.max(0, fraction))); + const boundary = fraction >= 1; + if (value <= reported) return; + if (!boundary && Date.now() - reportedAt < PROGRESS_INTERVAL_MS) return; + write(value); + }, + done: async () => { + if (reported < 1) write(1); + await writes; + }, + }; +} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index ba2c70b..40d8db3 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -10,6 +10,7 @@ export * from "./integration-settings.js"; export * from "./local-settings.js"; export * from "./mcp.js"; export * from "./media.js"; +export * from "./media-video.js"; export * from "./model-tiers.js"; export * from "./openai-compatible-ui.js"; export * from "./project-goals.js"; diff --git a/packages/contracts/src/media-video.ts b/packages/contracts/src/media-video.ts new file mode 100644 index 0000000..f0aaf5b --- /dev/null +++ b/packages/contracts/src/media-video.ts @@ -0,0 +1,80 @@ +import * as z from "zod"; +import { Id } from "./ids.js"; +import { MediaFitSchema, MediaPresetIdSchema } from "./media.js"; + +/** + * Inputs of the `edit_video` and `export_video` media operations, which are also the arguments + * of the tools with those names. Times are seconds on the timeline left by the previous steps. + */ + +const Seconds = z.number().min(0).max(86_400); + +const TimeRangeSchema = z + .object({ start: Seconds, end: Seconds }) + .refine((range) => range.end > range.start, "end must be after start"); + +/** Written by the bot in the user's language; shown on the media card and used as file name. */ +const MediaTitleSchema = z.string().trim().min(1).max(120); + +export const EditVideoStepSchema = z.discriminatedUnion("op", [ + z + .object({ op: z.literal("trim"), start: Seconds, end: Seconds }) + .refine((step) => step.end > step.start, "end must be after start"), + /** Keeps these parts and cuts everything between them. */ + z.object({ op: z.literal("keep"), segments: z.array(TimeRangeSchema).min(1).max(50) }), + /** Appends these videos, fitted to the current frame. */ + z.object({ op: z.literal("concat"), artifact_ids: z.array(Id).min(1).max(20) }), + z.object({ + op: z.literal("reframe"), + preset: MediaPresetIdSchema, + fit: MediaFitSchema.default("crop"), + }), + z.object({ op: z.literal("speed"), factor: z.number().min(0.25).max(4) }), + /** Linear gain: 0.5 halves the loudness, 2 doubles it. */ + z.object({ op: z.literal("volume"), gain: z.number().min(0).max(8) }), + z.object({ op: z.literal("mute") }), + /** Burns in an SRT or VTT artifact. */ + z.object({ op: z.literal("subtitles"), artifact_id: Id }), + /** Last step only: the result is the audio as M4A. */ + z.object({ op: z.literal("extract_audio") }), + /** Cover frame at this second of the result; as the only kind of step, the result is that image. */ + z.object({ op: z.literal("poster"), at: Seconds }), +]); +export type EditVideoStep = z.infer; + +export const EditVideoInputSchema = z + .object({ + artifact_id: Id, + steps: z.array(EditVideoStepSchema).min(1).max(20), + title: MediaTitleSchema, + }) + .superRefine((input, context) => { + const ops = input.steps.map((step) => step.op); + const extract = ops.indexOf("extract_audio"); + if (extract >= 0 && extract !== ops.length - 1) { + context.addIssue({ + code: "custom", + path: ["steps", extract], + message: "extract_audio must be the last step", + }); + } + if (extract >= 0 && ops.includes("poster")) { + context.addIssue({ + code: "custom", + path: ["steps"], + message: "poster cannot be combined with extract_audio", + }); + } + if (ops.filter((op) => op === "poster").length > 1) { + context.addIssue({ code: "custom", path: ["steps"], message: "use at most one poster step" }); + } + }); +export type EditVideoInput = z.infer; + +export const ExportVideoInputSchema = z.object({ + artifact_id: Id, + presets: z.array(MediaPresetIdSchema).min(1).max(6), + fit: MediaFitSchema.default("crop"), + title: MediaTitleSchema.optional(), +}); +export type ExportVideoInput = z.infer;