From 485090c391b72dfaba7a96e3eff4e633850a5ced Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:35:34 +0000 Subject: [PATCH 1/5] feat(media): edit and export video with ffmpeg Bots can now cut, reframe and export videos for every common format without a hosted service: ffmpeg and ffprobe from PATH do the work on the server. - ffmpeg wrapper: PATH detection, ffprobe JSON probing (rotation, pixel aspect, frame rate, audio, packet scan when the container has no duration), runs with -progress parsing, abort and stall kills, an output size cap, partial-output cleanup and short errors without temp paths. Inputs go through the file protocol with the demuxer forced from the MIME type, so a crafted upload cannot be probed as a playlist that reads other files or URLs. extractAudio (16 kHz mono WAV) and renderPoster for transcription and media cards. - A pure planner turns edit steps (trim, keep, concat, reframe, speed, volume, mute, subtitles, extract_audio, poster) into one filter_complex pass: leading cuts seek the input, appended clips are normalized to the frame, fps and 48 kHz stereo (silence where a clip has none), reframes crop, pad or blur to the presets, speed chains atempo, subtitles burn in from generated file names with a readable style that stays clear of the controls on 9:16 frames. H.264/AAC MP4 with yuv420p and faststart; location metadata is stripped. - edit_video and export_video operations and tool definitions; the inputs are shared contracts so the media card can start exports. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- packages/adapters/src/builtin-tools.ts | 31 + packages/adapters/src/index.ts | 2 + packages/adapters/src/media/ffmpeg.test.ts | 256 ++++++++ packages/adapters/src/media/ffmpeg.ts | 603 ++++++++++++++++++ .../adapters/src/media/media-test-support.ts | 106 +++ .../src/media/video-edit-plan.test.ts | 301 +++++++++ .../adapters/src/media/video-edit-plan.ts | 571 +++++++++++++++++ .../adapters/src/media/video-operations.ts | 331 ++++++++++ packages/contracts/src/index.ts | 1 + packages/contracts/src/media-video.ts | 80 +++ 10 files changed, 2282 insertions(+) create mode 100644 packages/adapters/src/media/ffmpeg.test.ts create mode 100644 packages/adapters/src/media/ffmpeg.ts create mode 100644 packages/adapters/src/media/media-test-support.ts create mode 100644 packages/adapters/src/media/video-edit-plan.test.ts create mode 100644 packages/adapters/src/media/video-edit-plan.ts create mode 100644 packages/adapters/src/media/video-operations.ts create mode 100644 packages/contracts/src/media-video.ts diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index 0ff52fa..3229a7d 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -2,6 +2,8 @@ import type { ConnectorTool } from "@rakazo/adapter-kit"; import { BotSecretDestination, BotSecretName, + EditVideoInputSchema, + ExportVideoInputSchema, MODEL_TIERS, SecretAskPurpose, SecretHttpRequest, @@ -12,6 +14,25 @@ import { z } from "zod"; export const SUBAGENT_MODEL_TIER_DESCRIPTION = "Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set."; +/** A media tool's arguments: its operation's input plus whether to wake the bot when it ends. */ +function mediaToolSchema(input: z.ZodType): Record { + const schema = z.toJSONSchema(input, { io: "input" }); + return { + ...schema, + properties: { + ...schema.properties, + continue_after: { + type: "boolean", + description: + "true to be woken with the result when the job ends, to continue a multi-step task. Default false: the card alone shows the result.", + }, + }, + }; +} + +const VIDEO_PRESETS_AND_FITS = + "Presets: vertical 9:16, landscape 16:9, square 1:1, portrait 4:5, classic 4:3, cinema 21:9. fit: crop (default) fills the frame and cuts the edges, pad adds bars, blur fills the bars with a blurred copy."; + export const DELEGATION_TOOL_NAMES = new Set([ "run_subagent", "spawn_bot", @@ -326,6 +347,16 @@ export const builtinAgentTools: ConnectorTool[] = [ }, }, }, + { + name: "edit_video", + description: `Edit a video artifact in a background job; a media card shows progress and the result, a new version of the source. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4 with a poster. title: a few words in the user's language for the card and file name.`, + inputSchema: mediaToolSchema(EditVideoInputSchema), + }, + { + name: "export_video", + description: `Export a video artifact to one or more presets in a background job: one H.264/AAC MP4 with a poster per preset, each a new version of the source, shown on a media card. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`, + inputSchema: mediaToolSchema(ExportVideoInputSchema), + }, { name: "add_mcp_server", description: diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 7b4ecd0..9e2c16d 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -76,6 +76,8 @@ export * from "./mcp-emulator.js"; export * from "./mcp-oauth.js"; export * from "./mcp-server-tool.js"; export * from "./mcp-transport.js"; +export * from "./media/ffmpeg.js"; +export * from "./media/video-operations.js"; export * from "./memory-provider-factory.js"; export * from "./messaging-context.js"; export * from "./messaging-delivery.js"; diff --git a/packages/adapters/src/media/ffmpeg.test.ts b/packages/adapters/src/media/ffmpeg.test.ts new file mode 100644 index 0000000..ab846da --- /dev/null +++ b/packages/adapters/src/media/ffmpeg.test.ts @@ -0,0 +1,256 @@ +import { existsSync } from "node:fs"; +import { rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { FfmpegProgress } from "./ffmpeg.js"; +import { + createFfmpegProgressParser, + detectFfmpeg, + extractAudio, + FfmpegError, + ffmpegInputArgs, + ffmpegSeconds, + packetDurationMs, + parseProbeOutput, + probeMedia, + renderPoster, + runFfmpeg, +} from "./ffmpeg.js"; +import type { TestClips } from "./media-test-support.js"; +import { createTestClips } from "./media-test-support.js"; + +const ffmpeg = await detectFfmpeg(); + +describe("ffmpeg progress", () => { + it("reports each block, tolerates split chunks and N/A times, and marks the end", () => { + const updates: FfmpegProgress[] = []; + const feed = createFfmpegProgressParser((update) => updates.push(update)); + feed("frame=0\nout_time_us=N/A\ntotal_size=48\nprogress=cont"); + feed("inue\nframe=75\nout_time_us=2995"); + feed("374\nout_time_ms=2995374\ntotal_size=42300\nprogress=end\n"); + expect(updates).toEqual([ + { totalSize: 48, done: false }, + { outTimeUs: 2_995_374, totalSize: 42_300, done: true }, + ]); + }); +}); + +describe("probe parsing", () => { + const stream = (fields: Record) => ({ + codec_type: "video", + codec_name: "h264", + ...fields, + }); + + it("swaps the size of rotated video and reads frame rate and audio", () => { + const probe = parseProbeOutput( + JSON.stringify({ + format: { duration: "12.345", format_name: "mov,mp4,m4a,3gp,3g2,mj2" }, + streams: [ + stream({ + width: 1920, + height: 1080, + avg_frame_rate: "30000/1001", + sample_aspect_ratio: "1:1", + side_data_list: [{ side_data_type: "Display Matrix", rotation: -90 }], + }), + { codec_type: "audio", codec_name: "aac", sample_rate: "48000", channels: 2 }, + ], + }), + ); + expect(probe).toEqual({ + durationMs: 12_345, + formatName: "mov,mp4,m4a,3gp,3g2,mj2", + video: { + codec: "h264", + width: 1080, + height: 1920, + rotation: 270, + fps: 30000 / 1001, + sampleAspectRatio: 1, + }, + audio: { codec: "aac", sampleRate: 48_000, channels: 2 }, + hasAudio: true, + }); + }); + + it("reads legacy rotate tags, skips cover art and falls back to stream durations", () => { + const probe = parseProbeOutput( + JSON.stringify({ + format: { format_name: "mov" }, + streams: [ + stream({ width: 600, height: 600, disposition: { attached_pic: 1 } }), + stream({ + width: 640, + height: 480, + avg_frame_rate: "0/0", + r_frame_rate: "90000/1", + sample_aspect_ratio: "4:3", + tags: { rotate: "90" }, + duration: "4.5", + }), + ], + }), + ); + expect(probe.durationMs).toBe(4500); + expect(probe.hasAudio).toBe(false); + expect(probe.video).toMatchObject({ + width: 480, + height: 640, + rotation: 90, + fps: 0, + sampleAspectRatio: 3 / 4, + }); + }); + + it("reads the end of the last packet when the container has no duration", () => { + expect(packetDurationMs("0.000000,0.040000\n2.960000,0.040000\nN/A,N/A\n")).toBe(3000); + expect(packetDurationMs("")).toBe(0); + }); +}); + +describe("ffmpeg arguments", () => { + it("opens inputs through the file protocol with the demuxer of their MIME type", () => { + expect( + ffmpegInputArgs("/a/b.webm", { + mimeType: "video/webm", + startSeconds: 2, + durationSeconds: 1.5, + }), + ).toEqual([ + ...["-ss", "2", "-t", "1.5", "-protocol_whitelist", "file", "-f", "matroska"], + ...["-i", "file:/a/b.webm"], + ]); + expect(ffmpegInputArgs("/a/b", { mimeType: "application/octet-stream" })).toEqual([ + ...["-protocol_whitelist", "file", "-i", "file:/a/b"], + ]); + expect(ffmpegSeconds(1e-7)).toBe("0"); + expect(ffmpegSeconds(12.34567)).toBe("12.346"); + }); +}); + +// CI runners may not have ffmpeg; these run wherever it is installed. +describe.skipIf(!ffmpeg)("ffmpeg integration", () => { + let clips: TestClips; + + beforeAll(async () => { + clips = await createTestClips(); + }); + + afterAll(async () => { + if (clips) await rm(clips.dir, { recursive: true, force: true }); + }); + + it("probes duration, size, frame rate and audio", async () => { + const probe = await probeMedia(clips.landscape, { mimeType: "video/mp4" }); + expect(probe.durationMs).toBeGreaterThanOrEqual(2900); + expect(probe.durationMs).toBeLessThanOrEqual(3100); + expect(probe.video).toMatchObject({ codec: "h264", width: 320, height: 180, fps: 25 }); + expect(probe.hasAudio).toBe(true); + const silent = await probeMedia(clips.silentPortrait, { mimeType: "video/mp4" }); + expect(silent.hasAudio).toBe(false); + expect(silent.video).toMatchObject({ width: 180, height: 320 }); + }); + + it("refuses a playlist disguised as an MP4 instead of following its URLs", async () => { + const playlist = join(clips.dir, "playlist.mp4"); + await writeFile( + playlist, + `#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:1.0,\nfile:${clips.srt}\n#EXT-X-ENDLIST\n`, + ); + await expect(probeMedia(playlist, { mimeType: "video/mp4" })).rejects.toThrow(FfmpegError); + }); + + it("reports progress up to 1 for a run", async () => { + const output = join(clips.dir, "progress.mp4"); + const fractions: number[] = []; + await runFfmpeg( + [ + ...ffmpegInputArgs(clips.landscape, { mimeType: "video/mp4" }), + ...[ + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-c:a", + "aac", + "-f", + "mp4", + `file:${output}`, + ], + ], + { durationMs: 3000, onProgress: (fraction) => fractions.push(fraction) }, + ); + expect(fractions.at(-1)).toBe(1); + expect(fractions).toEqual([...fractions].sort((a, b) => a - b)); + expect(existsSync(output)).toBe(true); + }); + + it("fails with the last stderr line, without directory paths, and removes outputs", async () => { + const output = join(clips.dir, "broken.mp4"); + await writeFile(output, "partial"); + const failure = runFfmpeg( + [ + ...ffmpegInputArgs(join(clips.dir, "missing.mp4"), { mimeType: "video/mp4" }), + `file:${output}`, + ], + { outputs: [output] }, + ); + const error = await failure.catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(FfmpegError); + expect((error as Error).message).toMatch(/^ffmpeg failed: .*missing\.mp4.*No such file/); + expect((error as Error).message).not.toContain(clips.dir); + expect(existsSync(output)).toBe(false); + }); + + it("kills ffmpeg on abort and removes the partial output", async () => { + const output = join(clips.dir, "aborted.mp4"); + const controller = new AbortController(); + const run = runFfmpeg( + [ + ...["-re", "-f", "lavfi", "-i", "testsrc=size=320x180:rate=25"], + ...["-t", "60", "-c:v", "libx264", "-preset", "ultrafast", "-f", "mp4", `file:${output}`], + ], + { + signal: controller.signal, + durationMs: 60_000, + onProgress: () => controller.abort(new Error("cancelled")), + outputs: [output], + }, + ); + await expect(run).rejects.toThrow("cancelled"); + expect(existsSync(output)).toBe(false); + }); + + it("stops when the output outgrows its cap", async () => { + const output = join(clips.dir, "too-large.mp4"); + const run = runFfmpeg( + [ + ...["-f", "lavfi", "-i", "testsrc=size=640x360:rate=25:duration=20"], + ...["-c:v", "libx264", "-preset", "ultrafast", "-crf", "0", "-f", "mp4", `file:${output}`], + ], + { maxOutputBytes: 64 * 1024, outputs: [output] }, + ); + await expect(run).rejects.toThrow("The result would exceed 0 MB."); + expect(existsSync(output)).toBe(false); + }); + + it("extracts 16 kHz mono WAV for transcription, optionally a range", async () => { + const output = join(clips.dir, "speech.wav"); + await extractAudio(clips.landscape, output, { mimeType: "video/mp4", startMs: 1000 }); + const probe = await probeMedia(output, { mimeType: "audio/wav" }); + expect(probe.audio).toEqual({ codec: "pcm_s16le", sampleRate: 16_000, channels: 1 }); + expect(probe.video).toBeUndefined(); + expect(Math.abs(probe.durationMs - 2000)).toBeLessThan(100); + await expect( + extractAudio(clips.silentPortrait, join(clips.dir, "none.wav"), { mimeType: "video/mp4" }), + ).rejects.toThrow("The file has no audio track."); + }); + + it("renders a poster frame, falling back to the first frame past the end", async () => { + const poster = join(clips.dir, "poster.jpg"); + await renderPoster(clips.landscape, poster, { mimeType: "video/mp4", atMs: 60_000 }); + const probe = await probeMedia(poster); + expect(probe.video).toMatchObject({ codec: "mjpeg", width: 320, height: 180 }); + }); +}); diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts new file mode 100644 index 0000000..b34e722 --- /dev/null +++ b/packages/adapters/src/media/ffmpeg.ts @@ -0,0 +1,603 @@ +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { access, rm, stat } from "node:fs/promises"; +import { setPriority } from "node:os"; +import { delimiter, join } from "node:path"; + +/** + * ffmpeg and ffprobe from PATH. Every call passes an argument array (never a shell), opens + * inputs through the `file` protocol only, and forces the demuxer from the artifact's MIME type + * so a crafted upload cannot make ffmpeg probe it as a playlist that reads other files or URLs. + */ + +export interface FfmpegInfo { + ffmpegPath: string; + ffprobePath: string; + version: string; +} + +export interface MediaProbe { + durationMs: number; + formatName: string; + /** Width and height as displayed: swapped when the stream is rotated by 90 or 270 degrees. */ + video?: { + codec: string; + width: number; + height: number; + rotation: 0 | 90 | 180 | 270; + /** Frames per second; 0 when unknown or implausible (variable frame rate files). */ + fps: number; + /** Pixel aspect ratio of the displayed frame; 1 for square pixels. */ + sampleAspectRatio: number; + }; + audio?: { codec: string; sampleRate: number; channels: number }; + hasAudio: boolean; +} + +export interface FfmpegProgress { + outTimeUs?: number; + totalSize?: number; + done: boolean; +} + +export interface RunFfmpegOptions { + signal?: AbortSignal; + /** Expected output duration; progress is reported as out_time / durationMs. */ + durationMs?: number; + onProgress?: (fraction: number) => void; + cwd?: string; + /** Kills ffmpeg once the output grows past this size. */ + maxOutputBytes?: number; + /** Files removed when the run fails or is aborted. */ + outputs?: readonly string[]; + /** Kills ffmpeg when it reports no progress for this long. */ + stallTimeoutMs?: number; + ffmpegPath?: string; +} + +export class FfmpegError extends Error { + constructor( + message: string, + readonly exitCode: number | null, + readonly stderrTail: string, + ) { + super(message); + this.name = "FfmpegError"; + } +} + +const DEFAULT_STALL_TIMEOUT_MS = 2 * 60_000; +const PROBE_TIMEOUT_MS = 60_000; +const STDERR_TAIL_BYTES = 8 * 1024; +const KILL_GRACE_MS = 5_000; +const MAX_PLAUSIBLE_FPS = 240; + +const DEMUXERS: Record = { + "video/mp4": "mov", + "video/quicktime": "mov", + "audio/mp4": "mov", + "video/webm": "matroska", + "audio/webm": "matroska", + "audio/mpeg": "mp3", + "audio/wav": "wav", + "audio/ogg": "ogg", +}; + +let detection: Promise | undefined; + +/** Finds ffmpeg and ffprobe on PATH once per process; null when either is missing or broken. */ +export function detectFfmpeg(): Promise { + detection ??= locateFfmpeg(); + return detection; +} + +async function locateFfmpeg(): Promise { + const [ffmpegPath, ffprobePath] = await Promise.all([ + findExecutable("ffmpeg"), + findExecutable("ffprobe"), + ]); + if (!ffmpegPath || !ffprobePath) return null; + try { + const [ffmpeg, ffprobe] = await Promise.all([ + capture(ffmpegPath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }), + capture(ffprobePath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }), + ]); + if (ffmpeg.code !== 0 || ffprobe.code !== 0) return null; + const version = /version (\S+)/.exec(ffmpeg.stdout)?.[1] ?? "unknown"; + return { ffmpegPath, ffprobePath, version }; + } catch { + return null; + } +} + +async function findExecutable(name: string): Promise { + const extensions = + process.platform === "win32" + ? (process.env.PATHEXT ?? ".EXE").split(";").filter(Boolean) + : [""]; + for (const directory of (process.env.PATH ?? "").split(delimiter)) { + if (!directory) continue; + for (const extension of extensions) { + const candidate = join(directory, `${name}${extension}`); + try { + await access(candidate, constants.X_OK); + if ((await stat(candidate)).isFile()) return candidate; + } catch { + // Not in this directory. + } + } + } + return null; +} + +export async function requireFfmpeg(): Promise { + const info = await detectFfmpeg(); + if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server."); + return info; +} + +/** ffmpeg time argument with millisecond precision and no exponent notation. */ +export function ffmpegSeconds(seconds: number): string { + return String(Number(Math.max(0, seconds).toFixed(3))); +} + +/** Input arguments for a local file: optional seek, file protocol only, forced demuxer. */ +export function ffmpegInputArgs( + path: string, + options: { mimeType?: string; startSeconds?: number; durationSeconds?: number } = {}, +): string[] { + const demuxer = options.mimeType ? DEMUXERS[options.mimeType] : undefined; + return [ + ...(options.startSeconds ? ["-ss", ffmpegSeconds(options.startSeconds)] : []), + ...(options.durationSeconds !== undefined + ? ["-t", ffmpegSeconds(options.durationSeconds)] + : []), + "-protocol_whitelist", + "file", + ...(demuxer ? ["-f", demuxer] : []), + "-i", + `file:${path}`, + ]; +} + +export async function probeMedia( + path: string, + options: { signal?: AbortSignal; mimeType?: string; ffprobePath?: string } = {}, +): Promise { + const ffprobePath = options.ffprobePath ?? (await requireFfmpeg()).ffprobePath; + const input = ffmpegInputArgs(path, { mimeType: options.mimeType }); + const result = await capture( + ffprobePath, + ["-v", "error", ...input, "-print_format", "json", "-show_format", "-show_streams"], + { signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS }, + ); + if (result.code !== 0) throw ffmpegFailure("ffprobe", result.code, result.stderr); + const probe = parseProbeOutput(result.stdout); + if (probe.durationMs > 0 || (!probe.video && !probe.audio)) return probe; + // Recordings streamed by browsers often carry no duration header; read it from the packets. + const packets = await capture( + ffprobePath, + [ + "-v", + "error", + ...input, + "-select_streams", + probe.video ? "v:0" : "a:0", + "-show_entries", + "packet=pts_time,duration_time", + "-of", + "csv=p=0", + ], + { signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS * 5 }, + ); + if (packets.code !== 0) throw ffmpegFailure("ffprobe", packets.code, packets.stderr); + return { ...probe, durationMs: packetDurationMs(packets.stdout) }; +} + +/** End of the last packet, from ffprobe `packet=pts_time,duration_time` CSV lines. */ +export function packetDurationMs(csv: string): number { + let end = 0; + for (const line of csv.split("\n")) { + const [pts, duration] = line.split(",").map(Number); + if (pts === undefined || !Number.isFinite(pts)) continue; + end = Math.max(end, pts + (duration !== undefined && Number.isFinite(duration) ? duration : 0)); + } + return Math.round(end * 1000); +} + +interface ProbeStream { + codec_type?: string; + codec_name?: string; + width?: number; + height?: number; + avg_frame_rate?: string; + r_frame_rate?: string; + sample_aspect_ratio?: string; + sample_rate?: string; + channels?: number; + duration?: string; + disposition?: { attached_pic?: number }; + tags?: { rotate?: string }; + side_data_list?: { rotation?: number }[]; +} + +/** Parses `ffprobe -print_format json -show_format -show_streams` output. */ +export function parseProbeOutput(json: string): MediaProbe { + const data = JSON.parse(json) as { + format?: { duration?: string; format_name?: string }; + streams?: ProbeStream[]; + }; + const streams = data.streams ?? []; + // Cover art in audio files is a single attached picture, not a video track. + const video = streams.find( + (stream) => stream.codec_type === "video" && stream.disposition?.attached_pic !== 1, + ); + const audio = streams.find((stream) => stream.codec_type === "audio"); + const seconds = + positive(Number(data.format?.duration)) ?? + Math.max(0, ...streams.map((stream) => positive(Number(stream.duration)) ?? 0)); + return { + durationMs: Math.round(seconds * 1000), + formatName: data.format?.format_name ?? "", + ...(video?.width && video.height + ? { video: probeVideo(video, video.width, video.height) } + : {}), + ...(audio + ? { + audio: { + codec: audio.codec_name ?? "", + sampleRate: Number(audio.sample_rate) || 0, + channels: audio.channels ?? 0, + }, + } + : {}), + hasAudio: Boolean(audio), + }; +} + +function probeVideo(stream: ProbeStream, width: number, height: number) { + const rotation = normalizeRotation( + stream.side_data_list?.find((entry) => typeof entry.rotation === "number")?.rotation ?? + Number(stream.tags?.rotate ?? 0), + ); + const sideways = rotation === 90 || rotation === 270; + const sar = ratio(stream.sample_aspect_ratio, ":") || 1; + const fps = ratio(stream.avg_frame_rate, "/") || ratio(stream.r_frame_rate, "/"); + return { + codec: stream.codec_name ?? "", + width: sideways ? height : width, + height: sideways ? width : height, + rotation, + fps: fps > 0 && fps <= MAX_PLAUSIBLE_FPS ? fps : 0, + sampleAspectRatio: sideways ? 1 / sar : sar, + }; +} + +function normalizeRotation(degrees: number): 0 | 90 | 180 | 270 { + if (!Number.isFinite(degrees)) return 0; + const quarter = (((Math.round(degrees / 90) % 4) + 4) % 4) as 0 | 1 | 2 | 3; + return ([0, 90, 180, 270] as const)[quarter]; +} + +function ratio(value: string | undefined, separator: string): number { + if (!value) return 0; + const [numerator, denominator] = value.split(separator).map(Number); + if (!numerator || !denominator || !Number.isFinite(numerator / denominator)) return 0; + return numerator / denominator; +} + +function positive(value: number): number | undefined { + return Number.isFinite(value) && value > 0 ? value : undefined; +} + +/** Feeds ffmpeg `-progress` output (key=value lines) and reports each completed block. */ +export function createFfmpegProgressParser( + onUpdate: (progress: FfmpegProgress) => void, +): (chunk: string) => void { + let buffer = ""; + let current: Omit = {}; + return (chunk) => { + buffer += chunk; + for (let newline = buffer.indexOf("\n"); newline >= 0; newline = buffer.indexOf("\n")) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + const separator = line.indexOf("="); + if (separator < 0) continue; + const key = line.slice(0, separator); + const value = line.slice(separator + 1).trim(); + const number = Number(value); + // out_time_ms is microseconds too (a long-standing ffmpeg naming bug). + if ((key === "out_time_us" || key === "out_time_ms") && value && number >= 0) { + current.outTimeUs = number; + } else if (key === "total_size" && value && number >= 0) { + current.totalSize = number; + } else if (key === "progress") { + onUpdate({ ...current, done: value === "end" }); + current = {}; + } + } + }; +} + +/** Runs ffmpeg with progress on stdout; rejects with the stderr tail when it fails. */ +export async function runFfmpeg( + args: readonly string[], + options: RunFfmpegOptions = {}, +): Promise { + options.signal?.throwIfAborted(); + const ffmpegPath = options.ffmpegPath ?? (await requireFfmpeg()).ffmpegPath; + try { + await spawnFfmpeg(ffmpegPath, args, options); + } catch (error) { + await Promise.all((options.outputs ?? []).map((path) => rm(path, { force: true }))); + throw error; + } +} + +function spawnFfmpeg( + ffmpegPath: string, + args: readonly string[], + options: RunFfmpegOptions, +): Promise { + const { signal, durationMs, onProgress, maxOutputBytes } = options; + return new Promise((resolve, reject) => { + const child = spawn( + ffmpegPath, + [ + "-hide_banner", + "-nostdin", + "-nostats", + "-loglevel", + "error", + "-progress", + "pipe:1", + "-y", + ...args, + ], + { cwd: options.cwd, stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + if (child.pid) { + try { + // Long encodes must not starve the API process that shares this machine. + setPriority(child.pid, 10); + } catch { + // Lowering priority is best effort. + } + } + let stderr = ""; + let settled = false; + let reported = 0; + let failure: unknown; + let stallTimer: NodeJS.Timeout | undefined; + const finish = (error?: unknown) => { + if (settled) return; + settled = true; + clearTimeout(stallTimer); + signal?.removeEventListener("abort", abort); + if (error === undefined) resolve(); + else reject(error); + }; + // Settles once the process is gone, so callers can delete its files right away. + const kill = (error: unknown) => { + if (failure !== undefined || settled) return; + failure = error; + clearTimeout(stallTimer); + child.kill("SIGKILL"); + setTimeout(() => finish(error), KILL_GRACE_MS).unref?.(); + }; + const abort = () => kill(signal?.reason ?? new Error("aborted")); + const stallTimeoutMs = options.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS; + const armStallTimer = () => { + clearTimeout(stallTimer); + stallTimer = setTimeout( + () => kill(new FfmpegError("ffmpeg stopped making progress.", null, tail(stderr))), + stallTimeoutMs, + ); + stallTimer.unref?.(); + }; + const parse = createFfmpegProgressParser((progress) => { + if (failure !== undefined) return; + if (maxOutputBytes !== undefined && (progress.totalSize ?? 0) > maxOutputBytes) { + kill(new Error(outputTooLargeMessage(maxOutputBytes))); + return; + } + if (!onProgress) return; + const fraction = progress.done + ? 1 + : durationMs && progress.outTimeUs !== undefined + ? Math.min(1, progress.outTimeUs / (durationMs * 1000)) + : 0; + if (fraction > reported) { + reported = fraction; + onProgress(fraction); + } + }); + signal?.addEventListener("abort", abort, { once: true }); + armStallTimer(); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + if (failure === undefined) armStallTimer(); + parse(chunk); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr = tail(stderr + chunk); + }); + child.on("error", (error) => finish(failure ?? error)); + child.on("close", (code) => { + if (failure !== undefined) finish(failure); + else if (code === 0) finish(); + else finish(ffmpegFailure("ffmpeg", code, stderr)); + }); + if (signal?.aborted) abort(); + }); +} + +export function outputTooLargeMessage(maxBytes: number): string { + return `The result would exceed ${Math.round(maxBytes / (1024 * 1024))} MB.`; +} + +function tail(text: string): string { + return text.length > STDERR_TAIL_BYTES ? text.slice(-STDERR_TAIL_BYTES) : text; +} + +function ffmpegFailure(tool: string, code: number | null, stderr: string): FfmpegError { + // The cause is usually the line before ffmpeg's generic summary ("Conversion failed!"). + const lines = stderr + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .slice(-2); + // Name files by their base name only; the job's temp paths mean nothing to the reader. + const detail = lines.join(" ").replaceAll(/(?:file:)?(?:[A-Za-z]:)?[\\/][^\s'"]*[\\/]/g, ""); + return new FfmpegError( + detail ? `${tool} failed: ${detail}` : `${tool} failed with exit code ${code}.`, + code, + tail(stderr), + ); +} + +/** Extracts the first audio track as 16 kHz mono 16-bit WAV, the format speech models expect. */ +export async function extractAudio( + input: string, + output: string, + options: { + signal?: AbortSignal; + mimeType?: string; + startMs?: number; + endMs?: number; + onProgress?: (fraction: number) => void; + } = {}, +): Promise { + const tools = await requireFfmpeg(); + const probe = await probeMedia(input, { + signal: options.signal, + mimeType: options.mimeType, + ffprobePath: tools.ffprobePath, + }); + if (!probe.hasAudio) throw new Error("The file has no audio track."); + const startMs = Math.max(0, options.startMs ?? 0); + const endMs = Math.min(probe.durationMs, options.endMs ?? probe.durationMs); + if (endMs <= startMs) throw new Error("The audio range is empty."); + await runFfmpeg( + [ + ...ffmpegInputArgs(input, { + mimeType: options.mimeType, + startSeconds: startMs / 1000, + durationSeconds: (endMs - startMs) / 1000, + }), + "-map", + "0:a:0", + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "pcm_s16le", + "-map_metadata", + "-1", + "-f", + "wav", + `file:${output}`, + ], + { + signal: options.signal, + durationMs: endMs - startMs, + onProgress: options.onProgress, + outputs: [output], + ffmpegPath: tools.ffmpegPath, + }, + ); +} + +/** Writes one JPEG frame at atMs (default 0), falling back to the first frame past the end. */ +export async function renderPoster( + input: string, + output: string, + options: { signal?: AbortSignal; mimeType?: string; atMs?: number } = {}, +): Promise { + const ffmpegPath = (await requireFfmpeg()).ffmpegPath; + const render = (atMs: number) => + runFfmpeg( + [ + ...ffmpegInputArgs(input, { mimeType: options.mimeType, startSeconds: atMs / 1000 }), + "-map", + "0:v:0", + "-frames:v", + "1", + "-c:v", + "mjpeg", + "-q:v", + "3", + "-map_metadata", + "-1", + "-f", + "image2", + "-update", + "1", + `file:${output}`, + ], + { signal: options.signal, outputs: [output], ffmpegPath }, + ); + const atMs = Math.max(0, options.atMs ?? 0); + await render(atMs); + // Seeking into the last frame's gap writes nothing; the first frame is better than no poster. + if (atMs > 0 && !(await nonEmptyFile(output))) await render(0); + if (!(await nonEmptyFile(output))) throw new Error("Could not render a poster frame."); +} + +async function nonEmptyFile(path: string): Promise { + try { + return (await stat(path)).size > 0; + } catch { + return false; + } +} + +interface CaptureResult { + code: number | null; + stdout: string; + stderr: string; +} + +function capture( + command: string, + args: readonly string[], + options: { signal?: AbortSignal; timeoutMs: number }, +): Promise { + options.signal?.throwIfAborted(); + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }); + const stdout: Buffer[] = []; + let stderr = ""; + let settled = false; + const finish = (error: unknown, result?: CaptureResult) => { + if (settled) return; + settled = true; + clearTimeout(timer); + options.signal?.removeEventListener("abort", abort); + if (result) resolve(result); + else reject(error); + }; + const kill = (error: unknown) => { + child.kill("SIGKILL"); + finish(error); + }; + const abort = () => kill(options.signal?.reason ?? new Error("aborted")); + const timer = setTimeout( + () => kill(new Error(`${command} timed out after ${options.timeoutMs} ms`)), + options.timeoutMs, + ); + timer.unref?.(); + options.signal?.addEventListener("abort", abort, { once: true }); + child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk)); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr = tail(stderr + chunk); + }); + child.on("error", (error) => finish(error)); + child.on("close", (code) => + finish(undefined, { code, stdout: Buffer.concat(stdout).toString("utf8"), stderr }), + ); + }); +} diff --git a/packages/adapters/src/media/media-test-support.ts b/packages/adapters/src/media/media-test-support.ts new file mode 100644 index 0000000..940419a --- /dev/null +++ b/packages/adapters/src/media/media-test-support.ts @@ -0,0 +1,106 @@ +import { copyFile, mkdtemp, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit"; +import type { MediaOutput, MediaOutputKind } from "@rakazo/contracts"; +import { attachmentKindOf } from "@rakazo/contracts"; +import { runFfmpeg } from "./ffmpeg.js"; + +export interface TestClips { + dir: string; + /** 320×180, 25 fps, 3 s, 440 Hz tone. */ + landscape: string; + /** 180×320, 30 fps, 2 s, no audio. */ + silentPortrait: string; + srt: string; + vtt: string; +} + +/** Tiny generated clips; nothing binary is committed. */ +export async function createTestClips(): Promise { + const dir = await mkdtemp(join(tmpdir(), "evoo-media-test-")); + const landscape = join(dir, "landscape.mp4"); + const silentPortrait = join(dir, "silent-portrait.mp4"); + const encode = ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"]; + await runFfmpeg([ + ...["-f", "lavfi", "-i", "testsrc=size=320x180:rate=25:duration=3"], + ...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=3"], + ...encode, + ...["-c:a", "aac", "-shortest", `file:${landscape}`], + ]); + await runFfmpeg([ + ...["-f", "lavfi", "-i", "testsrc2=size=180x320:rate=30:duration=2"], + ...encode, + `file:${silentPortrait}`, + ]); + const srt = join(dir, "captions.srt"); + await writeFile( + srt, + "1\n00:00:00,100 --> 00:00:01,500\nHello from the subtitles\n\n2\n00:00:01,500 --> 00:00:02,900\nZweite Zeile äöü\n", + ); + const vtt = join(dir, "captions.vtt"); + await writeFile(vtt, "WEBVTT\n\n00:00.100 --> 00:01.500\nHello from WebVTT\n"); + return { dir, landscape, silentPortrait, srt, vtt }; +} + +export interface TestArtifact { + path: string; + name: string; + mimeType: string; + kind?: MediaOutputKind; +} + +/** A MediaJobContext over local files that records progress and outputs. */ +export async function fakeMediaJobContext(input: unknown, artifacts: Record) { + const tempDir = await mkdtemp(join(tmpdir(), "evoo-media-job-")); + const controller = new AbortController(); + const progress: number[] = []; + const outputs: (MediaJobOutputFile & { size: number; posterSize?: number })[] = []; + const context: MediaJobContext = { + jobId: "job-1", + spaceId: "space-1", + userId: "user-1", + botId: null, + input, + tempDir, + signal: controller.signal, + progress: async (fraction) => { + progress.push(fraction); + }, + fetchArtifact: async (artifactId): Promise => { + const artifact = artifacts[artifactId]; + if (!artifact) throw new Error(`Artifact ${artifactId} not found.`); + const path = join(tempDir, `input-${artifactId}`); + await copyFile(artifact.path, path); + return { + artifactId, + path, + name: artifact.name, + mimeType: artifact.mimeType, + kind: artifact.kind ?? attachmentKindOf(artifact.mimeType), + size: (await stat(path)).size, + }; + }, + addOutput: async (file): Promise => { + // Files must exist when handed over; the runner stores them before the job ends. + const size = (await stat(file.path)).size; + const posterSize = file.posterPath ? (await stat(file.posterPath)).size : undefined; + outputs.push({ ...file, size, ...(posterSize !== undefined ? { posterSize } : {}) }); + const index = outputs.length; + return { + artifactId: `output-${index}`, + kind: attachmentKindOf(file.mimeType), + mimeType: file.mimeType, + name: file.name, + size, + ...(file.width ? { width: file.width } : {}), + ...(file.height ? { height: file.height } : {}), + ...(file.durationMs !== undefined ? { durationMs: file.durationMs } : {}), + ...(file.posterPath ? { posterArtifactId: `poster-${index}` } : {}), + ...(file.parentArtifactId ? { parentArtifactId: file.parentArtifactId } : {}), + ...(file.preset ? { preset: file.preset } : {}), + }; + }, + }; + return { context, controller, progress, outputs, tempDir }; +} diff --git a/packages/adapters/src/media/video-edit-plan.test.ts b/packages/adapters/src/media/video-edit-plan.test.ts new file mode 100644 index 0000000..ea37cd6 --- /dev/null +++ b/packages/adapters/src/media/video-edit-plan.test.ts @@ -0,0 +1,301 @@ +import type { EditVideoStep } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { MediaProbe } from "./ffmpeg.js"; +import type { PlanInput } from "./video-edit-plan.js"; +import { atempoFilters, planVideoEdit, subtitleStyle } from "./video-edit-plan.js"; + +function probe( + overrides: Partial> & { + durationMs?: number; + hasAudio?: boolean; + } = {}, +): MediaProbe { + const { durationMs = 10_000, hasAudio = true, ...video } = overrides; + return { + durationMs, + formatName: "mov,mp4,m4a,3gp,3g2,mj2", + video: { + codec: "h264", + width: 1920, + height: 1080, + rotation: 0, + fps: 25, + sampleAspectRatio: 1, + ...video, + }, + ...(hasAudio ? { audio: { codec: "aac", sampleRate: 44_100, channels: 2 } } : {}), + hasAudio, + }; +} + +const source: PlanInput = { + path: "/jobs/1/input-source", + mimeType: "video/mp4", + name: "holiday.mp4", + probe: probe(), +}; + +function plan(steps: EditVideoStep[], extra: Partial[0]> = {}) { + return planVideoEdit({ source, steps, outputPath: "/jobs/1/output.mp4", ...extra }); +} + +function graphOf(args: string[]): string { + return args[args.indexOf("-filter_complex") + 1]!; +} + +describe("planVideoEdit", () => { + it("seeks the input for a leading trim and encodes H.264/AAC MP4", () => { + const result = plan([{ op: "trim", start: 1.5, end: 4 }]); + expect(result.args).toEqual([ + ...["-ss", "1.5", "-t", "2.5", "-protocol_whitelist", "file", "-f", "mov"], + ...["-i", "file:/jobs/1/input-source"], + "-filter_complex", + "[0:v:0]format=yuv420p[v1];[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a2]", + ...["-map", "[v1]", "-map", "[a2]", "-c:a", "aac", "-b:a", "160k"], + ...["-c:v", "libx264", "-preset", "veryfast", "-crf", "21", "-pix_fmt", "yuv420p"], + ...["-movflags", "+faststart", "-map_metadata", "-1", "-map_chapters", "-1"], + ...["-f", "mp4", "file:/jobs/1/output.mp4"], + ]); + expect(result).toMatchObject({ kind: "video", durationMs: 2500, width: 1920, height: 1080 }); + }); + + it("keeps sorted, merged segments from one seek with split, trims and concat", () => { + const result = plan([ + { + op: "keep", + segments: [ + { start: 6, end: 8 }, + { start: 2, end: 3 }, + { start: 2.5, end: 4 }, + ], + }, + ]); + expect(result.args.slice(0, 4)).toEqual(["-ss", "2", "-t", "6"]); + const graph = graphOf(result.args); + expect(graph).toContain("[0:v:0]split=2[v1][v2]"); + expect(graph).toContain("[v1]trim=start=0:end=2,setpts=PTS-STARTPTS[v3]"); + expect(graph).toContain("[v2]trim=start=4:end=6,setpts=PTS-STARTPTS[v4]"); + expect(graph).toContain("[0:a:0]asplit=2[a5][a6]"); + expect(graph).toContain("[v3][a7][v4][a8]concat=n=2:v=1:a=1[v9][a10]"); + expect(result.durationMs).toBe(4000); + }); + + it("applies later cuts on the timeline left by earlier steps", () => { + const result = plan([ + { op: "speed", factor: 2 }, + { op: "trim", start: 1, end: 99 }, + ]); + expect(graphOf(result.args)).toContain( + "[0:v:0]setpts=PTS/2,fps=25,trim=start=1:end=5,setpts=PTS-STARTPTS,format=yuv420p[v1]", + ); + expect(result.durationMs).toBe(4000); + }); + + it("rejects ranges outside the video and too short ones", () => { + expect(() => plan([{ op: "trim", start: 12, end: 18 }])).toThrow( + "trim 12–18 s is outside the 10 s video.", + ); + expect(() => + plan([ + { op: "speed", factor: 2 }, + { + op: "keep", + segments: [ + { start: 1, end: 2 }, + { start: 6, end: 7 }, + ], + }, + ]), + ).toThrow("keep 6–7 s is outside the 5 s video."); + expect(() => plan([{ op: "trim", start: 9.95, end: 12 }])).toThrow("shorter than 0.1 s"); + }); + + it("appends clips fitted to the frame, with silence for clips without audio", () => { + const clip: PlanInput = { + path: "/jobs/1/input-clip", + mimeType: "video/webm", + name: "clip.webm", + probe: probe({ width: 1080, height: 1920, fps: 30, durationMs: 2000, hasAudio: false }), + }; + const result = plan([{ op: "concat", artifact_ids: ["clip"] }], { + clips: new Map([["clip", clip]]), + }); + expect(result.args).toContain("matroska"); + const graph = graphOf(result.args); + expect(graph).toContain("[0:v:0]setsar=1,format=yuv420p[v1]"); + expect(graph).toContain( + "[1:v:0]scale=1920:1080:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1920:1080:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=25,format=yuv420p[v3]", + ); + expect(graph).toContain( + "anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=2,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a4]", + ); + expect(graph).toContain("[v1][a2][v3][a4]concat=n=2:v=1:a=1"); + expect(result.durationMs).toBe(12_000); + }); + + it("adds silence to a silent source when an appended clip has audio", () => { + const silent = { ...source, probe: probe({ hasAudio: false }) }; + const result = planVideoEdit({ + source: silent, + steps: [{ op: "concat", artifact_ids: ["clip"] }], + clips: new Map([["clip", source]]), + outputPath: "/out.mp4", + }); + expect(graphOf(result.args)).toContain( + "anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=10", + ); + expect(result.args.filter((arg) => arg === "-map")).toHaveLength(2); + }); + + it("appends clips with the last reframe's fit", () => { + const result = plan( + [ + { op: "reframe", preset: "vertical", fit: "crop" }, + { op: "concat", artifact_ids: ["clip"] }, + ], + { clips: new Map([["clip", source]]) }, + ); + expect(graphOf(result.args)).toContain( + "[1:v:0]scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920,setsar=1,fps=25", + ); + }); + + it("reframes by crop, pad or blur, and only scales when the aspect already matches", () => { + expect(graphOf(plan([{ op: "reframe", preset: "square", fit: "crop" }]).args)).toContain( + "scale=1080:1080:force_original_aspect_ratio=increase,crop=1080:1080,setsar=1", + ); + expect(graphOf(plan([{ op: "reframe", preset: "portrait", fit: "pad" }]).args)).toContain( + "pad=1080:1350:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1", + ); + const blur = plan([{ op: "reframe", preset: "vertical", fit: "blur" }]); + expect(graphOf(blur.args)).toBe( + [ + "[0:v:0]split=2[v1][v2]", + "[v1]scale=270:480:force_original_aspect_ratio=increase,crop=270:480,boxblur=27:2,scale=1080:1920,setsar=1[v3]", + "[v2]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,setsar=1[v4]", + "[v3][v4]overlay=(W-w)/2:(H-h)/2[v5]", + "[v5]format=yuv420p[v6]", + "[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a7]", + ].join(";"), + ); + expect(blur).toMatchObject({ width: 1080, height: 1920 }); + expect(graphOf(plan([{ op: "reframe", preset: "landscape", fit: "blur" }]).args)).toContain( + "[0:v:0]scale=1920:1080,setsar=1,format=yuv420p", + ); + }); + + it("changes speed with atempo chains and keeps the source frame rate", () => { + const fast = plan([{ op: "speed", factor: 4 }]); + expect(graphOf(fast.args)).toContain("setpts=PTS/4,fps=25"); + expect(graphOf(fast.args)).toContain("atempo=2,atempo=2"); + expect(fast.durationMs).toBe(2500); + expect(atempoFilters(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]); + expect(atempoFilters(3)).toEqual(["atempo=2", "atempo=1.5"]); + expect(atempoFilters(0.8)).toEqual(["atempo=0.8"]); + }); + + it("changes volume, and mute drops the audio track", () => { + expect(graphOf(plan([{ op: "volume", gain: 0.5 }]).args)).toContain( + "[0:a:0]volume=0.5,aformat", + ); + const muted = plan([{ op: "volume", gain: 2 }, { op: "mute" }]); + expect(muted.args.filter((arg) => arg === "-map")).toHaveLength(1); + expect(muted.args).not.toContain("aac"); + }); + + it("burns in subtitles styled for the current frame from a generated file name", () => { + const result = plan( + [ + { op: "reframe", preset: "vertical", fit: "crop" }, + { op: "subtitles", artifact_id: "subs" }, + ], + { subtitleFiles: new Map([["subs", "subtitles-0.srt"]]) }, + ); + expect(graphOf(result.args)).toContain( + `subtitles=filename=subtitles-0.srt:force_style='${subtitleStyle(1080, 1920)}'`, + ); + expect(() => + plan([{ op: "subtitles", artifact_id: "subs" }], { + subtitleFiles: new Map([["subs", "x.srt:force_style='a'"]]), + }), + ).toThrow("Unsafe subtitle file name."); + }); + + it("styles subtitles above the controls of tall frames", () => { + expect(subtitleStyle(1080, 1920)).toContain("MarginV=384"); + expect(subtitleStyle(1080, 1920)).toContain("FontSize=65"); + expect(subtitleStyle(1920, 1080)).toContain("MarginV=65"); + expect(subtitleStyle(1920, 1080)).toContain("PlayResX=1920,PlayResY=1080"); + }); + + it("extracts audio without decoding video work", () => { + const result = planVideoEdit({ + source, + steps: [ + { op: "trim", start: 0, end: 5 }, + { op: "reframe", preset: "vertical", fit: "blur" }, + { op: "subtitles", artifact_id: "subs" }, + { op: "extract_audio" }, + ], + subtitleFiles: new Map([["subs", "subtitles-0.srt"]]), + outputPath: "/out.m4a", + }); + expect(graphOf(result.args)).toBe( + "[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a1]", + ); + expect(result.args).toEqual(expect.arrayContaining(["-map", "[a1]", "-b:a", "192k"])); + expect(result.args).not.toContain("libx264"); + expect(result).toMatchObject({ kind: "audio", durationMs: 5000 }); + expect(() => + planVideoEdit({ + source: { ...source, probe: probe({ hasAudio: false }) }, + steps: [{ op: "extract_audio" }], + outputPath: "/out.m4a", + }), + ).toThrow("holiday.mp4 has no audio."); + }); + + it("validates the poster time against the result", () => { + expect( + plan([ + { op: "trim", start: 0, end: 4 }, + { op: "poster", at: 3 }, + ]).posterAtMs, + ).toBe(3000); + expect(() => + plan([ + { op: "trim", start: 0, end: 4 }, + { op: "poster", at: 5 }, + ]), + ).toThrow("poster at 5 s is past the end of the 4 s result."); + }); + + it("keeps file paths out of the filter graph", () => { + const hostile = { ...source, path: "/jobs/1/a'b[c]:d;e,f.mp4", name: "x'];[0:v]drawtext=" }; + const result = planVideoEdit({ + source: hostile, + steps: [{ op: "reframe", preset: "square", fit: "blur" }], + outputPath: "/jobs/1/out.mp4", + }); + expect(graphOf(result.args)).not.toContain("a'b"); + expect(graphOf(result.args)).not.toContain("drawtext"); + expect(result.args).toContain("file:/jobs/1/a'b[c]:d;e,f.mp4"); + }); + + it("squares pixels and evens odd frame sizes", () => { + const anamorphic = { + ...source, + probe: probe({ width: 1440, height: 1080, sampleAspectRatio: 4 / 3 }), + }; + const result = planVideoEdit({ + source: anamorphic, + steps: [{ op: "mute" }], + outputPath: "/o.mp4", + }); + expect(graphOf(result.args)).toBe("[0:v:0]scale=1920:1080,setsar=1,format=yuv420p[v1]"); + expect(result).toMatchObject({ width: 1920, height: 1080 }); + const odd = { ...source, probe: probe({ width: 721, height: 405 }) }; + const evened = planVideoEdit({ source: odd, steps: [{ op: "mute" }], outputPath: "/o.mp4" }); + expect(graphOf(evened.args)).toBe("[0:v:0]crop=720:404:0:0,format=yuv420p[v1]"); + }); +}); diff --git a/packages/adapters/src/media/video-edit-plan.ts b/packages/adapters/src/media/video-edit-plan.ts new file mode 100644 index 0000000..f39f8d5 --- /dev/null +++ b/packages/adapters/src/media/video-edit-plan.ts @@ -0,0 +1,571 @@ +import type { EditVideoStep, MediaFit } from "@rakazo/contracts"; +import { MEDIA_PRESETS } from "@rakazo/contracts"; +import type { MediaProbe } from "./ffmpeg.js"; +import { ffmpegInputArgs, ffmpegSeconds } from "./ffmpeg.js"; + +/** + * Turns an ordered list of edit steps into one ffmpeg invocation with a single filter_complex + * pass. Pure: the caller probes the inputs and runs the arguments. + */ + +export interface PlanInput { + path: string; + mimeType: string; + name: string; + probe: MediaProbe; +} + +export interface VideoEditPlanRequest { + source: PlanInput; + steps: readonly EditVideoStep[]; + /** Videos named by concat steps, by artifact id. */ + clips?: ReadonlyMap; + /** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */ + subtitleFiles?: ReadonlyMap; + outputPath: string; +} + +export interface VideoEditPlan { + args: string[]; + kind: "video" | "audio"; + durationMs: number; + width?: number; + height?: number; + posterAtMs?: number; +} + +export const VIDEO_ENCODE_ARGS = [ + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "21", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", +]; +const AUDIO_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "160k"]; +const AUDIO_ONLY_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"]; +/** Location and other camera metadata stay private; chapters do not survive cuts anyway. */ +const STRIP_METADATA_ARGS = ["-map_metadata", "-1", "-map_chapters", "-1"]; +const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo"; +const SAFE_FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const MIN_SEGMENT_SECONDS = 0.1; +const FALLBACK_FPS = 30; + +interface Track { + label: string; + filters: string[]; +} + +interface Timeline { + /** Null when the result is audio only: video work is skipped. */ + video: Track | null; + audio: Track | null; + width: number; + height: number; + fps: number; + seconds: number; + /** How appended clips fill the frame: the last reframe's fit, otherwise bars. */ + fit: MediaFit; +} + +class FilterGraph { + private readonly chains: string[] = []; + private count = 0; + + label(kind: "v" | "a"): string { + this.count += 1; + return `${kind}${this.count}`; + } + + add(inputs: readonly string[], filters: readonly string[], outputs: readonly string[]) { + const pads = (labels: readonly string[]) => labels.map((label) => `[${label}]`).join(""); + this.chains.push(`${pads(inputs)}${filters.join(",")}${pads(outputs)}`); + } + + /** Ends a track's pending filters in a labelled pad other chains can consume. */ + flush(track: Track, kind: "v" | "a"): string { + if (track.filters.length === 0) return track.label; + const output = this.label(kind); + this.add([track.label], track.filters, [output]); + track.label = output; + track.filters = []; + return output; + } + + toString(): string { + return this.chains.join(";"); + } +} + +export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { + const { source } = request; + const video = source.probe.video; + if (!video) throw new Error(`${source.name} has no video.`); + const sourceSeconds = source.probe.durationMs / 1000; + const audioOnly = request.steps.at(-1)?.op === "extract_audio"; + const graph = new FilterGraph(); + const inputs: string[][] = []; + const addInput = (input: PlanInput, seek?: { start: number; end: number }) => { + inputs.push( + ffmpegInputArgs(input.path, { + mimeType: input.mimeType, + ...(seek ? { startSeconds: seek.start, durationSeconds: seek.end - seek.start } : {}), + }), + ); + return inputs.length - 1; + }; + + let steps = request.steps; + // A leading cut seeks the input instead of decoding everything before it. + const first = steps[0]; + let segments: Segment[] | undefined; + if (first?.op === "trim" || first?.op === "keep") { + segments = normalizeSegments( + first.op === "trim" ? [first] : first.segments, + sourceSeconds, + first.op, + ); + steps = steps.slice(1); + } + const seek = segments && { start: segments[0]!.start, end: segments.at(-1)!.end }; + const index = addInput(source, seek); + const size = displaySize(video); + const timeline: Timeline = { + video: audioOnly ? null : { label: `${index}:v:0`, filters: sourceFilters(video) }, + audio: source.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null, + width: size.width, + height: size.height, + fps: video.fps, + seconds: seek ? seek.end - seek.start : sourceSeconds, + fit: "pad", + }; + if (segments && seek && segments.length > 1) { + keepSegments( + graph, + timeline, + segments.map((segment) => ({ + start: segment.start - seek.start, + end: segment.end - seek.start, + })), + ); + } + + let posterAt: number | undefined; + for (const step of steps) { + switch (step.op) { + case "trim": + case "keep": + keepSegments( + graph, + timeline, + normalizeSegments(step.op === "trim" ? [step] : step.segments, timeline.seconds, step.op), + ); + break; + case "concat": + appendClips( + graph, + timeline, + step.artifact_ids.map((id) => { + const clip = request.clips?.get(id); + if (!clip) throw new Error(`Video ${id} is not available.`); + return { clip, index: addInput(clip) }; + }), + ); + break; + case "reframe": { + const preset = MEDIA_PRESETS[step.preset]; + if (timeline.video) { + timeline.video = fitFrame( + graph, + timeline.video, + timeline, + { width: preset.width, height: preset.height }, + step.fit, + ); + } + timeline.width = preset.width; + timeline.height = preset.height; + timeline.fit = step.fit; + break; + } + case "speed": + timeline.video?.filters.push( + `setpts=PTS/${number(step.factor)}`, + ...(timeline.fps ? [`fps=${number(timeline.fps)}`] : []), + ); + timeline.audio?.filters.push(...atempoFilters(step.factor)); + timeline.seconds /= step.factor; + break; + case "volume": + timeline.audio?.filters.push(`volume=${number(step.gain)}`); + break; + case "mute": + timeline.audio = null; + break; + case "subtitles": { + const file = request.subtitleFiles?.get(step.artifact_id); + if (!file) throw new Error(`Subtitles ${step.artifact_id} are not available.`); + // Only generated names reach the filter graph, so nothing here needs escaping. + if (!SAFE_FILE_NAME.test(file)) throw new Error("Unsafe subtitle file name."); + timeline.video?.filters.push( + `subtitles=filename=${file}:force_style='${subtitleStyle(timeline.width, timeline.height)}'`, + ); + break; + } + case "extract_audio": + break; + case "poster": + posterAt = step.at; + break; + } + } + + if (posterAt !== undefined && posterAt >= timeline.seconds) { + throw new Error( + `poster at ${seconds(posterAt)} s is past the end of the ${seconds(timeline.seconds)} s result.`, + ); + } + const durationMs = Math.round(timeline.seconds * 1000); + const output = `file:${request.outputPath}`; + if (audioOnly || !timeline.video) { + if (!timeline.audio) throw new Error(`${source.name} has no audio.`); + timeline.audio.filters.push(AUDIO_FORMAT); + const audio = graph.flush(timeline.audio, "a"); + return { + args: [ + ...inputs.flat(), + "-filter_complex", + graph.toString(), + "-map", + `[${audio}]`, + ...AUDIO_ONLY_ENCODE_ARGS, + ...STRIP_METADATA_ARGS, + "-f", + "mp4", + output, + ], + kind: "audio", + durationMs, + }; + } + timeline.video.filters.push("format=yuv420p"); + const videoOut = graph.flush(timeline.video, "v"); + let audioOut: string | undefined; + if (timeline.audio) { + timeline.audio.filters.push(AUDIO_FORMAT); + audioOut = graph.flush(timeline.audio, "a"); + } + return { + args: [ + ...inputs.flat(), + "-filter_complex", + graph.toString(), + "-map", + `[${videoOut}]`, + ...(audioOut ? ["-map", `[${audioOut}]`, ...AUDIO_ENCODE_ARGS] : []), + ...VIDEO_ENCODE_ARGS, + ...STRIP_METADATA_ARGS, + "-f", + "mp4", + output, + ], + kind: "video", + durationMs, + width: timeline.width, + height: timeline.height, + ...(posterAt !== undefined ? { posterAtMs: Math.round(posterAt * 1000) } : {}), + }; +} + +interface Segment { + start: number; + end: number; +} + +/** Validates ranges against the timeline, clamps ends to it, sorts and merges overlaps. */ +function normalizeSegments( + ranges: readonly Segment[], + duration: number, + op: "trim" | "keep", +): Segment[] { + const clamped = ranges.map((range) => { + if (range.start >= duration) { + throw new Error( + `${op} ${seconds(range.start)}–${seconds(range.end)} s is outside the ${seconds(duration)} s video.`, + ); + } + const end = Math.min(range.end, duration); + if (end - range.start < MIN_SEGMENT_SECONDS) { + throw new Error( + `${op} ${seconds(range.start)}–${seconds(end)} s is shorter than ${MIN_SEGMENT_SECONDS} s.`, + ); + } + return { start: range.start, end }; + }); + clamped.sort((a, b) => a.start - b.start); + const merged: Segment[] = []; + for (const segment of clamped) { + const last = merged.at(-1); + if (last && segment.start <= last.end) last.end = Math.max(last.end, segment.end); + else merged.push({ ...segment }); + } + return merged; +} + +/** Keeps sorted, disjoint segments of the timeline: one trim, or split + trims + concat. */ +function keepSegments(graph: FilterGraph, timeline: Timeline, segments: readonly Segment[]) { + const [only] = segments; + if (segments.length === 1 && only) { + const range = `start=${ffmpegSeconds(only.start)}:end=${ffmpegSeconds(only.end)}`; + timeline.video?.filters.push(`trim=${range}`, "setpts=PTS-STARTPTS"); + timeline.audio?.filters.push(`atrim=${range}`, "asetpts=PTS-STARTPTS"); + timeline.seconds = only.end - only.start; + return; + } + const count = segments.length; + const branch = (track: Track | null, kind: "v" | "a"): string[] => { + if (!track) return []; + const input = graph.flush(track, kind); + const splits = segments.map(() => graph.label(kind)); + graph.add([input], [`${kind === "v" ? "split" : "asplit"}=${count}`], splits); + return segments.map((segment, index) => { + const output = graph.label(kind); + const range = `start=${ffmpegSeconds(segment.start)}:end=${ffmpegSeconds(segment.end)}`; + graph.add( + [splits[index]!], + kind === "v" + ? [`trim=${range}`, "setpts=PTS-STARTPTS"] + : [`atrim=${range}`, "asetpts=PTS-STARTPTS"], + [output], + ); + return output; + }); + }; + const videoParts = branch(timeline.video, "v"); + const audioParts = branch(timeline.audio, "a"); + concatParts(graph, timeline, videoParts, audioParts, count); + timeline.seconds = segments.reduce((total, segment) => total + segment.end - segment.start, 0); +} + +/** Appends clips normalized to the timeline: frame size, fit, fps and 48 kHz stereo audio. */ +function appendClips( + graph: FilterGraph, + timeline: Timeline, + clips: readonly { clip: PlanInput; index: number }[], +) { + const fps = timeline.fps || FALLBACK_FPS; + const needsAudio = timeline.audio !== null || clips.some(({ clip }) => clip.probe.hasAudio); + const videoParts: string[] = []; + const audioParts: string[] = []; + if (timeline.video) { + timeline.video.filters.push("setsar=1", "format=yuv420p"); + videoParts.push(graph.flush(timeline.video, "v")); + } + if (needsAudio) audioParts.push(audioPart(graph, timeline.audio, timeline.seconds)); + let seconds = timeline.seconds; + for (const { clip, index } of clips) { + const video = clip.probe.video; + if (!video) throw new Error(`${clip.name} has no video.`); + const clipSeconds = clip.probe.durationMs / 1000; + if (clipSeconds <= 0) throw new Error(`Could not read the length of ${clip.name}.`); + if (timeline.video) { + const track = fitFrame( + graph, + { label: `${index}:v:0`, filters: sourceFilters(video) }, + displaySize(video), + timeline, + timeline.fit, + ); + track.filters.push(`fps=${number(fps)}`, "format=yuv420p"); + videoParts.push(graph.flush(track, "v")); + } + if (needsAudio) { + const track = clip.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null; + audioParts.push(audioPart(graph, track, clipSeconds)); + } + seconds += clipSeconds; + } + concatParts(graph, timeline, videoParts, audioParts, clips.length + 1); + timeline.seconds = seconds; +} + +/** A clip's audio in the common format, or silence as long as the clip when it has none. */ +function audioPart(graph: FilterGraph, track: Track | null, seconds: number): string { + if (track) { + track.filters.push(AUDIO_FORMAT); + return graph.flush(track, "a"); + } + const output = graph.label("a"); + graph.add( + [], + [ + "anullsrc=channel_layout=stereo:sample_rate=48000", + `atrim=end=${ffmpegSeconds(seconds)}`, + AUDIO_FORMAT, + ], + [output], + ); + return output; +} + +function concatParts( + graph: FilterGraph, + timeline: Timeline, + videoParts: readonly string[], + audioParts: readonly string[], + count: number, +) { + const inputs = Array.from({ length: count }, (_, index) => [ + ...(videoParts[index] ? [videoParts[index]] : []), + ...(audioParts[index] ? [audioParts[index]] : []), + ]).flat(); + const video = videoParts.length > 0 ? graph.label("v") : undefined; + const audio = audioParts.length > 0 ? graph.label("a") : undefined; + graph.add( + inputs, + [`concat=n=${count}:v=${video ? 1 : 0}:a=${audio ? 1 : 0}`], + [...(video ? [video] : []), ...(audio ? [audio] : [])], + ); + timeline.video = video ? { label: video, filters: [] } : null; + timeline.audio = audio ? { label: audio, filters: [] } : null; +} + +interface Size { + width: number; + height: number; +} + +/** Scales a frame of size `from` into `to`: center crop, bars, or a blurred copy behind it. */ +function fitFrame(graph: FilterGraph, track: Track, from: Size, to: Size, fit: MediaFit): Track { + const { width, height } = to; + if (Math.abs(from.width / from.height - width / height) < 0.01) { + track.filters.push(`scale=${width}:${height}`, "setsar=1"); + return track; + } + const contain = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`; + if (fit === "crop") { + track.filters.push( + `scale=${width}:${height}:force_original_aspect_ratio=increase`, + `crop=${width}:${height}`, + "setsar=1", + ); + return track; + } + if (fit === "pad") { + track.filters.push( + contain, + `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, + "setsar=1", + ); + return track; + } + // Blur a small copy and scale it up: as smooth as a large blur at a fraction of the cost. + const input = graph.flush(track, "v"); + const [background, foreground, blurred, contained, output] = [ + graph.label("v"), + graph.label("v"), + graph.label("v"), + graph.label("v"), + graph.label("v"), + ]; + const smallWidth = even(width / 4); + const smallHeight = even(height / 4); + const radius = Math.max(2, Math.floor(Math.min(smallWidth, smallHeight) / 10)); + graph.add([input], ["split=2"], [background, foreground]); + graph.add( + [background], + [ + `scale=${smallWidth}:${smallHeight}:force_original_aspect_ratio=increase`, + `crop=${smallWidth}:${smallHeight}`, + `boxblur=${radius}:2`, + `scale=${width}:${height}`, + "setsar=1", + ], + [blurred], + ); + graph.add([foreground], [contain, "setsar=1"], [contained]); + graph.add([blurred, contained], ["overlay=(W-w)/2:(H-h)/2"], [output]); + return { label: output, filters: [] }; +} + +/** Square pixels and even dimensions, which yuv420p H.264 requires. */ +function sourceFilters(video: NonNullable): string[] { + const size = displaySize(video); + if (Math.abs(video.sampleAspectRatio - 1) > 0.01) { + return [`scale=${size.width}:${size.height}`, "setsar=1"]; + } + if (size.width !== video.width || size.height !== video.height) { + return [`crop=${size.width}:${size.height}:0:0`]; + } + return []; +} + +function displaySize(video: NonNullable): Size { + if (Math.abs(video.sampleAspectRatio - 1) > 0.01) { + return { + width: even(Math.round(video.width * video.sampleAspectRatio)), + height: even(video.height), + }; + } + return { width: even(video.width), height: even(video.height) }; +} + +/** atempo takes 0.5–2 per instance on older ffmpeg builds, so larger changes chain it. */ +export function atempoFilters(factor: number): string[] { + const filters: string[] = []; + let rest = factor; + while (rest > 2) { + filters.push("atempo=2"); + rest /= 2; + } + while (rest < 0.5) { + filters.push("atempo=0.5"); + rest /= 0.5; + } + filters.push(`atempo=${number(rest)}`); + return filters; +} + +/** + * Readable burned-in subtitles: bold white text with a dark outline, sized to the frame, and + * lifted above the bottom fifth of tall frames where 9:16 apps place their own controls. + */ +export function subtitleStyle(width: number, height: number): string { + const tall = height >= width * 1.5; + const fontSize = Math.round(Math.min(width, height) * 0.06); + const marginX = Math.round(width * (tall ? 0.1 : 0.05)); + const marginY = Math.round(height * (tall ? 0.2 : 0.06)); + return [ + `PlayResX=${width}`, + `PlayResY=${height}`, + "FontName=DejaVu Sans", + `FontSize=${fontSize}`, + "Bold=1", + "PrimaryColour=&H00FFFFFF", + "OutlineColour=&H00000000", + "BackColour=&H80000000", + "BorderStyle=1", + `Outline=${Math.max(2, Math.round(fontSize / 16))}`, + "Shadow=1", + "Alignment=2", + `MarginL=${marginX}`, + `MarginR=${marginX}`, + `MarginV=${marginY}`, + ].join(","); +} + +function even(value: number): number { + return Math.max(2, 2 * Math.floor(value / 2)); +} + +function number(value: number): string { + return String(Number(value.toFixed(6))); +} + +function seconds(value: number): string { + return String(Number(value.toFixed(2))); +} diff --git a/packages/adapters/src/media/video-operations.ts b/packages/adapters/src/media/video-operations.ts new file mode 100644 index 0000000..b21d9df --- /dev/null +++ b/packages/adapters/src/media/video-operations.ts @@ -0,0 +1,331 @@ +import { copyFile, mkdtemp, open, stat } from "node:fs/promises"; +import { join } from "node:path"; +import type { + MediaJobContext, + MediaJobInputFile, + MediaOperationHandler, + MediaOperationResult, +} from "@rakazo/adapter-kit"; +import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts"; +import { + ARTIFACT_MAX_BYTES_BY_KIND, + EditVideoInputSchema, + ExportVideoInputSchema, + MEDIA_PRESETS, +} from "@rakazo/contracts"; +import type { FfmpegInfo } from "./ffmpeg.js"; +import { + detectFfmpeg, + outputTooLargeMessage, + probeMedia, + renderPoster, + runFfmpeg, +} from "./ffmpeg.js"; +import type { PlanInput, VideoEditPlan } from "./video-edit-plan.js"; +import { planVideoEdit } from "./video-edit-plan.js"; + +export interface VideoOperationsDeps { + /** Finds ffmpeg; defaults to PATH lookup. */ + detect?: () => Promise; +} + +const PROGRESS_INTERVAL_MS = 1_000; +/** Share of each output's progress spent encoding; the rest is probing and the poster. */ +const ENCODE_SHARE = 0.95; +const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024; + +/** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */ +export function createVideoOperations(deps: VideoOperationsDeps = {}) { + const detect = deps.detect ?? detectFfmpeg; + const available = async () => (await detect()) !== null; + const tools = async () => { + const info = await detect(); + if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server."); + return info; + }; + const editVideo: MediaOperationHandler = { + operation: "edit_video", + run: async (context) => runEditVideo(context, await tools()), + }; + const exportVideo: MediaOperationHandler = { + operation: "export_video", + run: async (context) => runExportVideo(context, await tools()), + }; + return [ + { + operation: "edit_video" as const, + inputSchema: EditVideoInputSchema, + title: (input: EditVideoInput) => input.title, + available, + handler: editVideo, + }, + { + operation: "export_video" as const, + inputSchema: ExportVideoInputSchema, + title: (input: ExportVideoInput) => + input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "), + available, + handler: exportVideo, + }, + ]; +} + +async function runEditVideo( + context: MediaJobContext, + tools: FfmpegInfo, +): Promise { + const input = EditVideoInputSchema.parse(context.input); + const workDir = await mkdtemp(join(context.tempDir, "edit-")); + const progress = createProgressReporter(context); + const source = await loadVideo(context, tools, input.artifact_id); + const name = cleanFileName(input.title, cleanFileName(withoutExtension(source.name), "video")); + + if (input.steps.every((step) => step.op === "poster")) { + const at = input.steps[0]?.op === "poster" ? input.steps[0].at : 0; + if (at * 1000 >= source.probe.durationMs) { + throw new Error( + `poster at ${at} s is past the end of the ${source.probe.durationMs / 1000} s video.`, + ); + } + const posterPath = join(workDir, "poster.jpg"); + await renderPoster(source.path, posterPath, { + signal: context.signal, + mimeType: source.mimeType, + atMs: at * 1000, + }); + const output = await context.addOutput({ + path: posterPath, + name: `${name}.jpg`, + mimeType: "image/jpeg", + width: source.probe.video?.width, + height: source.probe.video?.height, + parentArtifactId: source.artifactId, + }); + await progress.done(); + return { summary: `Saved the frame at ${at} s of ${source.name} as ${describe(output)}.` }; + } + + const clips = new Map(); + const subtitleFiles = new Map(); + for (const step of input.steps) { + if (step.op === "concat") { + for (const id of step.artifact_ids) { + if (!clips.has(id)) clips.set(id, await loadVideo(context, tools, id)); + } + } else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) { + subtitleFiles.set( + step.artifact_id, + await prepareSubtitles(context, step.artifact_id, workDir, subtitleFiles.size), + ); + } + } + const audioOnly = input.steps.at(-1)?.op === "extract_audio"; + const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4"); + const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath }); + const output = await encode(context, tools, plan, outputPath, workDir, progress.range(0, 1), { + name: `${name}${audioOnly ? ".m4a" : ".mp4"}`, + parentArtifactId: source.artifactId, + }); + await progress.done(); + return { + summary: `Edited ${source.name} into ${describe(output)}.`, + }; +} + +async function runExportVideo( + context: MediaJobContext, + tools: FfmpegInfo, +): Promise { + const input = ExportVideoInputSchema.parse(context.input); + const presets = [...new Set(input.presets)]; + const workDir = await mkdtemp(join(context.tempDir, "export-")); + const progress = createProgressReporter(context); + const source = await loadVideo(context, tools, input.artifact_id); + const base = cleanFileName(withoutExtension(source.name), "video"); + const outputs: MediaOutput[] = []; + for (const [index, preset] of presets.entries()) { + const outputPath = join(workDir, `${preset}.mp4`); + const plan = planVideoEdit({ + source, + steps: [{ op: "reframe", preset, fit: input.fit }], + outputPath, + }); + const share = 1 / presets.length; + outputs.push( + await encode( + context, + tools, + plan, + outputPath, + workDir, + progress.range(index * share, (index + 1) * share), + { + name: `${base}-${MEDIA_PRESETS[preset].aspect.replace(":", "x")}.mp4`, + parentArtifactId: source.artifactId, + preset, + }, + ), + ); + } + await progress.done(); + return { summary: `Exported ${source.name} as ${outputs.map(describe).join(", ")}.` }; +} + +/** Runs a plan, checks the result, renders its poster and stores it as an output. */ +async function encode( + context: MediaJobContext, + tools: FfmpegInfo, + plan: VideoEditPlan, + outputPath: string, + workDir: string, + report: (fraction: number) => void, + file: Pick[0], "name" | "parentArtifactId" | "preset">, +): Promise { + const maxBytes = ARTIFACT_MAX_BYTES_BY_KIND[plan.kind]; + await runFfmpeg(plan.args, { + signal: context.signal, + cwd: workDir, + durationMs: plan.durationMs, + onProgress: (fraction) => report(fraction * ENCODE_SHARE), + maxOutputBytes: maxBytes, + outputs: [outputPath], + ffmpegPath: tools.ffmpegPath, + }); + if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes)); + const probe = await probeMedia(outputPath, { + signal: context.signal, + mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4", + ffprobePath: tools.ffprobePath, + }); + if (plan.kind === "audio") { + const output = await context.addOutput({ + ...file, + path: outputPath, + mimeType: "audio/mp4", + durationMs: probe.durationMs, + }); + report(1); + return output; + } + const posterPath = outputPath.replace(/\.mp4$/, ".jpg"); + await renderPoster(outputPath, posterPath, { + signal: context.signal, + mimeType: "video/mp4", + atMs: plan.posterAtMs ?? Math.min(1_000, probe.durationMs / 4), + }); + const output = await context.addOutput({ + ...file, + path: outputPath, + mimeType: "video/mp4", + width: probe.video?.width ?? plan.width, + height: probe.video?.height ?? plan.height, + durationMs: probe.durationMs, + posterPath, + }); + report(1); + return output; +} + +async function loadVideo( + context: MediaJobContext, + tools: FfmpegInfo, + artifactId: string, +): Promise { + const file = await context.fetchArtifact(artifactId); + if (file.kind !== "video") throw new Error(`${file.name} is not a video.`); + const probe = await probeMedia(file.path, { + signal: context.signal, + mimeType: file.mimeType, + ffprobePath: tools.ffprobePath, + }); + if (!probe.video) throw new Error(`${file.name} has no video.`); + if (probe.durationMs <= 0) throw new Error(`Could not read the length of ${file.name}.`); + return { ...file, probe }; +} + +/** Copies an SRT/VTT artifact to a generated name, so no user text reaches the filter graph. */ +async function prepareSubtitles( + context: MediaJobContext, + artifactId: string, + workDir: string, + index: number, +): Promise { + const file = await context.fetchArtifact(artifactId); + const format = file.size <= SUBTITLE_MAX_BYTES ? subtitleFormat(await readHead(file.path)) : null; + if (!format) throw new Error(`${file.name} is not an SRT or VTT file.`); + const name = `subtitles-${index}.${format}`; + await copyFile(file.path, join(workDir, name)); + return name; +} + +async function readHead(path: string): Promise { + const handle = await open(path, "r"); + try { + const buffer = Buffer.alloc(1024); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + return buffer.subarray(0, bytesRead).toString("utf8"); + } finally { + await handle.close(); + } +} + +/** + * Recognizes WebVTT and SubRip by content. Anything else is refused, so ffmpeg never probes + * an arbitrary file (a playlist, say) handed in as subtitles. + */ +export function subtitleFormat(head: string): "srt" | "vtt" | null { + const text = head.replace(/^/, "").trimStart(); + if (/^WEBVTT(?:[ \t\r\n]|$)/.test(text)) return "vtt"; + if (/^(?:\d+[ \t]*\r?\n)?\d{1,2}:\d{2}:\d{2}[,.]\d{1,3}[ \t]*-->/.test(text)) return "srt"; + return null; +} + +/** A name for produced files: no path separators or control characters, at most 80 chars. */ +export function cleanFileName(text: string, fallback: string): string { + const name = text + .replaceAll(/[\p{Cc}\\/:*?"<>|]+/gu, " ") + .replaceAll(/\s+/g, " ") + .replace(/^[\s.]+/, "") + .slice(0, 80) + .trim(); + return name || fallback; +} + +function withoutExtension(name: string): string { + return name.replace(/\.[A-Za-z0-9]{1,5}$/, ""); +} + +function describe(output: MediaOutput): string { + const size = output.width && output.height ? ` ${output.width}×${output.height}` : ""; + const length = + output.durationMs !== undefined ? ` ${Number((output.durationMs / 1000).toFixed(1))} s` : ""; + return `${output.name}${size}${length} (artifact ${output.artifactId})`; +} + +/** + * Maps ffmpeg progress onto the card: monotonic, at most once a second, written in order and + * never failing the job when a write fails. + */ +function createProgressReporter(context: MediaJobContext) { + let reported = 0; + let reportedAt = 0; + let writes = Promise.resolve(); + const write = (fraction: number) => { + reported = fraction; + reportedAt = Date.now(); + writes = writes.then(() => context.progress(fraction)).catch(() => undefined); + }; + return { + range: (from: number, to: number) => (fraction: number) => { + const value = Math.min(1, from + (to - from) * Math.min(1, Math.max(0, fraction))); + const boundary = fraction >= 1; + if (value <= reported) return; + if (!boundary && Date.now() - reportedAt < PROGRESS_INTERVAL_MS) return; + write(value); + }, + done: async () => { + if (reported < 1) write(1); + await writes; + }, + }; +} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index ba2c70b..40d8db3 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -10,6 +10,7 @@ export * from "./integration-settings.js"; export * from "./local-settings.js"; export * from "./mcp.js"; export * from "./media.js"; +export * from "./media-video.js"; export * from "./model-tiers.js"; export * from "./openai-compatible-ui.js"; export * from "./project-goals.js"; diff --git a/packages/contracts/src/media-video.ts b/packages/contracts/src/media-video.ts new file mode 100644 index 0000000..f0aaf5b --- /dev/null +++ b/packages/contracts/src/media-video.ts @@ -0,0 +1,80 @@ +import * as z from "zod"; +import { Id } from "./ids.js"; +import { MediaFitSchema, MediaPresetIdSchema } from "./media.js"; + +/** + * Inputs of the `edit_video` and `export_video` media operations, which are also the arguments + * of the tools with those names. Times are seconds on the timeline left by the previous steps. + */ + +const Seconds = z.number().min(0).max(86_400); + +const TimeRangeSchema = z + .object({ start: Seconds, end: Seconds }) + .refine((range) => range.end > range.start, "end must be after start"); + +/** Written by the bot in the user's language; shown on the media card and used as file name. */ +const MediaTitleSchema = z.string().trim().min(1).max(120); + +export const EditVideoStepSchema = z.discriminatedUnion("op", [ + z + .object({ op: z.literal("trim"), start: Seconds, end: Seconds }) + .refine((step) => step.end > step.start, "end must be after start"), + /** Keeps these parts and cuts everything between them. */ + z.object({ op: z.literal("keep"), segments: z.array(TimeRangeSchema).min(1).max(50) }), + /** Appends these videos, fitted to the current frame. */ + z.object({ op: z.literal("concat"), artifact_ids: z.array(Id).min(1).max(20) }), + z.object({ + op: z.literal("reframe"), + preset: MediaPresetIdSchema, + fit: MediaFitSchema.default("crop"), + }), + z.object({ op: z.literal("speed"), factor: z.number().min(0.25).max(4) }), + /** Linear gain: 0.5 halves the loudness, 2 doubles it. */ + z.object({ op: z.literal("volume"), gain: z.number().min(0).max(8) }), + z.object({ op: z.literal("mute") }), + /** Burns in an SRT or VTT artifact. */ + z.object({ op: z.literal("subtitles"), artifact_id: Id }), + /** Last step only: the result is the audio as M4A. */ + z.object({ op: z.literal("extract_audio") }), + /** Cover frame at this second of the result; as the only kind of step, the result is that image. */ + z.object({ op: z.literal("poster"), at: Seconds }), +]); +export type EditVideoStep = z.infer; + +export const EditVideoInputSchema = z + .object({ + artifact_id: Id, + steps: z.array(EditVideoStepSchema).min(1).max(20), + title: MediaTitleSchema, + }) + .superRefine((input, context) => { + const ops = input.steps.map((step) => step.op); + const extract = ops.indexOf("extract_audio"); + if (extract >= 0 && extract !== ops.length - 1) { + context.addIssue({ + code: "custom", + path: ["steps", extract], + message: "extract_audio must be the last step", + }); + } + if (extract >= 0 && ops.includes("poster")) { + context.addIssue({ + code: "custom", + path: ["steps"], + message: "poster cannot be combined with extract_audio", + }); + } + if (ops.filter((op) => op === "poster").length > 1) { + context.addIssue({ code: "custom", path: ["steps"], message: "use at most one poster step" }); + } + }); +export type EditVideoInput = z.infer; + +export const ExportVideoInputSchema = z.object({ + artifact_id: Id, + presets: z.array(MediaPresetIdSchema).min(1).max(6), + fit: MediaFitSchema.default("crop"), + title: MediaTitleSchema.optional(), +}); +export type ExportVideoInput = z.infer; From 44702e3770952527707c0603e9032011296a9d7b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:45:50 +0000 Subject: [PATCH 2/5] feat(media): register video operations with the media runtime edit_video and export_video now run as media jobs: both registrations join the shared operation list the API and the worker build their runtime from, name every artifact they read (source, appended clips, subtitle files) so starting a job refuses artifacts the user cannot read, and their tool schemas use the runtime's continue_after helper. Container deployments get ffmpeg, ffprobe and the subtitle font in the API/worker image. Tests cover the handlers against a fake job context: cuts, blurred 9:16 reframes with subtitles, appended silent clips, audio extraction, single frames, batch exports with posters and progress, rotated phone video, clear input errors and cancellation. They skip where ffmpeg is not installed; the planner and parser tests always run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- infra/compose/Dockerfile | 4 + packages/adapters/src/builtin-tools.ts | 23 +- packages/adapters/src/media-operations.ts | 3 +- packages/adapters/src/media/ffmpeg.ts | 18 +- .../src/media/video-operations.test.ts | 289 ++++++++++++++++++ .../adapters/src/media/video-operations.ts | 127 ++++---- packages/contracts/src/media-video.test.ts | 56 ++++ 7 files changed, 423 insertions(+), 97 deletions(-) create mode 100644 packages/adapters/src/media/video-operations.test.ts create mode 100644 packages/contracts/src/media-video.test.ts diff --git a/infra/compose/Dockerfile b/infra/compose/Dockerfile index fa6b508..e40d6f5 100644 --- a/infra/compose/Dockerfile +++ b/infra/compose/Dockerfile @@ -1,4 +1,8 @@ FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a +# ffmpeg/ffprobe cut, reframe and export videos; DejaVu is the burned-in subtitle font. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* RUN corepack enable WORKDIR /app # Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index 3229a7d..da32e33 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -10,24 +10,15 @@ import { USER_FACT_CATEGORIES, } from "@rakazo/contracts"; import { z } from "zod"; +import { mediaToolInputSchema } from "./media-tools.js"; export const SUBAGENT_MODEL_TIER_DESCRIPTION = "Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set."; -/** A media tool's arguments: its operation's input plus whether to wake the bot when it ends. */ -function mediaToolSchema(input: z.ZodType): Record { - const schema = z.toJSONSchema(input, { io: "input" }); - return { - ...schema, - properties: { - ...schema.properties, - continue_after: { - type: "boolean", - description: - "true to be woken with the result when the job ends, to continue a multi-step task. Default false: the card alone shows the result.", - }, - }, - }; +/** A media tool's JSON schema, generated from its operation's input schema. */ +function mediaToolSchemaOf(input: z.ZodType): ConnectorTool["inputSchema"] { + const { properties = {}, required } = z.toJSONSchema(input, { io: "input" }); + return mediaToolInputSchema({ properties, required }); } const VIDEO_PRESETS_AND_FITS = @@ -350,12 +341,12 @@ export const builtinAgentTools: ConnectorTool[] = [ { name: "edit_video", description: `Edit a video artifact in a background job; a media card shows progress and the result, a new version of the source. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4 with a poster. title: a few words in the user's language for the card and file name.`, - inputSchema: mediaToolSchema(EditVideoInputSchema), + inputSchema: mediaToolSchemaOf(EditVideoInputSchema), }, { name: "export_video", description: `Export a video artifact to one or more presets in a background job: one H.264/AAC MP4 with a poster per preset, each a new version of the source, shown on a media card. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`, - inputSchema: mediaToolSchema(ExportVideoInputSchema), + inputSchema: mediaToolSchemaOf(ExportVideoInputSchema), }, { name: "add_mcp_server", diff --git a/packages/adapters/src/media-operations.ts b/packages/adapters/src/media-operations.ts index 98dbf7e..b5e0710 100644 --- a/packages/adapters/src/media-operations.ts +++ b/packages/adapters/src/media-operations.ts @@ -1,4 +1,5 @@ import type { MediaSettings } from "@rakazo/contracts"; +import { createVideoOperations } from "./media/video-operations.js"; import type { MediaOperationRegistration } from "./media-runtime.js"; export interface MediaOperationDeps { @@ -11,5 +12,5 @@ export interface MediaOperationDeps { * from this one list, so both processes run the same operations. */ export function createMediaOperations(_deps: MediaOperationDeps): MediaOperationRegistration[] { - return []; + return [...createVideoOperations()]; } diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts index b34e722..e2ebc77 100644 --- a/packages/adapters/src/media/ffmpeg.ts +++ b/packages/adapters/src/media/ffmpeg.ts @@ -52,7 +52,6 @@ export interface RunFfmpegOptions { outputs?: readonly string[]; /** Kills ffmpeg when it reports no progress for this long. */ stallTimeoutMs?: number; - ffmpegPath?: string; } export class FfmpegError extends Error { @@ -162,9 +161,9 @@ export function ffmpegInputArgs( export async function probeMedia( path: string, - options: { signal?: AbortSignal; mimeType?: string; ffprobePath?: string } = {}, + options: { signal?: AbortSignal; mimeType?: string } = {}, ): Promise { - const ffprobePath = options.ffprobePath ?? (await requireFfmpeg()).ffprobePath; + const { ffprobePath } = await requireFfmpeg(); const input = ffmpegInputArgs(path, { mimeType: options.mimeType }); const result = await capture( ffprobePath, @@ -325,7 +324,7 @@ export async function runFfmpeg( options: RunFfmpegOptions = {}, ): Promise { options.signal?.throwIfAborted(); - const ffmpegPath = options.ffmpegPath ?? (await requireFfmpeg()).ffmpegPath; + const { ffmpegPath } = await requireFfmpeg(); try { await spawnFfmpeg(ffmpegPath, args, options); } catch (error) { @@ -469,12 +468,7 @@ export async function extractAudio( onProgress?: (fraction: number) => void; } = {}, ): Promise { - const tools = await requireFfmpeg(); - const probe = await probeMedia(input, { - signal: options.signal, - mimeType: options.mimeType, - ffprobePath: tools.ffprobePath, - }); + const probe = await probeMedia(input, { signal: options.signal, mimeType: options.mimeType }); if (!probe.hasAudio) throw new Error("The file has no audio track."); const startMs = Math.max(0, options.startMs ?? 0); const endMs = Math.min(probe.durationMs, options.endMs ?? probe.durationMs); @@ -505,7 +499,6 @@ export async function extractAudio( durationMs: endMs - startMs, onProgress: options.onProgress, outputs: [output], - ffmpegPath: tools.ffmpegPath, }, ); } @@ -516,7 +509,6 @@ export async function renderPoster( output: string, options: { signal?: AbortSignal; mimeType?: string; atMs?: number } = {}, ): Promise { - const ffmpegPath = (await requireFfmpeg()).ffmpegPath; const render = (atMs: number) => runFfmpeg( [ @@ -537,7 +529,7 @@ export async function renderPoster( "1", `file:${output}`, ], - { signal: options.signal, outputs: [output], ffmpegPath }, + { signal: options.signal, outputs: [output] }, ); const atMs = Math.max(0, options.atMs ?? 0); await render(atMs); diff --git a/packages/adapters/src/media/video-operations.test.ts b/packages/adapters/src/media/video-operations.test.ts new file mode 100644 index 0000000..1d8d6f9 --- /dev/null +++ b/packages/adapters/src/media/video-operations.test.ts @@ -0,0 +1,289 @@ +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { builtinAgentTools } from "../builtin-tools.js"; +import { createMediaOperations } from "../media-operations.js"; +import { MEDIA_CONTINUE_AFTER_PROPERTY } from "../media-tools.js"; +import { parametersFor } from "../pi-runtime.js"; +import { detectFfmpeg, probeMedia, runFfmpeg } from "./ffmpeg.js"; +import type { TestArtifact, TestClips } from "./media-test-support.js"; +import { createTestClips, fakeMediaJobContext } from "./media-test-support.js"; +import { cleanFileName, createVideoOperations, subtitleFormat } from "./video-operations.js"; + +const ffmpeg = await detectFfmpeg(); + +function operation(name: "edit_video" | "export_video", deps = {}) { + const registration = createVideoOperations(deps).find((entry) => entry.operation === name); + if (!registration) throw new Error(`missing ${name}`); + return registration; +} + +describe("video operation registrations", () => { + it("are registered for the runtime and unavailable without ffmpeg", async () => { + const names = createMediaOperations({ loadSettings: async () => ({}) as never }).map( + (entry) => entry.operation, + ); + expect(names).toEqual(expect.arrayContaining(["edit_video", "export_video"])); + const edit = operation("edit_video", { detect: async () => null }); + expect(await edit.available({ spaceId: "space-1" })).toBe(false); + const { context } = await fakeMediaJobContext({}, {}); + await expect(edit.handler.run(context)).rejects.toThrow( + "ffmpeg and ffprobe are not installed on this server.", + ); + }); + + it("names every artifact the input reads so start() can authorize them", () => { + const edit = operation("edit_video"); + const input = edit.inputSchema.parse({ + artifact_id: "src", + title: "Reel", + steps: [ + { op: "concat", artifact_ids: ["clip-1", "clip-2"] }, + { op: "subtitles", artifact_id: "subs" }, + ], + }); + expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs"]); + const exported = operation("export_video"); + const exportInput = exported.inputSchema.parse({ artifact_id: "src", presets: ["vertical"] }); + expect(exported.artifactIds?.(exportInput)).toEqual(["src"]); + }); + + it("titles exports by aspect ratio unless the bot gave a title", () => { + const exported = operation("export_video"); + expect( + exported.title( + exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical", "square"] }), + ), + ).toBe("9:16 · 1:1"); + expect( + exported.title( + exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical"], title: "Reel" }), + ), + ).toBe("Reel"); + }); + + it("defines tools named like the operations, with continue_after, that the runtime can encode", () => { + for (const name of ["edit_video", "export_video"]) { + const tool = builtinAgentTools.find((entry) => entry.name === name); + expect(tool).toBeDefined(); + expect(tool?.inputSchema.properties).toMatchObject(MEDIA_CONTINUE_AFTER_PROPERTY); + expect(() => parametersFor(tool!)).not.toThrow(); + } + }); +}); + +describe("subtitle and file name helpers", () => { + it("recognizes SRT and VTT by content only", () => { + expect(subtitleFormat("WEBVTT\n\n00:00.000 --> 00:01.000\nHi")).toBe("vtt"); + expect(subtitleFormat("1\r\n00:00:01,000 --> 00:00:02,000\r\nHi")).toBe("srt"); + expect(subtitleFormat("00:00:01.000 --> 00:00:02.000\nHi")).toBe("srt"); + expect(subtitleFormat("#EXTM3U\n#EXTINF:1,\nfile:/etc/passwd")).toBeNull(); + expect(subtitleFormat("WEBVTTX")).toBeNull(); + }); + + it("cleans file names without losing the user's language", () => { + expect(cleanFileName("Reel: Sommer/Urlaub 2026", "video")).toBe("Reel Sommer Urlaub 2026"); + expect(cleanFileName("../..\u0000", "video")).toBe("video"); + expect(cleanFileName("x".repeat(200), "video")).toHaveLength(80); + }); +}); + +// CI runners may not have ffmpeg; these run wherever it is installed. +describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => { + let clips: TestClips; + let artifacts: Record; + + beforeAll(async () => { + clips = await createTestClips(); + artifacts = { + land: { path: clips.landscape, name: "Landscape clip.mp4", mimeType: "video/mp4" }, + silent: { path: clips.silentPortrait, name: "silent.mp4", mimeType: "video/mp4" }, + srt: { path: clips.srt, name: "captions.srt", mimeType: "application/x-subrip" }, + vtt: { path: clips.vtt, name: "captions.vtt", mimeType: "text/vtt" }, + notes: { path: clips.landscape, name: "notes.txt", mimeType: "text/plain" }, + }; + }); + + afterAll(async () => { + if (clips) await rm(clips.dir, { recursive: true, force: true }); + }); + + async function run(name: "edit_video" | "export_video", input: Record) { + const registration = operation(name); + const job = await fakeMediaJobContext(registration.inputSchema.parse(input), artifacts); + try { + const result = await registration.handler.run(job.context); + return { ...job, result }; + } finally { + await rm(job.tempDir, { recursive: true, force: true }); + } + } + + it("cuts, reframes to 9:16 with a blurred fill, burns in subtitles and speeds up", async () => { + const { outputs, progress, result } = await run("edit_video", { + artifact_id: "land", + title: "Sommer-Reel", + steps: [ + { op: "trim", start: 0.5, end: 2.5 }, + { op: "reframe", preset: "vertical", fit: "blur" }, + { op: "subtitles", artifact_id: "srt" }, + { op: "speed", factor: 2 }, + { op: "poster", at: 0.5 }, + ], + }); + expect(outputs).toHaveLength(1); + const [output] = outputs; + expect(output).toMatchObject({ + name: "Sommer-Reel.mp4", + mimeType: "video/mp4", + width: 1080, + height: 1920, + parentArtifactId: "land", + }); + expect(output!.posterSize).toBeGreaterThan(0); + expect(Math.abs(output!.durationMs! - 1000)).toBeLessThan(150); + expect(progress.at(-1)).toBe(1); + expect(progress).toEqual([...progress].sort((a, b) => a - b)); + expect(result.summary).toContain("Sommer-Reel.mp4 (1080×1920, "); + }); + + it("keeps segments, appends a silent clip with generated silence, and changes volume", async () => { + const registration = operation("edit_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ + artifact_id: "land", + title: "Cut", + steps: [ + { + op: "keep", + segments: [ + { start: 0, end: 0.5 }, + { start: 2, end: 2.8 }, + ], + }, + { op: "concat", artifact_ids: ["silent"] }, + { op: "volume", gain: 0.5 }, + { op: "subtitles", artifact_id: "vtt" }, + ], + }), + artifacts, + ); + await registration.handler.run(job.context); + const [output] = job.outputs; + const probe = await probeMedia(output!.path, { mimeType: "video/mp4" }); + expect(probe.hasAudio).toBe(true); + expect(probe.video).toMatchObject({ width: 320, height: 180, rotation: 0 }); + expect(Math.abs(probe.durationMs - 3300)).toBeLessThan(200); + await rm(job.tempDir, { recursive: true, force: true }); + }); + + it("plays rotated phone video upright without rotating it twice", async (context) => { + const rotated = join(clips.dir, "rotated.mp4"); + try { + await runFfmpeg([ + ...["-display_rotation", "90", "-i", `file:${clips.landscape}`], + ...["-c", "copy", `file:${rotated}`], + ]); + } catch { + context.skip("this ffmpeg cannot write rotation metadata"); + } + const registration = operation("edit_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ + artifact_id: "rotated", + title: "Upright", + steps: [{ op: "trim", start: 0, end: 1 }], + }), + { rotated: { path: rotated, name: "rotated.mp4", mimeType: "video/mp4" } }, + ); + await registration.handler.run(job.context); + const probe = await probeMedia(job.outputs[0]!.path, { mimeType: "video/mp4" }); + expect(probe.video).toMatchObject({ width: 180, height: 320, rotation: 0 }); + await rm(job.tempDir, { recursive: true, force: true }); + }); + + it("extracts the audio as M4A", async () => { + const { outputs } = await run("edit_video", { + artifact_id: "land", + title: "Ton", + steps: [{ op: "trim", start: 1, end: 2 }, { op: "extract_audio" }], + }); + expect(outputs).toEqual([ + expect.objectContaining({ name: "Ton.m4a", mimeType: "audio/mp4", parentArtifactId: "land" }), + ]); + expect(outputs[0]?.posterPath).toBeUndefined(); + expect(Math.abs(outputs[0]!.durationMs! - 1000)).toBeLessThan(100); + }); + + it("saves a single frame when poster is the only step", async () => { + const { outputs } = await run("edit_video", { + artifact_id: "land", + title: "Cover", + steps: [{ op: "poster", at: 1 }], + }); + expect(outputs).toEqual([ + expect.objectContaining({ + name: "Cover.jpg", + mimeType: "image/jpeg", + width: 320, + height: 180, + }), + ]); + }); + + it("exports one output per distinct preset with posters and batch progress", async () => { + const { outputs, progress } = await run("export_video", { + artifact_id: "land", + presets: ["square", "portrait", "square"], + fit: "pad", + }); + expect( + outputs.map((output) => [output.name, output.preset, output.width, output.height]), + ).toEqual([ + ["Landscape clip-1x1.mp4", "square", 1080, 1080], + ["Landscape clip-4x5.mp4", "portrait", 1080, 1350], + ]); + for (const output of outputs) { + expect(output.parentArtifactId).toBe("land"); + expect(output.posterSize).toBeGreaterThan(0); + } + expect(progress).toContain(0.5); + expect(progress.at(-1)).toBe(1); + }); + + it("fails clearly on ranges past the end, non-subtitle files and non-videos", async () => { + await expect( + run("edit_video", { + artifact_id: "land", + title: "x", + steps: [{ op: "trim", start: 5, end: 6 }], + }), + ).rejects.toThrow("trim 5–6 s is outside the 3 s video."); + await expect( + run("edit_video", { + artifact_id: "land", + title: "x", + steps: [{ op: "subtitles", artifact_id: "land" }], + }), + ).rejects.toThrow("Landscape clip.mp4 is not an SRT or VTT file."); + await expect( + run("export_video", { artifact_id: "notes", presets: ["square"] }), + ).rejects.toThrow("notes.txt is not a video."); + }); + + it("stops ffmpeg and stores nothing when the job is cancelled", async () => { + const registration = operation("export_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ artifact_id: "land", presets: ["cinema"], fit: "blur" }), + artifacts, + ); + const progress = job.context.progress; + job.context.progress = async (fraction) => { + await progress(fraction); + job.controller.abort(new Error("cancelled")); + }; + await expect(registration.handler.run(job.context)).rejects.toThrow("cancelled"); + expect(job.outputs).toEqual([]); + await rm(job.tempDir, { recursive: true, force: true }); + }); +}); diff --git a/packages/adapters/src/media/video-operations.ts b/packages/adapters/src/media/video-operations.ts index b21d9df..e4e1e6d 100644 --- a/packages/adapters/src/media/video-operations.ts +++ b/packages/adapters/src/media/video-operations.ts @@ -1,11 +1,6 @@ import { copyFile, mkdtemp, open, stat } from "node:fs/promises"; import { join } from "node:path"; -import type { - MediaJobContext, - MediaJobInputFile, - MediaOperationHandler, - MediaOperationResult, -} from "@rakazo/adapter-kit"; +import type { MediaJobContext, MediaJobInputFile, MediaOperationResult } from "@rakazo/adapter-kit"; import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts"; import { ARTIFACT_MAX_BYTES_BY_KIND, @@ -13,6 +8,7 @@ import { ExportVideoInputSchema, MEDIA_PRESETS, } from "@rakazo/contracts"; +import type { MediaOperationRegistration } from "../media-runtime.js"; import type { FfmpegInfo } from "./ffmpeg.js"; import { detectFfmpeg, @@ -29,59 +25,71 @@ export interface VideoOperationsDeps { detect?: () => Promise; } -const PROGRESS_INTERVAL_MS = 1_000; /** Share of each output's progress spent encoding; the rest is probing and the poster. */ const ENCODE_SHARE = 0.95; const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024; /** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */ -export function createVideoOperations(deps: VideoOperationsDeps = {}) { +export function createVideoOperations( + deps: VideoOperationsDeps = {}, +): MediaOperationRegistration[] { const detect = deps.detect ?? detectFfmpeg; const available = async () => (await detect()) !== null; - const tools = async () => { - const info = await detect(); - if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server."); - return info; + const requireTools = async () => { + if (!(await available())) + throw new Error("ffmpeg and ffprobe are not installed on this server."); }; - const editVideo: MediaOperationHandler = { + const editVideo: MediaOperationRegistration = { operation: "edit_video", - run: async (context) => runEditVideo(context, await tools()), + inputSchema: EditVideoInputSchema, + title: (input) => input.title, + available, + artifactIds: (input) => [ + input.artifact_id, + ...input.steps.flatMap((step) => + step.op === "concat" + ? step.artifact_ids + : step.op === "subtitles" + ? [step.artifact_id] + : [], + ), + ], + handler: { + operation: "edit_video", + run: async (context) => { + await requireTools(); + return runEditVideo(context); + }, + }, }; - const exportVideo: MediaOperationHandler = { + const exportVideo: MediaOperationRegistration = { operation: "export_video", - run: async (context) => runExportVideo(context, await tools()), + inputSchema: ExportVideoInputSchema, + title: (input) => + input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "), + available, + artifactIds: (input) => [input.artifact_id], + handler: { + operation: "export_video", + run: async (context) => { + await requireTools(); + return runExportVideo(context); + }, + }, }; - return [ - { - operation: "edit_video" as const, - inputSchema: EditVideoInputSchema, - title: (input: EditVideoInput) => input.title, - available, - handler: editVideo, - }, - { - operation: "export_video" as const, - inputSchema: ExportVideoInputSchema, - title: (input: ExportVideoInput) => - input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "), - available, - handler: exportVideo, - }, - ]; + return [editVideo, exportVideo]; } -async function runEditVideo( - context: MediaJobContext, - tools: FfmpegInfo, -): Promise { +async function runEditVideo(context: MediaJobContext): Promise { const input = EditVideoInputSchema.parse(context.input); const workDir = await mkdtemp(join(context.tempDir, "edit-")); const progress = createProgressReporter(context); - const source = await loadVideo(context, tools, input.artifact_id); + const source = await loadVideo(context, input.artifact_id); const name = cleanFileName(input.title, cleanFileName(withoutExtension(source.name), "video")); - if (input.steps.every((step) => step.op === "poster")) { - const at = input.steps[0]?.op === "poster" ? input.steps[0].at : 0; + const [first] = input.steps; + if (input.steps.length === 1 && first?.op === "poster") { + const at = first.at; if (at * 1000 >= source.probe.durationMs) { throw new Error( `poster at ${at} s is past the end of the ${source.probe.durationMs / 1000} s video.`, @@ -110,7 +118,7 @@ async function runEditVideo( for (const step of input.steps) { if (step.op === "concat") { for (const id of step.artifact_ids) { - if (!clips.has(id)) clips.set(id, await loadVideo(context, tools, id)); + if (!clips.has(id)) clips.set(id, await loadVideo(context, id)); } } else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) { subtitleFiles.set( @@ -122,7 +130,7 @@ async function runEditVideo( const audioOnly = input.steps.at(-1)?.op === "extract_audio"; const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4"); const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath }); - const output = await encode(context, tools, plan, outputPath, workDir, progress.range(0, 1), { + const output = await encode(context, plan, outputPath, workDir, progress.range(0, 1), { name: `${name}${audioOnly ? ".m4a" : ".mp4"}`, parentArtifactId: source.artifactId, }); @@ -132,15 +140,12 @@ async function runEditVideo( }; } -async function runExportVideo( - context: MediaJobContext, - tools: FfmpegInfo, -): Promise { +async function runExportVideo(context: MediaJobContext): Promise { const input = ExportVideoInputSchema.parse(context.input); const presets = [...new Set(input.presets)]; const workDir = await mkdtemp(join(context.tempDir, "export-")); const progress = createProgressReporter(context); - const source = await loadVideo(context, tools, input.artifact_id); + const source = await loadVideo(context, input.artifact_id); const base = cleanFileName(withoutExtension(source.name), "video"); const outputs: MediaOutput[] = []; for (const [index, preset] of presets.entries()) { @@ -154,7 +159,6 @@ async function runExportVideo( outputs.push( await encode( context, - tools, plan, outputPath, workDir, @@ -174,7 +178,6 @@ async function runExportVideo( /** Runs a plan, checks the result, renders its poster and stores it as an output. */ async function encode( context: MediaJobContext, - tools: FfmpegInfo, plan: VideoEditPlan, outputPath: string, workDir: string, @@ -189,13 +192,11 @@ async function encode( onProgress: (fraction) => report(fraction * ENCODE_SHARE), maxOutputBytes: maxBytes, outputs: [outputPath], - ffmpegPath: tools.ffmpegPath, }); if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes)); const probe = await probeMedia(outputPath, { signal: context.signal, mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4", - ffprobePath: tools.ffprobePath, }); if (plan.kind === "audio") { const output = await context.addOutput({ @@ -228,7 +229,6 @@ async function encode( async function loadVideo( context: MediaJobContext, - tools: FfmpegInfo, artifactId: string, ): Promise { const file = await context.fetchArtifact(artifactId); @@ -236,7 +236,6 @@ async function loadVideo( const probe = await probeMedia(file.path, { signal: context.signal, mimeType: file.mimeType, - ffprobePath: tools.ffprobePath, }); if (!probe.video) throw new Error(`${file.name} has no video.`); if (probe.durationMs <= 0) throw new Error(`Could not read the length of ${file.name}.`); @@ -295,33 +294,27 @@ function withoutExtension(name: string): string { return name.replace(/\.[A-Za-z0-9]{1,5}$/, ""); } +/** "name (1080×1920, 12.5 s)" for the bot's summary; the runtime adds the artifact ids. */ function describe(output: MediaOutput): string { - const size = output.width && output.height ? ` ${output.width}×${output.height}` : ""; - const length = - output.durationMs !== undefined ? ` ${Number((output.durationMs / 1000).toFixed(1))} s` : ""; - return `${output.name}${size}${length} (artifact ${output.artifactId})`; + const details = [ + output.width && output.height ? `${output.width}×${output.height}` : "", + output.durationMs !== undefined ? `${Number((output.durationMs / 1000).toFixed(1))} s` : "", + ].filter(Boolean); + return details.length ? `${output.name} (${details.join(", ")})` : output.name; } -/** - * Maps ffmpeg progress onto the card: monotonic, at most once a second, written in order and - * never failing the job when a write fails. - */ +/** Maps ffmpeg progress onto the card: monotonic, written in order, never failing the job. */ function createProgressReporter(context: MediaJobContext) { let reported = 0; - let reportedAt = 0; let writes = Promise.resolve(); const write = (fraction: number) => { reported = fraction; - reportedAt = Date.now(); writes = writes.then(() => context.progress(fraction)).catch(() => undefined); }; return { range: (from: number, to: number) => (fraction: number) => { const value = Math.min(1, from + (to - from) * Math.min(1, Math.max(0, fraction))); - const boundary = fraction >= 1; - if (value <= reported) return; - if (!boundary && Date.now() - reportedAt < PROGRESS_INTERVAL_MS) return; - write(value); + if (value > reported) write(value); }, done: async () => { if (reported < 1) write(1); diff --git a/packages/contracts/src/media-video.test.ts b/packages/contracts/src/media-video.test.ts new file mode 100644 index 0000000..eb3fc17 --- /dev/null +++ b/packages/contracts/src/media-video.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { EditVideoInputSchema, ExportVideoInputSchema } from "./media-video.js"; + +const edit = (steps: unknown[]) => + EditVideoInputSchema.safeParse({ artifact_id: "a", title: "Reel", steps }); + +describe("edit_video input", () => { + it("accepts ordered steps and fills defaults", () => { + const parsed = EditVideoInputSchema.parse({ + artifact_id: "a", + title: " Reel ", + steps: [ + { op: "trim", start: 1, end: 5 }, + { op: "reframe", preset: "vertical" }, + { op: "extract_audio" }, + ], + }); + expect(parsed.title).toBe("Reel"); + expect(parsed.steps[1]).toEqual({ op: "reframe", preset: "vertical", fit: "crop" }); + // The job stores the parsed input and parses it again when it runs. + expect(EditVideoInputSchema.parse(parsed)).toEqual(parsed); + }); + + it("rejects malformed step lists", () => { + expect(edit([]).success).toBe(false); + expect(edit([{ op: "trim", start: 5, end: 5 }]).success).toBe(false); + expect(edit([{ op: "keep", segments: [{ start: 3, end: 1 }] }]).success).toBe(false); + expect(edit([{ op: "speed", factor: 8 }]).success).toBe(false); + expect(edit([{ op: "extract_audio" }, { op: "mute" }]).success).toBe(false); + expect(edit([{ op: "poster", at: 1 }, { op: "extract_audio" }]).success).toBe(false); + expect( + edit([ + { op: "poster", at: 1 }, + { op: "poster", at: 2 }, + ]).success, + ).toBe(false); + expect(edit([{ op: "blur" }]).success).toBe(false); + expect( + EditVideoInputSchema.safeParse({ artifact_id: "a", steps: [{ op: "mute" }] }).success, + ).toBe(false); + }); +}); + +describe("export_video input", () => { + it("needs at least one preset and defaults to cropping", () => { + expect(ExportVideoInputSchema.parse({ artifact_id: "a", presets: ["square"] })).toEqual({ + artifact_id: "a", + presets: ["square"], + fit: "crop", + }); + expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: [] }).success).toBe(false); + expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: ["imax"] }).success).toBe( + false, + ); + }); +}); From f14d91d24a3316b92b03db44298c91a7064b525a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 13:51:44 +0000 Subject: [PATCH 3/5] refactor(media): trim unused ffmpeg wrapper options Binary paths, the ffmpeg version, the probe's container name and a configurable stall timeout had no caller; detection is cached per process, so every helper looks the binaries up itself. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- packages/adapters/src/media/ffmpeg.test.ts | 1 - packages/adapters/src/media/ffmpeg.ts | 19 ++++++------------- .../src/media/video-edit-plan.test.ts | 1 - 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/adapters/src/media/ffmpeg.test.ts b/packages/adapters/src/media/ffmpeg.test.ts index ab846da..e431827 100644 --- a/packages/adapters/src/media/ffmpeg.test.ts +++ b/packages/adapters/src/media/ffmpeg.test.ts @@ -60,7 +60,6 @@ describe("probe parsing", () => { ); expect(probe).toEqual({ durationMs: 12_345, - formatName: "mov,mp4,m4a,3gp,3g2,mj2", video: { codec: "h264", width: 1080, diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts index e2ebc77..72eefc1 100644 --- a/packages/adapters/src/media/ffmpeg.ts +++ b/packages/adapters/src/media/ffmpeg.ts @@ -13,12 +13,10 @@ import { delimiter, join } from "node:path"; export interface FfmpegInfo { ffmpegPath: string; ffprobePath: string; - version: string; } export interface MediaProbe { durationMs: number; - formatName: string; /** Width and height as displayed: swapped when the stream is rotated by 90 or 270 degrees. */ video?: { codec: string; @@ -50,8 +48,6 @@ export interface RunFfmpegOptions { maxOutputBytes?: number; /** Files removed when the run fails or is aborted. */ outputs?: readonly string[]; - /** Kills ffmpeg when it reports no progress for this long. */ - stallTimeoutMs?: number; } export class FfmpegError extends Error { @@ -65,7 +61,8 @@ export class FfmpegError extends Error { } } -const DEFAULT_STALL_TIMEOUT_MS = 2 * 60_000; +/** ffmpeg reports progress twice a second while it works; silence this long means a hang. */ +const STALL_TIMEOUT_MS = 2 * 60_000; const PROBE_TIMEOUT_MS = 60_000; const STDERR_TAIL_BYTES = 8 * 1024; const KILL_GRACE_MS = 5_000; @@ -101,9 +98,7 @@ async function locateFfmpeg(): Promise { capture(ffmpegPath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }), capture(ffprobePath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }), ]); - if (ffmpeg.code !== 0 || ffprobe.code !== 0) return null; - const version = /version (\S+)/.exec(ffmpeg.stdout)?.[1] ?? "unknown"; - return { ffmpegPath, ffprobePath, version }; + return ffmpeg.code === 0 && ffprobe.code === 0 ? { ffmpegPath, ffprobePath } : null; } catch { return null; } @@ -129,7 +124,7 @@ async function findExecutable(name: string): Promise { return null; } -export async function requireFfmpeg(): Promise { +async function requireFfmpeg(): Promise { const info = await detectFfmpeg(); if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server."); return info; @@ -223,7 +218,7 @@ interface ProbeStream { /** Parses `ffprobe -print_format json -show_format -show_streams` output. */ export function parseProbeOutput(json: string): MediaProbe { const data = JSON.parse(json) as { - format?: { duration?: string; format_name?: string }; + format?: { duration?: string }; streams?: ProbeStream[]; }; const streams = data.streams ?? []; @@ -237,7 +232,6 @@ export function parseProbeOutput(json: string): MediaProbe { Math.max(0, ...streams.map((stream) => positive(Number(stream.duration)) ?? 0)); return { durationMs: Math.round(seconds * 1000), - formatName: data.format?.format_name ?? "", ...(video?.width && video.height ? { video: probeVideo(video, video.width, video.height) } : {}), @@ -385,12 +379,11 @@ function spawnFfmpeg( setTimeout(() => finish(error), KILL_GRACE_MS).unref?.(); }; const abort = () => kill(signal?.reason ?? new Error("aborted")); - const stallTimeoutMs = options.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS; const armStallTimer = () => { clearTimeout(stallTimer); stallTimer = setTimeout( () => kill(new FfmpegError("ffmpeg stopped making progress.", null, tail(stderr))), - stallTimeoutMs, + STALL_TIMEOUT_MS, ); stallTimer.unref?.(); }; diff --git a/packages/adapters/src/media/video-edit-plan.test.ts b/packages/adapters/src/media/video-edit-plan.test.ts index ea37cd6..4927f9e 100644 --- a/packages/adapters/src/media/video-edit-plan.test.ts +++ b/packages/adapters/src/media/video-edit-plan.test.ts @@ -13,7 +13,6 @@ function probe( const { durationMs = 10_000, hasAudio = true, ...video } = overrides; return { durationMs, - formatName: "mov,mp4,m4a,3gp,3g2,mj2", video: { codec: "h264", width: 1920, From 97cede6c9c3c306adb687d5ebc13e6c8036cfb50 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 14:01:58 +0000 Subject: [PATCH 4/5] test(media): run video edits and exports through the media runtime A Postgres suite starts edit_video and export_video through the real runtime and local artifact store: outputs become linked versions with posters on the card, exports store one version per preset, appended clips and subtitle files of another user are refused at start, and an out-of-range cut fails the job with a readable error. It skips without Postgres or ffmpeg. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- .../media/video-operations.postgres.test.ts | 214 ++++++++++++++++++ packages/testkit/src/cli/harness.ts | 1 + 2 files changed, 215 insertions(+) create mode 100644 packages/adapters/src/media/video-operations.postgres.test.ts diff --git a/packages/adapters/src/media/video-operations.postgres.test.ts b/packages/adapters/src/media/video-operations.postgres.test.ts new file mode 100644 index 0000000..7f5592d --- /dev/null +++ b/packages/adapters/src/media/video-operations.postgres.test.ts @@ -0,0 +1,214 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { JobPublisher } from "@rakazo/adapter-kit"; +import type { MediaJobBlock, MessageBlock } from "@rakazo/contracts"; +import type { PrismaClient } from "@rakazo/db"; +import { createDb } from "@rakazo/db"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { storeArtifactFromFile } from "../artifact-files.js"; +import { LocalArtifactStore } from "../artifacts.js"; +import { createMediaRuntime, MediaJobRejectedError } from "../media-runtime.js"; +import { detectFfmpeg } from "./ffmpeg.js"; +import type { TestClips } from "./media-test-support.js"; +import { createTestClips } from "./media-test-support.js"; +import { createVideoOperations } from "./video-operations.js"; + +// CI runners may not have ffmpeg; the suite runs wherever both it and Postgres are available. +const describePostgres = + process.env.VERIFY_DATABASE && process.env.DATABASE_URL && (await detectFfmpeg()) + ? describe.sequential + : describe.skip; + +describePostgres("video operations in the media runtime (PostgreSQL + ffmpeg)", () => { + let db: ReturnType; + let prisma: PrismaClient; + let root: string; + let clips: TestClips; + const users: string[] = []; + + beforeAll(async () => { + db = createDb(process.env.DATABASE_URL!); + prisma = db.prisma; + root = await mkdtemp(join(tmpdir(), "video-operations-pg-")); + clips = await createTestClips(); + }); + + afterAll(async () => { + if (!db) return; + await prisma.mediaJob.deleteMany({ where: { userId: { in: users } } }); + await prisma.artifact.deleteMany({ where: { userId: { in: users } } }); + await prisma.organization.deleteMany({ where: { id: { in: users } } }); + await prisma.user.deleteMany({ where: { id: { in: users } } }); + await prisma.$disconnect(); + await db.pool.end(); + await rm(root, { recursive: true, force: true }); + await rm(clips.dir, { recursive: true, force: true }); + }); + + async function setup() { + const id = randomUUID(); + users.push(id); + await prisma.user.create({ + data: { id, name: "Video test", email: `${id}@example.test`, emailVerified: true }, + }); + await prisma.organization.create({ + data: { id, name: "Video test", slug: id, createdAt: new Date() }, + }); + await prisma.space.create({ data: { id, organizationId: id, name: "Media", isDefault: true } }); + await prisma.member.create({ + data: { id, organizationId: id, userId: id, role: "owner", createdAt: new Date() }, + }); + const bot = await prisma.bot.create({ + data: { spaceId: id, userId: id, name: "Studio", color: "test-color" }, + }); + const thread = await prisma.thread.create({ data: { spaceId: id, userId: id, botId: bot.id } }); + const jobs: JobPublisher = { + enqueue: async () => undefined, + cancel: async () => undefined, + close: async () => undefined, + }; + const artifacts = new LocalArtifactStore(join(root, id, "artifacts")); + const runtime = createMediaRuntime( + { + prisma, + events: { notify: async () => undefined }, + jobs, + artifacts, + registrations: createVideoOperations(), + }, + { tempRoot: join(root, id, "jobs") }, + ); + const scope = { + spaceId: id, + userId: id, + botId: bot.id, + groupId: null, + threadId: thread.id, + runId: null, + }; + const store = async (path: string, name: string, mimeType: string) => + ( + await storeArtifactFromFile( + { prisma, artifacts }, + { + path, + name, + mimeType, + spaceId: id, + userId: id, + botId: bot.id, + groupId: null, + runId: null, + }, + ) + ).id; + return { runtime, scope, store }; + } + + async function card(messageId: string, jobId: string): Promise { + const message = await prisma.message.findUniqueOrThrow({ where: { id: messageId } }); + return (message.blocks as MessageBlock[]).find( + (block): block is MediaJobBlock => block.kind === "media" && block.jobId === jobId, + ); + } + + it("edits a stored video into a linked 9:16 version with a poster on the card", async () => { + const { runtime, scope, store } = await setup(); + const source = await store(clips.landscape, "holiday.mp4", "video/mp4"); + const subtitles = await store(clips.srt, "holiday.srt", "application/x-subrip"); + const started = await runtime.start({ + ...scope, + operation: "edit_video", + input: { + artifact_id: source, + title: "Urlaub 9:16", + steps: [ + { op: "trim", start: 0.5, end: 2.5 }, + { op: "reframe", preset: "vertical", fit: "blur" }, + { op: "subtitles", artifact_id: subtitles }, + ], + }, + }); + await runtime.runJob(started.jobId); + const job = await runtime.get(started.jobId); + expect(job).toMatchObject({ status: "done", title: "Urlaub 9:16", progress: 1 }); + const [output] = job!.outputs; + expect(output).toMatchObject({ + kind: "video", + mimeType: "video/mp4", + name: "Urlaub 9 16.mp4", + width: 1080, + height: 1920, + parentArtifactId: source, + }); + expect(output?.posterArtifactId).toBeDefined(); + const poster = await prisma.artifact.findUniqueOrThrow({ + where: { id: output!.posterArtifactId! }, + }); + expect(poster.mimeType).toBe("image/jpeg"); + const stored = await prisma.artifact.findUniqueOrThrow({ where: { id: output!.artifactId } }); + expect(stored).toMatchObject({ parentArtifactId: source, width: 1080, height: 1920 }); + expect(Math.abs((stored.durationMs ?? 0) - 2000)).toBeLessThan(150); + expect(await card(started.messageId, started.jobId)).toMatchObject({ + status: "done", + outputs: job?.outputs, + }); + }); + + it("exports one stored version per preset", async () => { + const { runtime, scope, store } = await setup(); + const source = await store(clips.landscape, "holiday.mp4", "video/mp4"); + const started = await runtime.start({ + ...scope, + operation: "export_video", + input: { artifact_id: source, presets: ["square", "portrait"] }, + }); + await runtime.runJob(started.jobId); + const job = await runtime.get(started.jobId); + expect(job).toMatchObject({ status: "done", title: "1:1 · 4:5" }); + expect( + job?.outputs.map((output) => [output.name, output.preset, output.width, output.height]), + ).toEqual([ + ["holiday-1x1.mp4", "square", 1080, 1080], + ["holiday-4x5.mp4", "portrait", 1080, 1350], + ]); + }); + + it("refuses appended clips and subtitles the user cannot read", async () => { + const { runtime, scope, store } = await setup(); + const other = await setup(); + const source = await store(clips.landscape, "mine.mp4", "video/mp4"); + const foreignClip = await other.store(clips.silentPortrait, "theirs.mp4", "video/mp4"); + const foreignSubtitles = await other.store(clips.srt, "theirs.srt", "application/x-subrip"); + for (const step of [ + { op: "concat", artifact_ids: [foreignClip] }, + { op: "subtitles", artifact_id: foreignSubtitles }, + ]) { + await expect( + runtime.start({ + ...scope, + operation: "edit_video", + input: { artifact_id: source, title: "x", steps: [step] }, + }), + ).rejects.toThrow(MediaJobRejectedError); + } + }); + + it("fails the job with a readable error when a range is outside the video", async () => { + const { runtime, scope, store } = await setup(); + const source = await store(clips.landscape, "short.mp4", "video/mp4"); + const started = await runtime.start({ + ...scope, + operation: "edit_video", + input: { artifact_id: source, title: "x", steps: [{ op: "trim", start: 10, end: 20 }] }, + }); + await runtime.runJob(started.jobId); + expect(await runtime.get(started.jobId)).toMatchObject({ + status: "failed", + error: "trim 10–20 s is outside the 3 s video.", + outputs: [], + }); + }); +}); diff --git a/packages/testkit/src/cli/harness.ts b/packages/testkit/src/cli/harness.ts index fc38326..ce36780 100644 --- a/packages/testkit/src/cli/harness.ts +++ b/packages/testkit/src/cli/harness.ts @@ -118,6 +118,7 @@ async function main() { "packages/adapters/src/job-reconciler.postgres.test.ts", "packages/adapters/src/cloud-agent.postgres.test.ts", "packages/adapters/src/media/image-operations.postgres.test.ts", + "packages/adapters/src/media/video-operations.postgres.test.ts", "packages/adapters/src/media-runtime.postgres.test.ts", "packages/adapters/src/builtin-memory-provider.postgres.test.ts", "packages/adapters/src/profile-consolidation.postgres.test.ts", From 31f4c2531d89600144ee9a0a460e6980c8e3139d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 14:11:27 +0000 Subject: [PATCH 5/5] fix(media): let bots title image edits in the user's language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Edit cards showed English words such as "crop · flip · extend" to every user. edit_image now takes an optional title written by the bot in the user's language, used for the card and the file name like edit_video. Without one, the card falls back to the repaint prompt or to language-neutral tokens only: sizes, ratios, degrees, arrows and format names. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --- .../src/media/image-operations.test.ts | 19 +++++++++++++++++- .../adapters/src/media/image-operations.ts | 20 ++++++++++++------- packages/contracts/src/media-generation.ts | 4 ++++ packages/contracts/src/media-video.ts | 2 +- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/packages/adapters/src/media/image-operations.test.ts b/packages/adapters/src/media/image-operations.test.ts index e028552..f414c08 100644 --- a/packages/adapters/src/media/image-operations.test.ts +++ b/packages/adapters/src/media/image-operations.test.ts @@ -223,6 +223,13 @@ describe("image operations", () => { expect((await sharp(outputs[0]!.path).metadata()).format).toBe("webp"); expect(result.summary).toBe("Edited photo.jpg into photo-edit.webp (1080×1920)."); expect(emulator.requests).toHaveLength(0); + const titled = await run( + operation, + "edit_image", + { artifactId: "photo-1", title: "Titelbild: Sommer", rotate: 180 }, + { "photo-1": photo }, + ); + expect(titled.outputs[0]?.name).toBe("Titelbild Sommer.jpg"); await expect( run(operation, "edit_image", { artifactId: "photo-1", prompt: "snow" }, { "photo-1": photo }), ).rejects.toThrow("ComfyUI is not configured"); @@ -267,7 +274,17 @@ describe("helpers", () => { resize: { aspect: "vertical", fit: "crop" }, format: "png", }), - ).toBe("crop · 90° · 9:16 · PNG"); + ).toBe("1×1 · 90° · 9:16 · PNG"); + expect( + editImageTitle({ + artifactId: "a", + flip: "vertical", + extend: { top: 4, right: 0, bottom: 0, left: 0 }, + }), + ).toBe("↕"); + expect( + editImageTitle({ artifactId: "a", title: "Titelbild", prompt: "snow", rotate: 90 }), + ).toBe("Titelbild"); expect(editImageTitle({ artifactId: "a", resize: { width: 800, fit: "crop" } })).toBe("800×…"); expect(editImageTitle({ artifactId: "a", prompt: "make it snow" })).toBe("make it snow"); }); diff --git a/packages/adapters/src/media/image-operations.ts b/packages/adapters/src/media/image-operations.ts index 68c08aa..f19ab09 100644 --- a/packages/adapters/src/media/image-operations.ts +++ b/packages/adapters/src/media/image-operations.ts @@ -36,6 +36,7 @@ import { imageSize, sharpAvailable, } from "./image-edit.js"; +import { cleanFileName } from "./video-operations.js"; /** Free GPU memory a generation needs before it starts; fixed, so a custom workflow cannot lower it. */ export const IMAGE_GENERATION_MIN_VRAM_MB = 6_000; @@ -75,19 +76,22 @@ export function fileStem(text: string, fallback: string): string { return words?.slice(0, 6).join("-").slice(0, 48).replace(/-+$/, "") || fallback; } -/** Short card title for an edit without a prompt, such as "crop · 9:16 · PNG". */ +/** + * Card title: the bot's title or repaint prompt, else language-neutral tokens for what the edit + * does, such as "300×300 · 90° · ↔ · 9:16 · PNG". + */ export function editImageTitle(input: EditImageInput): string { + if (input.title) return input.title; if (input.prompt) return input.prompt; - const { resize } = input; + const { crop, resize } = input; const parts = [ - input.crop && "crop", + crop && `${crop.width}×${crop.height}`, input.rotate && `${input.rotate}°`, - input.flip && "flip", + input.flip && (input.flip === "horizontal" ? "↔" : "↕"), resize && (resize.aspect ? MEDIA_PRESETS[resize.aspect].aspect : `${resize.width ?? "…"}×${resize.height ?? "…"}`), - input.extend && "extend", input.format?.toUpperCase(), ]; return parts.filter(Boolean).join(" · "); @@ -256,8 +260,10 @@ async function editImage( const format = input.format ?? defaultFormat(source.mimeType); const encoded = await encodeImage(png, format); const mimeType = IMAGE_FORMAT_MIME_TYPES[format]; - const stem = source.name.replace(/\.[^.]+$/, "") || "image"; - const name = `${stem}-edit${attachmentExtensionForMimeType(mimeType)}`; + const stem = input.title + ? cleanFileName(input.title, "image") + : `${cleanFileName(source.name.replace(/\.[^.]+$/, ""), "image")}-edit`; + const name = `${stem}${attachmentExtensionForMimeType(mimeType)}`; const path = join(context.tempDir, name); await writeFile(path, encoded.bytes); await context.addOutput({ diff --git a/packages/contracts/src/media-generation.ts b/packages/contracts/src/media-generation.ts index 2e61c42..cbec033 100644 --- a/packages/contracts/src/media-generation.ts +++ b/packages/contracts/src/media-generation.ts @@ -1,6 +1,7 @@ import * as z from "zod"; import { Id } from "./ids.js"; import { MediaFitSchema, MediaPresetIdSchema } from "./media.js"; +import { MediaTitleSchema } from "./media-video.js"; export const MEDIA_IMAGE_MAX_COUNT = 4; export const MEDIA_VIDEO_MAX_SECONDS = 10; @@ -55,6 +56,9 @@ export type ImageFormat = z.infer; export const EditImageInputSchema = z .object({ artifactId: Id.describe("Image artifact to edit."), + title: MediaTitleSchema.optional().describe( + "A few words in the user's language for the card and file name.", + ), crop: z .object({ left: Pixels, top: Pixels, width: Pixels.min(1), height: Pixels.min(1) }) .optional() diff --git a/packages/contracts/src/media-video.ts b/packages/contracts/src/media-video.ts index f0aaf5b..a12bf51 100644 --- a/packages/contracts/src/media-video.ts +++ b/packages/contracts/src/media-video.ts @@ -14,7 +14,7 @@ const TimeRangeSchema = z .refine((range) => range.end > range.start, "end must be after start"); /** Written by the bot in the user's language; shown on the media card and used as file name. */ -const MediaTitleSchema = z.string().trim().min(1).max(120); +export const MediaTitleSchema = z.string().trim().min(1).max(120); export const EditVideoStepSchema = z.discriminatedUnion("op", [ z