diff --git a/infra/compose/Dockerfile b/infra/compose/Dockerfile index fa6b508..e40d6f5 100644 --- a/infra/compose/Dockerfile +++ b/infra/compose/Dockerfile @@ -1,4 +1,8 @@ FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a +# ffmpeg/ffprobe cut, reframe and export videos; DejaVu is the burned-in subtitle font. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* RUN corepack enable WORKDIR /app # Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index df1e679..9907789 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -3,6 +3,8 @@ import { BotSecretDestination, BotSecretName, EditImageInputSchema, + EditVideoInputSchema, + ExportVideoInputSchema, GenerateImageInputSchema, GenerateVideoInputSchema, MODEL_TIERS, @@ -25,6 +27,9 @@ function mediaParameters(schema: z.ZodType) { export const SUBAGENT_MODEL_TIER_DESCRIPTION = "Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set."; +const VIDEO_PRESETS_AND_FITS = + "Presets: vertical 9:16, landscape 16:9, square 1:1, portrait 4:5, classic 4:3, cinema 21:9. fit: crop (default) fills the frame and cuts the edges, pad adds bars, blur fills the bars with a blurred copy."; + export const DELEGATION_TOOL_NAMES = new Set([ "run_subagent", "spawn_bot", @@ -357,6 +362,16 @@ export const builtinAgentTools: ConnectorTool[] = [ "Edit an image artifact into a new one: crop, rotate, flip, resize (crop, pad, or blur to fit a format), add borders, convert the format, or repaint it from a prompt (only when generate_image is available). Edits apply in that order.", inputSchema: mediaParameters(EditImageInputSchema), }, + { + name: "edit_video", + description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`, + inputSchema: mediaParameters(EditVideoInputSchema), + }, + { + name: "export_video", + description: `Export a video artifact as one H.264/AAC MP4 per preset. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`, + inputSchema: mediaParameters(ExportVideoInputSchema), + }, { name: "add_mcp_server", description: diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 8b84fdd..f8a7d51 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -78,6 +78,16 @@ export * from "./mcp-emulator.js"; export * from "./mcp-oauth.js"; export * from "./mcp-server-tool.js"; export * from "./mcp-transport.js"; +export type { MediaProbe, RunFfmpegOptions } from "./media/ffmpeg.js"; +export { + detectFfmpeg, + extractAudio, + FfmpegError, + probeMedia, + renderPoster, + runFfmpeg, +} from "./media/ffmpeg.js"; +export { createVideoOperations } from "./media/video-operations.js"; export * from "./media-operations.js"; export * from "./media-runtime.js"; export * from "./media-settings.js"; diff --git a/packages/adapters/src/media-operations.ts b/packages/adapters/src/media-operations.ts index 8a6d556..bdfa74e 100644 --- a/packages/adapters/src/media-operations.ts +++ b/packages/adapters/src/media-operations.ts @@ -1,5 +1,6 @@ import type { MediaSettings } from "@rakazo/contracts"; import { createImageOperations } from "./media/image-operations.js"; +import { createVideoOperations } from "./media/video-operations.js"; import type { MediaOperationRegistration } from "./media-runtime.js"; export interface MediaOperationDeps { @@ -12,5 +13,5 @@ export interface MediaOperationDeps { * from this one list, so both processes run the same operations. */ export function createMediaOperations(deps: MediaOperationDeps): MediaOperationRegistration[] { - return [...createImageOperations(deps)]; + return [...createImageOperations(deps), ...createVideoOperations()]; } diff --git a/packages/adapters/src/media/ffmpeg.test.ts b/packages/adapters/src/media/ffmpeg.test.ts new file mode 100644 index 0000000..e431827 --- /dev/null +++ b/packages/adapters/src/media/ffmpeg.test.ts @@ -0,0 +1,255 @@ +import { existsSync } from "node:fs"; +import { rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { FfmpegProgress } from "./ffmpeg.js"; +import { + createFfmpegProgressParser, + detectFfmpeg, + extractAudio, + FfmpegError, + ffmpegInputArgs, + ffmpegSeconds, + packetDurationMs, + parseProbeOutput, + probeMedia, + renderPoster, + runFfmpeg, +} from "./ffmpeg.js"; +import type { TestClips } from "./media-test-support.js"; +import { createTestClips } from "./media-test-support.js"; + +const ffmpeg = await detectFfmpeg(); + +describe("ffmpeg progress", () => { + it("reports each block, tolerates split chunks and N/A times, and marks the end", () => { + const updates: FfmpegProgress[] = []; + const feed = createFfmpegProgressParser((update) => updates.push(update)); + feed("frame=0\nout_time_us=N/A\ntotal_size=48\nprogress=cont"); + feed("inue\nframe=75\nout_time_us=2995"); + feed("374\nout_time_ms=2995374\ntotal_size=42300\nprogress=end\n"); + expect(updates).toEqual([ + { totalSize: 48, done: false }, + { outTimeUs: 2_995_374, totalSize: 42_300, done: true }, + ]); + }); +}); + +describe("probe parsing", () => { + const stream = (fields: Record) => ({ + codec_type: "video", + codec_name: "h264", + ...fields, + }); + + it("swaps the size of rotated video and reads frame rate and audio", () => { + const probe = parseProbeOutput( + JSON.stringify({ + format: { duration: "12.345", format_name: "mov,mp4,m4a,3gp,3g2,mj2" }, + streams: [ + stream({ + width: 1920, + height: 1080, + avg_frame_rate: "30000/1001", + sample_aspect_ratio: "1:1", + side_data_list: [{ side_data_type: "Display Matrix", rotation: -90 }], + }), + { codec_type: "audio", codec_name: "aac", sample_rate: "48000", channels: 2 }, + ], + }), + ); + expect(probe).toEqual({ + durationMs: 12_345, + video: { + codec: "h264", + width: 1080, + height: 1920, + rotation: 270, + fps: 30000 / 1001, + sampleAspectRatio: 1, + }, + audio: { codec: "aac", sampleRate: 48_000, channels: 2 }, + hasAudio: true, + }); + }); + + it("reads legacy rotate tags, skips cover art and falls back to stream durations", () => { + const probe = parseProbeOutput( + JSON.stringify({ + format: { format_name: "mov" }, + streams: [ + stream({ width: 600, height: 600, disposition: { attached_pic: 1 } }), + stream({ + width: 640, + height: 480, + avg_frame_rate: "0/0", + r_frame_rate: "90000/1", + sample_aspect_ratio: "4:3", + tags: { rotate: "90" }, + duration: "4.5", + }), + ], + }), + ); + expect(probe.durationMs).toBe(4500); + expect(probe.hasAudio).toBe(false); + expect(probe.video).toMatchObject({ + width: 480, + height: 640, + rotation: 90, + fps: 0, + sampleAspectRatio: 3 / 4, + }); + }); + + it("reads the end of the last packet when the container has no duration", () => { + expect(packetDurationMs("0.000000,0.040000\n2.960000,0.040000\nN/A,N/A\n")).toBe(3000); + expect(packetDurationMs("")).toBe(0); + }); +}); + +describe("ffmpeg arguments", () => { + it("opens inputs through the file protocol with the demuxer of their MIME type", () => { + expect( + ffmpegInputArgs("/a/b.webm", { + mimeType: "video/webm", + startSeconds: 2, + durationSeconds: 1.5, + }), + ).toEqual([ + ...["-ss", "2", "-t", "1.5", "-protocol_whitelist", "file", "-f", "matroska"], + ...["-i", "file:/a/b.webm"], + ]); + expect(ffmpegInputArgs("/a/b", { mimeType: "application/octet-stream" })).toEqual([ + ...["-protocol_whitelist", "file", "-i", "file:/a/b"], + ]); + expect(ffmpegSeconds(1e-7)).toBe("0"); + expect(ffmpegSeconds(12.34567)).toBe("12.346"); + }); +}); + +// CI runners may not have ffmpeg; these run wherever it is installed. +describe.skipIf(!ffmpeg)("ffmpeg integration", () => { + let clips: TestClips; + + beforeAll(async () => { + clips = await createTestClips(); + }); + + afterAll(async () => { + if (clips) await rm(clips.dir, { recursive: true, force: true }); + }); + + it("probes duration, size, frame rate and audio", async () => { + const probe = await probeMedia(clips.landscape, { mimeType: "video/mp4" }); + expect(probe.durationMs).toBeGreaterThanOrEqual(2900); + expect(probe.durationMs).toBeLessThanOrEqual(3100); + expect(probe.video).toMatchObject({ codec: "h264", width: 320, height: 180, fps: 25 }); + expect(probe.hasAudio).toBe(true); + const silent = await probeMedia(clips.silentPortrait, { mimeType: "video/mp4" }); + expect(silent.hasAudio).toBe(false); + expect(silent.video).toMatchObject({ width: 180, height: 320 }); + }); + + it("refuses a playlist disguised as an MP4 instead of following its URLs", async () => { + const playlist = join(clips.dir, "playlist.mp4"); + await writeFile( + playlist, + `#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:1.0,\nfile:${clips.srt}\n#EXT-X-ENDLIST\n`, + ); + await expect(probeMedia(playlist, { mimeType: "video/mp4" })).rejects.toThrow(FfmpegError); + }); + + it("reports progress up to 1 for a run", async () => { + const output = join(clips.dir, "progress.mp4"); + const fractions: number[] = []; + await runFfmpeg( + [ + ...ffmpegInputArgs(clips.landscape, { mimeType: "video/mp4" }), + ...[ + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-c:a", + "aac", + "-f", + "mp4", + `file:${output}`, + ], + ], + { durationMs: 3000, onProgress: (fraction) => fractions.push(fraction) }, + ); + expect(fractions.at(-1)).toBe(1); + expect(fractions).toEqual([...fractions].sort((a, b) => a - b)); + expect(existsSync(output)).toBe(true); + }); + + it("fails with the last stderr line, without directory paths, and removes outputs", async () => { + const output = join(clips.dir, "broken.mp4"); + await writeFile(output, "partial"); + const failure = runFfmpeg( + [ + ...ffmpegInputArgs(join(clips.dir, "missing.mp4"), { mimeType: "video/mp4" }), + `file:${output}`, + ], + { outputs: [output] }, + ); + const error = await failure.catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(FfmpegError); + expect((error as Error).message).toMatch(/^ffmpeg failed: .*missing\.mp4.*No such file/); + expect((error as Error).message).not.toContain(clips.dir); + expect(existsSync(output)).toBe(false); + }); + + it("kills ffmpeg on abort and removes the partial output", async () => { + const output = join(clips.dir, "aborted.mp4"); + const controller = new AbortController(); + const run = runFfmpeg( + [ + ...["-re", "-f", "lavfi", "-i", "testsrc=size=320x180:rate=25"], + ...["-t", "60", "-c:v", "libx264", "-preset", "ultrafast", "-f", "mp4", `file:${output}`], + ], + { + signal: controller.signal, + durationMs: 60_000, + onProgress: () => controller.abort(new Error("cancelled")), + outputs: [output], + }, + ); + await expect(run).rejects.toThrow("cancelled"); + expect(existsSync(output)).toBe(false); + }); + + it("stops when the output outgrows its cap", async () => { + const output = join(clips.dir, "too-large.mp4"); + const run = runFfmpeg( + [ + ...["-f", "lavfi", "-i", "testsrc=size=640x360:rate=25:duration=20"], + ...["-c:v", "libx264", "-preset", "ultrafast", "-crf", "0", "-f", "mp4", `file:${output}`], + ], + { maxOutputBytes: 64 * 1024, outputs: [output] }, + ); + await expect(run).rejects.toThrow("The result would exceed 0 MB."); + expect(existsSync(output)).toBe(false); + }); + + it("extracts 16 kHz mono WAV for transcription, optionally a range", async () => { + const output = join(clips.dir, "speech.wav"); + await extractAudio(clips.landscape, output, { mimeType: "video/mp4", startMs: 1000 }); + const probe = await probeMedia(output, { mimeType: "audio/wav" }); + expect(probe.audio).toEqual({ codec: "pcm_s16le", sampleRate: 16_000, channels: 1 }); + expect(probe.video).toBeUndefined(); + expect(Math.abs(probe.durationMs - 2000)).toBeLessThan(100); + await expect( + extractAudio(clips.silentPortrait, join(clips.dir, "none.wav"), { mimeType: "video/mp4" }), + ).rejects.toThrow("The file has no audio track."); + }); + + it("renders a poster frame, falling back to the first frame past the end", async () => { + const poster = join(clips.dir, "poster.jpg"); + await renderPoster(clips.landscape, poster, { mimeType: "video/mp4", atMs: 60_000 }); + const probe = await probeMedia(poster); + expect(probe.video).toMatchObject({ codec: "mjpeg", width: 320, height: 180 }); + }); +}); diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts new file mode 100644 index 0000000..72eefc1 --- /dev/null +++ b/packages/adapters/src/media/ffmpeg.ts @@ -0,0 +1,588 @@ +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { access, rm, stat } from "node:fs/promises"; +import { setPriority } from "node:os"; +import { delimiter, join } from "node:path"; + +/** + * ffmpeg and ffprobe from PATH. Every call passes an argument array (never a shell), opens + * inputs through the `file` protocol only, and forces the demuxer from the artifact's MIME type + * so a crafted upload cannot make ffmpeg probe it as a playlist that reads other files or URLs. + */ + +export interface FfmpegInfo { + ffmpegPath: string; + ffprobePath: string; +} + +export interface MediaProbe { + durationMs: number; + /** Width and height as displayed: swapped when the stream is rotated by 90 or 270 degrees. */ + video?: { + codec: string; + width: number; + height: number; + rotation: 0 | 90 | 180 | 270; + /** Frames per second; 0 when unknown or implausible (variable frame rate files). */ + fps: number; + /** Pixel aspect ratio of the displayed frame; 1 for square pixels. */ + sampleAspectRatio: number; + }; + audio?: { codec: string; sampleRate: number; channels: number }; + hasAudio: boolean; +} + +export interface FfmpegProgress { + outTimeUs?: number; + totalSize?: number; + done: boolean; +} + +export interface RunFfmpegOptions { + signal?: AbortSignal; + /** Expected output duration; progress is reported as out_time / durationMs. */ + durationMs?: number; + onProgress?: (fraction: number) => void; + cwd?: string; + /** Kills ffmpeg once the output grows past this size. */ + maxOutputBytes?: number; + /** Files removed when the run fails or is aborted. */ + outputs?: readonly string[]; +} + +export class FfmpegError extends Error { + constructor( + message: string, + readonly exitCode: number | null, + readonly stderrTail: string, + ) { + super(message); + this.name = "FfmpegError"; + } +} + +/** ffmpeg reports progress twice a second while it works; silence this long means a hang. */ +const STALL_TIMEOUT_MS = 2 * 60_000; +const PROBE_TIMEOUT_MS = 60_000; +const STDERR_TAIL_BYTES = 8 * 1024; +const KILL_GRACE_MS = 5_000; +const MAX_PLAUSIBLE_FPS = 240; + +const DEMUXERS: Record = { + "video/mp4": "mov", + "video/quicktime": "mov", + "audio/mp4": "mov", + "video/webm": "matroska", + "audio/webm": "matroska", + "audio/mpeg": "mp3", + "audio/wav": "wav", + "audio/ogg": "ogg", +}; + +let detection: Promise | undefined; + +/** Finds ffmpeg and ffprobe on PATH once per process; null when either is missing or broken. */ +export function detectFfmpeg(): Promise { + detection ??= locateFfmpeg(); + return detection; +} + +async function locateFfmpeg(): Promise { + const [ffmpegPath, ffprobePath] = await Promise.all([ + findExecutable("ffmpeg"), + findExecutable("ffprobe"), + ]); + if (!ffmpegPath || !ffprobePath) return null; + try { + const [ffmpeg, ffprobe] = await Promise.all([ + capture(ffmpegPath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }), + capture(ffprobePath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }), + ]); + return ffmpeg.code === 0 && ffprobe.code === 0 ? { ffmpegPath, ffprobePath } : null; + } catch { + return null; + } +} + +async function findExecutable(name: string): Promise { + const extensions = + process.platform === "win32" + ? (process.env.PATHEXT ?? ".EXE").split(";").filter(Boolean) + : [""]; + for (const directory of (process.env.PATH ?? "").split(delimiter)) { + if (!directory) continue; + for (const extension of extensions) { + const candidate = join(directory, `${name}${extension}`); + try { + await access(candidate, constants.X_OK); + if ((await stat(candidate)).isFile()) return candidate; + } catch { + // Not in this directory. + } + } + } + return null; +} + +async function requireFfmpeg(): Promise { + const info = await detectFfmpeg(); + if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server."); + return info; +} + +/** ffmpeg time argument with millisecond precision and no exponent notation. */ +export function ffmpegSeconds(seconds: number): string { + return String(Number(Math.max(0, seconds).toFixed(3))); +} + +/** Input arguments for a local file: optional seek, file protocol only, forced demuxer. */ +export function ffmpegInputArgs( + path: string, + options: { mimeType?: string; startSeconds?: number; durationSeconds?: number } = {}, +): string[] { + const demuxer = options.mimeType ? DEMUXERS[options.mimeType] : undefined; + return [ + ...(options.startSeconds ? ["-ss", ffmpegSeconds(options.startSeconds)] : []), + ...(options.durationSeconds !== undefined + ? ["-t", ffmpegSeconds(options.durationSeconds)] + : []), + "-protocol_whitelist", + "file", + ...(demuxer ? ["-f", demuxer] : []), + "-i", + `file:${path}`, + ]; +} + +export async function probeMedia( + path: string, + options: { signal?: AbortSignal; mimeType?: string } = {}, +): Promise { + const { ffprobePath } = await requireFfmpeg(); + const input = ffmpegInputArgs(path, { mimeType: options.mimeType }); + const result = await capture( + ffprobePath, + ["-v", "error", ...input, "-print_format", "json", "-show_format", "-show_streams"], + { signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS }, + ); + if (result.code !== 0) throw ffmpegFailure("ffprobe", result.code, result.stderr); + const probe = parseProbeOutput(result.stdout); + if (probe.durationMs > 0 || (!probe.video && !probe.audio)) return probe; + // Recordings streamed by browsers often carry no duration header; read it from the packets. + const packets = await capture( + ffprobePath, + [ + "-v", + "error", + ...input, + "-select_streams", + probe.video ? "v:0" : "a:0", + "-show_entries", + "packet=pts_time,duration_time", + "-of", + "csv=p=0", + ], + { signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS * 5 }, + ); + if (packets.code !== 0) throw ffmpegFailure("ffprobe", packets.code, packets.stderr); + return { ...probe, durationMs: packetDurationMs(packets.stdout) }; +} + +/** End of the last packet, from ffprobe `packet=pts_time,duration_time` CSV lines. */ +export function packetDurationMs(csv: string): number { + let end = 0; + for (const line of csv.split("\n")) { + const [pts, duration] = line.split(",").map(Number); + if (pts === undefined || !Number.isFinite(pts)) continue; + end = Math.max(end, pts + (duration !== undefined && Number.isFinite(duration) ? duration : 0)); + } + return Math.round(end * 1000); +} + +interface ProbeStream { + codec_type?: string; + codec_name?: string; + width?: number; + height?: number; + avg_frame_rate?: string; + r_frame_rate?: string; + sample_aspect_ratio?: string; + sample_rate?: string; + channels?: number; + duration?: string; + disposition?: { attached_pic?: number }; + tags?: { rotate?: string }; + side_data_list?: { rotation?: number }[]; +} + +/** Parses `ffprobe -print_format json -show_format -show_streams` output. */ +export function parseProbeOutput(json: string): MediaProbe { + const data = JSON.parse(json) as { + format?: { duration?: string }; + streams?: ProbeStream[]; + }; + const streams = data.streams ?? []; + // Cover art in audio files is a single attached picture, not a video track. + const video = streams.find( + (stream) => stream.codec_type === "video" && stream.disposition?.attached_pic !== 1, + ); + const audio = streams.find((stream) => stream.codec_type === "audio"); + const seconds = + positive(Number(data.format?.duration)) ?? + Math.max(0, ...streams.map((stream) => positive(Number(stream.duration)) ?? 0)); + return { + durationMs: Math.round(seconds * 1000), + ...(video?.width && video.height + ? { video: probeVideo(video, video.width, video.height) } + : {}), + ...(audio + ? { + audio: { + codec: audio.codec_name ?? "", + sampleRate: Number(audio.sample_rate) || 0, + channels: audio.channels ?? 0, + }, + } + : {}), + hasAudio: Boolean(audio), + }; +} + +function probeVideo(stream: ProbeStream, width: number, height: number) { + const rotation = normalizeRotation( + stream.side_data_list?.find((entry) => typeof entry.rotation === "number")?.rotation ?? + Number(stream.tags?.rotate ?? 0), + ); + const sideways = rotation === 90 || rotation === 270; + const sar = ratio(stream.sample_aspect_ratio, ":") || 1; + const fps = ratio(stream.avg_frame_rate, "/") || ratio(stream.r_frame_rate, "/"); + return { + codec: stream.codec_name ?? "", + width: sideways ? height : width, + height: sideways ? width : height, + rotation, + fps: fps > 0 && fps <= MAX_PLAUSIBLE_FPS ? fps : 0, + sampleAspectRatio: sideways ? 1 / sar : sar, + }; +} + +function normalizeRotation(degrees: number): 0 | 90 | 180 | 270 { + if (!Number.isFinite(degrees)) return 0; + const quarter = (((Math.round(degrees / 90) % 4) + 4) % 4) as 0 | 1 | 2 | 3; + return ([0, 90, 180, 270] as const)[quarter]; +} + +function ratio(value: string | undefined, separator: string): number { + if (!value) return 0; + const [numerator, denominator] = value.split(separator).map(Number); + if (!numerator || !denominator || !Number.isFinite(numerator / denominator)) return 0; + return numerator / denominator; +} + +function positive(value: number): number | undefined { + return Number.isFinite(value) && value > 0 ? value : undefined; +} + +/** Feeds ffmpeg `-progress` output (key=value lines) and reports each completed block. */ +export function createFfmpegProgressParser( + onUpdate: (progress: FfmpegProgress) => void, +): (chunk: string) => void { + let buffer = ""; + let current: Omit = {}; + return (chunk) => { + buffer += chunk; + for (let newline = buffer.indexOf("\n"); newline >= 0; newline = buffer.indexOf("\n")) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + const separator = line.indexOf("="); + if (separator < 0) continue; + const key = line.slice(0, separator); + const value = line.slice(separator + 1).trim(); + const number = Number(value); + // out_time_ms is microseconds too (a long-standing ffmpeg naming bug). + if ((key === "out_time_us" || key === "out_time_ms") && value && number >= 0) { + current.outTimeUs = number; + } else if (key === "total_size" && value && number >= 0) { + current.totalSize = number; + } else if (key === "progress") { + onUpdate({ ...current, done: value === "end" }); + current = {}; + } + } + }; +} + +/** Runs ffmpeg with progress on stdout; rejects with the stderr tail when it fails. */ +export async function runFfmpeg( + args: readonly string[], + options: RunFfmpegOptions = {}, +): Promise { + options.signal?.throwIfAborted(); + const { ffmpegPath } = await requireFfmpeg(); + try { + await spawnFfmpeg(ffmpegPath, args, options); + } catch (error) { + await Promise.all((options.outputs ?? []).map((path) => rm(path, { force: true }))); + throw error; + } +} + +function spawnFfmpeg( + ffmpegPath: string, + args: readonly string[], + options: RunFfmpegOptions, +): Promise { + const { signal, durationMs, onProgress, maxOutputBytes } = options; + return new Promise((resolve, reject) => { + const child = spawn( + ffmpegPath, + [ + "-hide_banner", + "-nostdin", + "-nostats", + "-loglevel", + "error", + "-progress", + "pipe:1", + "-y", + ...args, + ], + { cwd: options.cwd, stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + if (child.pid) { + try { + // Long encodes must not starve the API process that shares this machine. + setPriority(child.pid, 10); + } catch { + // Lowering priority is best effort. + } + } + let stderr = ""; + let settled = false; + let reported = 0; + let failure: unknown; + let stallTimer: NodeJS.Timeout | undefined; + const finish = (error?: unknown) => { + if (settled) return; + settled = true; + clearTimeout(stallTimer); + signal?.removeEventListener("abort", abort); + if (error === undefined) resolve(); + else reject(error); + }; + // Settles once the process is gone, so callers can delete its files right away. + const kill = (error: unknown) => { + if (failure !== undefined || settled) return; + failure = error; + clearTimeout(stallTimer); + child.kill("SIGKILL"); + setTimeout(() => finish(error), KILL_GRACE_MS).unref?.(); + }; + const abort = () => kill(signal?.reason ?? new Error("aborted")); + const armStallTimer = () => { + clearTimeout(stallTimer); + stallTimer = setTimeout( + () => kill(new FfmpegError("ffmpeg stopped making progress.", null, tail(stderr))), + STALL_TIMEOUT_MS, + ); + stallTimer.unref?.(); + }; + const parse = createFfmpegProgressParser((progress) => { + if (failure !== undefined) return; + if (maxOutputBytes !== undefined && (progress.totalSize ?? 0) > maxOutputBytes) { + kill(new Error(outputTooLargeMessage(maxOutputBytes))); + return; + } + if (!onProgress) return; + const fraction = progress.done + ? 1 + : durationMs && progress.outTimeUs !== undefined + ? Math.min(1, progress.outTimeUs / (durationMs * 1000)) + : 0; + if (fraction > reported) { + reported = fraction; + onProgress(fraction); + } + }); + signal?.addEventListener("abort", abort, { once: true }); + armStallTimer(); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + if (failure === undefined) armStallTimer(); + parse(chunk); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr = tail(stderr + chunk); + }); + child.on("error", (error) => finish(failure ?? error)); + child.on("close", (code) => { + if (failure !== undefined) finish(failure); + else if (code === 0) finish(); + else finish(ffmpegFailure("ffmpeg", code, stderr)); + }); + if (signal?.aborted) abort(); + }); +} + +export function outputTooLargeMessage(maxBytes: number): string { + return `The result would exceed ${Math.round(maxBytes / (1024 * 1024))} MB.`; +} + +function tail(text: string): string { + return text.length > STDERR_TAIL_BYTES ? text.slice(-STDERR_TAIL_BYTES) : text; +} + +function ffmpegFailure(tool: string, code: number | null, stderr: string): FfmpegError { + // The cause is usually the line before ffmpeg's generic summary ("Conversion failed!"). + const lines = stderr + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .slice(-2); + // Name files by their base name only; the job's temp paths mean nothing to the reader. + const detail = lines.join(" ").replaceAll(/(?:file:)?(?:[A-Za-z]:)?[\\/][^\s'"]*[\\/]/g, ""); + return new FfmpegError( + detail ? `${tool} failed: ${detail}` : `${tool} failed with exit code ${code}.`, + code, + tail(stderr), + ); +} + +/** Extracts the first audio track as 16 kHz mono 16-bit WAV, the format speech models expect. */ +export async function extractAudio( + input: string, + output: string, + options: { + signal?: AbortSignal; + mimeType?: string; + startMs?: number; + endMs?: number; + onProgress?: (fraction: number) => void; + } = {}, +): Promise { + const probe = await probeMedia(input, { signal: options.signal, mimeType: options.mimeType }); + if (!probe.hasAudio) throw new Error("The file has no audio track."); + const startMs = Math.max(0, options.startMs ?? 0); + const endMs = Math.min(probe.durationMs, options.endMs ?? probe.durationMs); + if (endMs <= startMs) throw new Error("The audio range is empty."); + await runFfmpeg( + [ + ...ffmpegInputArgs(input, { + mimeType: options.mimeType, + startSeconds: startMs / 1000, + durationSeconds: (endMs - startMs) / 1000, + }), + "-map", + "0:a:0", + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "pcm_s16le", + "-map_metadata", + "-1", + "-f", + "wav", + `file:${output}`, + ], + { + signal: options.signal, + durationMs: endMs - startMs, + onProgress: options.onProgress, + outputs: [output], + }, + ); +} + +/** Writes one JPEG frame at atMs (default 0), falling back to the first frame past the end. */ +export async function renderPoster( + input: string, + output: string, + options: { signal?: AbortSignal; mimeType?: string; atMs?: number } = {}, +): Promise { + const render = (atMs: number) => + runFfmpeg( + [ + ...ffmpegInputArgs(input, { mimeType: options.mimeType, startSeconds: atMs / 1000 }), + "-map", + "0:v:0", + "-frames:v", + "1", + "-c:v", + "mjpeg", + "-q:v", + "3", + "-map_metadata", + "-1", + "-f", + "image2", + "-update", + "1", + `file:${output}`, + ], + { signal: options.signal, outputs: [output] }, + ); + const atMs = Math.max(0, options.atMs ?? 0); + await render(atMs); + // Seeking into the last frame's gap writes nothing; the first frame is better than no poster. + if (atMs > 0 && !(await nonEmptyFile(output))) await render(0); + if (!(await nonEmptyFile(output))) throw new Error("Could not render a poster frame."); +} + +async function nonEmptyFile(path: string): Promise { + try { + return (await stat(path)).size > 0; + } catch { + return false; + } +} + +interface CaptureResult { + code: number | null; + stdout: string; + stderr: string; +} + +function capture( + command: string, + args: readonly string[], + options: { signal?: AbortSignal; timeoutMs: number }, +): Promise { + options.signal?.throwIfAborted(); + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }); + const stdout: Buffer[] = []; + let stderr = ""; + let settled = false; + const finish = (error: unknown, result?: CaptureResult) => { + if (settled) return; + settled = true; + clearTimeout(timer); + options.signal?.removeEventListener("abort", abort); + if (result) resolve(result); + else reject(error); + }; + const kill = (error: unknown) => { + child.kill("SIGKILL"); + finish(error); + }; + const abort = () => kill(options.signal?.reason ?? new Error("aborted")); + const timer = setTimeout( + () => kill(new Error(`${command} timed out after ${options.timeoutMs} ms`)), + options.timeoutMs, + ); + timer.unref?.(); + options.signal?.addEventListener("abort", abort, { once: true }); + child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk)); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr = tail(stderr + chunk); + }); + child.on("error", (error) => finish(error)); + child.on("close", (code) => + finish(undefined, { code, stdout: Buffer.concat(stdout).toString("utf8"), stderr }), + ); + }); +} diff --git a/packages/adapters/src/media/image-operations.test.ts b/packages/adapters/src/media/image-operations.test.ts index e028552..f414c08 100644 --- a/packages/adapters/src/media/image-operations.test.ts +++ b/packages/adapters/src/media/image-operations.test.ts @@ -223,6 +223,13 @@ describe("image operations", () => { expect((await sharp(outputs[0]!.path).metadata()).format).toBe("webp"); expect(result.summary).toBe("Edited photo.jpg into photo-edit.webp (1080×1920)."); expect(emulator.requests).toHaveLength(0); + const titled = await run( + operation, + "edit_image", + { artifactId: "photo-1", title: "Titelbild: Sommer", rotate: 180 }, + { "photo-1": photo }, + ); + expect(titled.outputs[0]?.name).toBe("Titelbild Sommer.jpg"); await expect( run(operation, "edit_image", { artifactId: "photo-1", prompt: "snow" }, { "photo-1": photo }), ).rejects.toThrow("ComfyUI is not configured"); @@ -267,7 +274,17 @@ describe("helpers", () => { resize: { aspect: "vertical", fit: "crop" }, format: "png", }), - ).toBe("crop · 90° · 9:16 · PNG"); + ).toBe("1×1 · 90° · 9:16 · PNG"); + expect( + editImageTitle({ + artifactId: "a", + flip: "vertical", + extend: { top: 4, right: 0, bottom: 0, left: 0 }, + }), + ).toBe("↕"); + expect( + editImageTitle({ artifactId: "a", title: "Titelbild", prompt: "snow", rotate: 90 }), + ).toBe("Titelbild"); expect(editImageTitle({ artifactId: "a", resize: { width: 800, fit: "crop" } })).toBe("800×…"); expect(editImageTitle({ artifactId: "a", prompt: "make it snow" })).toBe("make it snow"); }); diff --git a/packages/adapters/src/media/image-operations.ts b/packages/adapters/src/media/image-operations.ts index 68c08aa..f19ab09 100644 --- a/packages/adapters/src/media/image-operations.ts +++ b/packages/adapters/src/media/image-operations.ts @@ -36,6 +36,7 @@ import { imageSize, sharpAvailable, } from "./image-edit.js"; +import { cleanFileName } from "./video-operations.js"; /** Free GPU memory a generation needs before it starts; fixed, so a custom workflow cannot lower it. */ export const IMAGE_GENERATION_MIN_VRAM_MB = 6_000; @@ -75,19 +76,22 @@ export function fileStem(text: string, fallback: string): string { return words?.slice(0, 6).join("-").slice(0, 48).replace(/-+$/, "") || fallback; } -/** Short card title for an edit without a prompt, such as "crop · 9:16 · PNG". */ +/** + * Card title: the bot's title or repaint prompt, else language-neutral tokens for what the edit + * does, such as "300×300 · 90° · ↔ · 9:16 · PNG". + */ export function editImageTitle(input: EditImageInput): string { + if (input.title) return input.title; if (input.prompt) return input.prompt; - const { resize } = input; + const { crop, resize } = input; const parts = [ - input.crop && "crop", + crop && `${crop.width}×${crop.height}`, input.rotate && `${input.rotate}°`, - input.flip && "flip", + input.flip && (input.flip === "horizontal" ? "↔" : "↕"), resize && (resize.aspect ? MEDIA_PRESETS[resize.aspect].aspect : `${resize.width ?? "…"}×${resize.height ?? "…"}`), - input.extend && "extend", input.format?.toUpperCase(), ]; return parts.filter(Boolean).join(" · "); @@ -256,8 +260,10 @@ async function editImage( const format = input.format ?? defaultFormat(source.mimeType); const encoded = await encodeImage(png, format); const mimeType = IMAGE_FORMAT_MIME_TYPES[format]; - const stem = source.name.replace(/\.[^.]+$/, "") || "image"; - const name = `${stem}-edit${attachmentExtensionForMimeType(mimeType)}`; + const stem = input.title + ? cleanFileName(input.title, "image") + : `${cleanFileName(source.name.replace(/\.[^.]+$/, ""), "image")}-edit`; + const name = `${stem}${attachmentExtensionForMimeType(mimeType)}`; const path = join(context.tempDir, name); await writeFile(path, encoded.bytes); await context.addOutput({ diff --git a/packages/adapters/src/media/media-test-support.ts b/packages/adapters/src/media/media-test-support.ts new file mode 100644 index 0000000..940419a --- /dev/null +++ b/packages/adapters/src/media/media-test-support.ts @@ -0,0 +1,106 @@ +import { copyFile, mkdtemp, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit"; +import type { MediaOutput, MediaOutputKind } from "@rakazo/contracts"; +import { attachmentKindOf } from "@rakazo/contracts"; +import { runFfmpeg } from "./ffmpeg.js"; + +export interface TestClips { + dir: string; + /** 320×180, 25 fps, 3 s, 440 Hz tone. */ + landscape: string; + /** 180×320, 30 fps, 2 s, no audio. */ + silentPortrait: string; + srt: string; + vtt: string; +} + +/** Tiny generated clips; nothing binary is committed. */ +export async function createTestClips(): Promise { + const dir = await mkdtemp(join(tmpdir(), "evoo-media-test-")); + const landscape = join(dir, "landscape.mp4"); + const silentPortrait = join(dir, "silent-portrait.mp4"); + const encode = ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"]; + await runFfmpeg([ + ...["-f", "lavfi", "-i", "testsrc=size=320x180:rate=25:duration=3"], + ...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=3"], + ...encode, + ...["-c:a", "aac", "-shortest", `file:${landscape}`], + ]); + await runFfmpeg([ + ...["-f", "lavfi", "-i", "testsrc2=size=180x320:rate=30:duration=2"], + ...encode, + `file:${silentPortrait}`, + ]); + const srt = join(dir, "captions.srt"); + await writeFile( + srt, + "1\n00:00:00,100 --> 00:00:01,500\nHello from the subtitles\n\n2\n00:00:01,500 --> 00:00:02,900\nZweite Zeile äöü\n", + ); + const vtt = join(dir, "captions.vtt"); + await writeFile(vtt, "WEBVTT\n\n00:00.100 --> 00:01.500\nHello from WebVTT\n"); + return { dir, landscape, silentPortrait, srt, vtt }; +} + +export interface TestArtifact { + path: string; + name: string; + mimeType: string; + kind?: MediaOutputKind; +} + +/** A MediaJobContext over local files that records progress and outputs. */ +export async function fakeMediaJobContext(input: unknown, artifacts: Record) { + const tempDir = await mkdtemp(join(tmpdir(), "evoo-media-job-")); + const controller = new AbortController(); + const progress: number[] = []; + const outputs: (MediaJobOutputFile & { size: number; posterSize?: number })[] = []; + const context: MediaJobContext = { + jobId: "job-1", + spaceId: "space-1", + userId: "user-1", + botId: null, + input, + tempDir, + signal: controller.signal, + progress: async (fraction) => { + progress.push(fraction); + }, + fetchArtifact: async (artifactId): Promise => { + const artifact = artifacts[artifactId]; + if (!artifact) throw new Error(`Artifact ${artifactId} not found.`); + const path = join(tempDir, `input-${artifactId}`); + await copyFile(artifact.path, path); + return { + artifactId, + path, + name: artifact.name, + mimeType: artifact.mimeType, + kind: artifact.kind ?? attachmentKindOf(artifact.mimeType), + size: (await stat(path)).size, + }; + }, + addOutput: async (file): Promise => { + // Files must exist when handed over; the runner stores them before the job ends. + const size = (await stat(file.path)).size; + const posterSize = file.posterPath ? (await stat(file.posterPath)).size : undefined; + outputs.push({ ...file, size, ...(posterSize !== undefined ? { posterSize } : {}) }); + const index = outputs.length; + return { + artifactId: `output-${index}`, + kind: attachmentKindOf(file.mimeType), + mimeType: file.mimeType, + name: file.name, + size, + ...(file.width ? { width: file.width } : {}), + ...(file.height ? { height: file.height } : {}), + ...(file.durationMs !== undefined ? { durationMs: file.durationMs } : {}), + ...(file.posterPath ? { posterArtifactId: `poster-${index}` } : {}), + ...(file.parentArtifactId ? { parentArtifactId: file.parentArtifactId } : {}), + ...(file.preset ? { preset: file.preset } : {}), + }; + }, + }; + return { context, controller, progress, outputs, tempDir }; +} diff --git a/packages/adapters/src/media/video-edit-plan.test.ts b/packages/adapters/src/media/video-edit-plan.test.ts new file mode 100644 index 0000000..4927f9e --- /dev/null +++ b/packages/adapters/src/media/video-edit-plan.test.ts @@ -0,0 +1,300 @@ +import type { EditVideoStep } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { MediaProbe } from "./ffmpeg.js"; +import type { PlanInput } from "./video-edit-plan.js"; +import { atempoFilters, planVideoEdit, subtitleStyle } from "./video-edit-plan.js"; + +function probe( + overrides: Partial> & { + durationMs?: number; + hasAudio?: boolean; + } = {}, +): MediaProbe { + const { durationMs = 10_000, hasAudio = true, ...video } = overrides; + return { + durationMs, + video: { + codec: "h264", + width: 1920, + height: 1080, + rotation: 0, + fps: 25, + sampleAspectRatio: 1, + ...video, + }, + ...(hasAudio ? { audio: { codec: "aac", sampleRate: 44_100, channels: 2 } } : {}), + hasAudio, + }; +} + +const source: PlanInput = { + path: "/jobs/1/input-source", + mimeType: "video/mp4", + name: "holiday.mp4", + probe: probe(), +}; + +function plan(steps: EditVideoStep[], extra: Partial[0]> = {}) { + return planVideoEdit({ source, steps, outputPath: "/jobs/1/output.mp4", ...extra }); +} + +function graphOf(args: string[]): string { + return args[args.indexOf("-filter_complex") + 1]!; +} + +describe("planVideoEdit", () => { + it("seeks the input for a leading trim and encodes H.264/AAC MP4", () => { + const result = plan([{ op: "trim", start: 1.5, end: 4 }]); + expect(result.args).toEqual([ + ...["-ss", "1.5", "-t", "2.5", "-protocol_whitelist", "file", "-f", "mov"], + ...["-i", "file:/jobs/1/input-source"], + "-filter_complex", + "[0:v:0]format=yuv420p[v1];[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a2]", + ...["-map", "[v1]", "-map", "[a2]", "-c:a", "aac", "-b:a", "160k"], + ...["-c:v", "libx264", "-preset", "veryfast", "-crf", "21", "-pix_fmt", "yuv420p"], + ...["-movflags", "+faststart", "-map_metadata", "-1", "-map_chapters", "-1"], + ...["-f", "mp4", "file:/jobs/1/output.mp4"], + ]); + expect(result).toMatchObject({ kind: "video", durationMs: 2500, width: 1920, height: 1080 }); + }); + + it("keeps sorted, merged segments from one seek with split, trims and concat", () => { + const result = plan([ + { + op: "keep", + segments: [ + { start: 6, end: 8 }, + { start: 2, end: 3 }, + { start: 2.5, end: 4 }, + ], + }, + ]); + expect(result.args.slice(0, 4)).toEqual(["-ss", "2", "-t", "6"]); + const graph = graphOf(result.args); + expect(graph).toContain("[0:v:0]split=2[v1][v2]"); + expect(graph).toContain("[v1]trim=start=0:end=2,setpts=PTS-STARTPTS[v3]"); + expect(graph).toContain("[v2]trim=start=4:end=6,setpts=PTS-STARTPTS[v4]"); + expect(graph).toContain("[0:a:0]asplit=2[a5][a6]"); + expect(graph).toContain("[v3][a7][v4][a8]concat=n=2:v=1:a=1[v9][a10]"); + expect(result.durationMs).toBe(4000); + }); + + it("applies later cuts on the timeline left by earlier steps", () => { + const result = plan([ + { op: "speed", factor: 2 }, + { op: "trim", start: 1, end: 99 }, + ]); + expect(graphOf(result.args)).toContain( + "[0:v:0]setpts=PTS/2,fps=25,trim=start=1:end=5,setpts=PTS-STARTPTS,format=yuv420p[v1]", + ); + expect(result.durationMs).toBe(4000); + }); + + it("rejects ranges outside the video and too short ones", () => { + expect(() => plan([{ op: "trim", start: 12, end: 18 }])).toThrow( + "trim 12–18 s is outside the 10 s video.", + ); + expect(() => + plan([ + { op: "speed", factor: 2 }, + { + op: "keep", + segments: [ + { start: 1, end: 2 }, + { start: 6, end: 7 }, + ], + }, + ]), + ).toThrow("keep 6–7 s is outside the 5 s video."); + expect(() => plan([{ op: "trim", start: 9.95, end: 12 }])).toThrow("shorter than 0.1 s"); + }); + + it("appends clips fitted to the frame, with silence for clips without audio", () => { + const clip: PlanInput = { + path: "/jobs/1/input-clip", + mimeType: "video/webm", + name: "clip.webm", + probe: probe({ width: 1080, height: 1920, fps: 30, durationMs: 2000, hasAudio: false }), + }; + const result = plan([{ op: "concat", artifact_ids: ["clip"] }], { + clips: new Map([["clip", clip]]), + }); + expect(result.args).toContain("matroska"); + const graph = graphOf(result.args); + expect(graph).toContain("[0:v:0]setsar=1,format=yuv420p[v1]"); + expect(graph).toContain( + "[1:v:0]scale=1920:1080:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1920:1080:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=25,format=yuv420p[v3]", + ); + expect(graph).toContain( + "anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=2,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a4]", + ); + expect(graph).toContain("[v1][a2][v3][a4]concat=n=2:v=1:a=1"); + expect(result.durationMs).toBe(12_000); + }); + + it("adds silence to a silent source when an appended clip has audio", () => { + const silent = { ...source, probe: probe({ hasAudio: false }) }; + const result = planVideoEdit({ + source: silent, + steps: [{ op: "concat", artifact_ids: ["clip"] }], + clips: new Map([["clip", source]]), + outputPath: "/out.mp4", + }); + expect(graphOf(result.args)).toContain( + "anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=10", + ); + expect(result.args.filter((arg) => arg === "-map")).toHaveLength(2); + }); + + it("appends clips with the last reframe's fit", () => { + const result = plan( + [ + { op: "reframe", preset: "vertical", fit: "crop" }, + { op: "concat", artifact_ids: ["clip"] }, + ], + { clips: new Map([["clip", source]]) }, + ); + expect(graphOf(result.args)).toContain( + "[1:v:0]scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920,setsar=1,fps=25", + ); + }); + + it("reframes by crop, pad or blur, and only scales when the aspect already matches", () => { + expect(graphOf(plan([{ op: "reframe", preset: "square", fit: "crop" }]).args)).toContain( + "scale=1080:1080:force_original_aspect_ratio=increase,crop=1080:1080,setsar=1", + ); + expect(graphOf(plan([{ op: "reframe", preset: "portrait", fit: "pad" }]).args)).toContain( + "pad=1080:1350:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1", + ); + const blur = plan([{ op: "reframe", preset: "vertical", fit: "blur" }]); + expect(graphOf(blur.args)).toBe( + [ + "[0:v:0]split=2[v1][v2]", + "[v1]scale=270:480:force_original_aspect_ratio=increase,crop=270:480,boxblur=27:2,scale=1080:1920,setsar=1[v3]", + "[v2]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,setsar=1[v4]", + "[v3][v4]overlay=(W-w)/2:(H-h)/2[v5]", + "[v5]format=yuv420p[v6]", + "[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a7]", + ].join(";"), + ); + expect(blur).toMatchObject({ width: 1080, height: 1920 }); + expect(graphOf(plan([{ op: "reframe", preset: "landscape", fit: "blur" }]).args)).toContain( + "[0:v:0]scale=1920:1080,setsar=1,format=yuv420p", + ); + }); + + it("changes speed with atempo chains and keeps the source frame rate", () => { + const fast = plan([{ op: "speed", factor: 4 }]); + expect(graphOf(fast.args)).toContain("setpts=PTS/4,fps=25"); + expect(graphOf(fast.args)).toContain("atempo=2,atempo=2"); + expect(fast.durationMs).toBe(2500); + expect(atempoFilters(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]); + expect(atempoFilters(3)).toEqual(["atempo=2", "atempo=1.5"]); + expect(atempoFilters(0.8)).toEqual(["atempo=0.8"]); + }); + + it("changes volume, and mute drops the audio track", () => { + expect(graphOf(plan([{ op: "volume", gain: 0.5 }]).args)).toContain( + "[0:a:0]volume=0.5,aformat", + ); + const muted = plan([{ op: "volume", gain: 2 }, { op: "mute" }]); + expect(muted.args.filter((arg) => arg === "-map")).toHaveLength(1); + expect(muted.args).not.toContain("aac"); + }); + + it("burns in subtitles styled for the current frame from a generated file name", () => { + const result = plan( + [ + { op: "reframe", preset: "vertical", fit: "crop" }, + { op: "subtitles", artifact_id: "subs" }, + ], + { subtitleFiles: new Map([["subs", "subtitles-0.srt"]]) }, + ); + expect(graphOf(result.args)).toContain( + `subtitles=filename=subtitles-0.srt:force_style='${subtitleStyle(1080, 1920)}'`, + ); + expect(() => + plan([{ op: "subtitles", artifact_id: "subs" }], { + subtitleFiles: new Map([["subs", "x.srt:force_style='a'"]]), + }), + ).toThrow("Unsafe subtitle file name."); + }); + + it("styles subtitles above the controls of tall frames", () => { + expect(subtitleStyle(1080, 1920)).toContain("MarginV=384"); + expect(subtitleStyle(1080, 1920)).toContain("FontSize=65"); + expect(subtitleStyle(1920, 1080)).toContain("MarginV=65"); + expect(subtitleStyle(1920, 1080)).toContain("PlayResX=1920,PlayResY=1080"); + }); + + it("extracts audio without decoding video work", () => { + const result = planVideoEdit({ + source, + steps: [ + { op: "trim", start: 0, end: 5 }, + { op: "reframe", preset: "vertical", fit: "blur" }, + { op: "subtitles", artifact_id: "subs" }, + { op: "extract_audio" }, + ], + subtitleFiles: new Map([["subs", "subtitles-0.srt"]]), + outputPath: "/out.m4a", + }); + expect(graphOf(result.args)).toBe( + "[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a1]", + ); + expect(result.args).toEqual(expect.arrayContaining(["-map", "[a1]", "-b:a", "192k"])); + expect(result.args).not.toContain("libx264"); + expect(result).toMatchObject({ kind: "audio", durationMs: 5000 }); + expect(() => + planVideoEdit({ + source: { ...source, probe: probe({ hasAudio: false }) }, + steps: [{ op: "extract_audio" }], + outputPath: "/out.m4a", + }), + ).toThrow("holiday.mp4 has no audio."); + }); + + it("validates the poster time against the result", () => { + expect( + plan([ + { op: "trim", start: 0, end: 4 }, + { op: "poster", at: 3 }, + ]).posterAtMs, + ).toBe(3000); + expect(() => + plan([ + { op: "trim", start: 0, end: 4 }, + { op: "poster", at: 5 }, + ]), + ).toThrow("poster at 5 s is past the end of the 4 s result."); + }); + + it("keeps file paths out of the filter graph", () => { + const hostile = { ...source, path: "/jobs/1/a'b[c]:d;e,f.mp4", name: "x'];[0:v]drawtext=" }; + const result = planVideoEdit({ + source: hostile, + steps: [{ op: "reframe", preset: "square", fit: "blur" }], + outputPath: "/jobs/1/out.mp4", + }); + expect(graphOf(result.args)).not.toContain("a'b"); + expect(graphOf(result.args)).not.toContain("drawtext"); + expect(result.args).toContain("file:/jobs/1/a'b[c]:d;e,f.mp4"); + }); + + it("squares pixels and evens odd frame sizes", () => { + const anamorphic = { + ...source, + probe: probe({ width: 1440, height: 1080, sampleAspectRatio: 4 / 3 }), + }; + const result = planVideoEdit({ + source: anamorphic, + steps: [{ op: "mute" }], + outputPath: "/o.mp4", + }); + expect(graphOf(result.args)).toBe("[0:v:0]scale=1920:1080,setsar=1,format=yuv420p[v1]"); + expect(result).toMatchObject({ width: 1920, height: 1080 }); + const odd = { ...source, probe: probe({ width: 721, height: 405 }) }; + const evened = planVideoEdit({ source: odd, steps: [{ op: "mute" }], outputPath: "/o.mp4" }); + expect(graphOf(evened.args)).toBe("[0:v:0]crop=720:404:0:0,format=yuv420p[v1]"); + }); +}); diff --git a/packages/adapters/src/media/video-edit-plan.ts b/packages/adapters/src/media/video-edit-plan.ts new file mode 100644 index 0000000..f39f8d5 --- /dev/null +++ b/packages/adapters/src/media/video-edit-plan.ts @@ -0,0 +1,571 @@ +import type { EditVideoStep, MediaFit } from "@rakazo/contracts"; +import { MEDIA_PRESETS } from "@rakazo/contracts"; +import type { MediaProbe } from "./ffmpeg.js"; +import { ffmpegInputArgs, ffmpegSeconds } from "./ffmpeg.js"; + +/** + * Turns an ordered list of edit steps into one ffmpeg invocation with a single filter_complex + * pass. Pure: the caller probes the inputs and runs the arguments. + */ + +export interface PlanInput { + path: string; + mimeType: string; + name: string; + probe: MediaProbe; +} + +export interface VideoEditPlanRequest { + source: PlanInput; + steps: readonly EditVideoStep[]; + /** Videos named by concat steps, by artifact id. */ + clips?: ReadonlyMap; + /** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */ + subtitleFiles?: ReadonlyMap; + outputPath: string; +} + +export interface VideoEditPlan { + args: string[]; + kind: "video" | "audio"; + durationMs: number; + width?: number; + height?: number; + posterAtMs?: number; +} + +export const VIDEO_ENCODE_ARGS = [ + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "21", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", +]; +const AUDIO_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "160k"]; +const AUDIO_ONLY_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"]; +/** Location and other camera metadata stay private; chapters do not survive cuts anyway. */ +const STRIP_METADATA_ARGS = ["-map_metadata", "-1", "-map_chapters", "-1"]; +const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo"; +const SAFE_FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const MIN_SEGMENT_SECONDS = 0.1; +const FALLBACK_FPS = 30; + +interface Track { + label: string; + filters: string[]; +} + +interface Timeline { + /** Null when the result is audio only: video work is skipped. */ + video: Track | null; + audio: Track | null; + width: number; + height: number; + fps: number; + seconds: number; + /** How appended clips fill the frame: the last reframe's fit, otherwise bars. */ + fit: MediaFit; +} + +class FilterGraph { + private readonly chains: string[] = []; + private count = 0; + + label(kind: "v" | "a"): string { + this.count += 1; + return `${kind}${this.count}`; + } + + add(inputs: readonly string[], filters: readonly string[], outputs: readonly string[]) { + const pads = (labels: readonly string[]) => labels.map((label) => `[${label}]`).join(""); + this.chains.push(`${pads(inputs)}${filters.join(",")}${pads(outputs)}`); + } + + /** Ends a track's pending filters in a labelled pad other chains can consume. */ + flush(track: Track, kind: "v" | "a"): string { + if (track.filters.length === 0) return track.label; + const output = this.label(kind); + this.add([track.label], track.filters, [output]); + track.label = output; + track.filters = []; + return output; + } + + toString(): string { + return this.chains.join(";"); + } +} + +export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { + const { source } = request; + const video = source.probe.video; + if (!video) throw new Error(`${source.name} has no video.`); + const sourceSeconds = source.probe.durationMs / 1000; + const audioOnly = request.steps.at(-1)?.op === "extract_audio"; + const graph = new FilterGraph(); + const inputs: string[][] = []; + const addInput = (input: PlanInput, seek?: { start: number; end: number }) => { + inputs.push( + ffmpegInputArgs(input.path, { + mimeType: input.mimeType, + ...(seek ? { startSeconds: seek.start, durationSeconds: seek.end - seek.start } : {}), + }), + ); + return inputs.length - 1; + }; + + let steps = request.steps; + // A leading cut seeks the input instead of decoding everything before it. + const first = steps[0]; + let segments: Segment[] | undefined; + if (first?.op === "trim" || first?.op === "keep") { + segments = normalizeSegments( + first.op === "trim" ? [first] : first.segments, + sourceSeconds, + first.op, + ); + steps = steps.slice(1); + } + const seek = segments && { start: segments[0]!.start, end: segments.at(-1)!.end }; + const index = addInput(source, seek); + const size = displaySize(video); + const timeline: Timeline = { + video: audioOnly ? null : { label: `${index}:v:0`, filters: sourceFilters(video) }, + audio: source.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null, + width: size.width, + height: size.height, + fps: video.fps, + seconds: seek ? seek.end - seek.start : sourceSeconds, + fit: "pad", + }; + if (segments && seek && segments.length > 1) { + keepSegments( + graph, + timeline, + segments.map((segment) => ({ + start: segment.start - seek.start, + end: segment.end - seek.start, + })), + ); + } + + let posterAt: number | undefined; + for (const step of steps) { + switch (step.op) { + case "trim": + case "keep": + keepSegments( + graph, + timeline, + normalizeSegments(step.op === "trim" ? [step] : step.segments, timeline.seconds, step.op), + ); + break; + case "concat": + appendClips( + graph, + timeline, + step.artifact_ids.map((id) => { + const clip = request.clips?.get(id); + if (!clip) throw new Error(`Video ${id} is not available.`); + return { clip, index: addInput(clip) }; + }), + ); + break; + case "reframe": { + const preset = MEDIA_PRESETS[step.preset]; + if (timeline.video) { + timeline.video = fitFrame( + graph, + timeline.video, + timeline, + { width: preset.width, height: preset.height }, + step.fit, + ); + } + timeline.width = preset.width; + timeline.height = preset.height; + timeline.fit = step.fit; + break; + } + case "speed": + timeline.video?.filters.push( + `setpts=PTS/${number(step.factor)}`, + ...(timeline.fps ? [`fps=${number(timeline.fps)}`] : []), + ); + timeline.audio?.filters.push(...atempoFilters(step.factor)); + timeline.seconds /= step.factor; + break; + case "volume": + timeline.audio?.filters.push(`volume=${number(step.gain)}`); + break; + case "mute": + timeline.audio = null; + break; + case "subtitles": { + const file = request.subtitleFiles?.get(step.artifact_id); + if (!file) throw new Error(`Subtitles ${step.artifact_id} are not available.`); + // Only generated names reach the filter graph, so nothing here needs escaping. + if (!SAFE_FILE_NAME.test(file)) throw new Error("Unsafe subtitle file name."); + timeline.video?.filters.push( + `subtitles=filename=${file}:force_style='${subtitleStyle(timeline.width, timeline.height)}'`, + ); + break; + } + case "extract_audio": + break; + case "poster": + posterAt = step.at; + break; + } + } + + if (posterAt !== undefined && posterAt >= timeline.seconds) { + throw new Error( + `poster at ${seconds(posterAt)} s is past the end of the ${seconds(timeline.seconds)} s result.`, + ); + } + const durationMs = Math.round(timeline.seconds * 1000); + const output = `file:${request.outputPath}`; + if (audioOnly || !timeline.video) { + if (!timeline.audio) throw new Error(`${source.name} has no audio.`); + timeline.audio.filters.push(AUDIO_FORMAT); + const audio = graph.flush(timeline.audio, "a"); + return { + args: [ + ...inputs.flat(), + "-filter_complex", + graph.toString(), + "-map", + `[${audio}]`, + ...AUDIO_ONLY_ENCODE_ARGS, + ...STRIP_METADATA_ARGS, + "-f", + "mp4", + output, + ], + kind: "audio", + durationMs, + }; + } + timeline.video.filters.push("format=yuv420p"); + const videoOut = graph.flush(timeline.video, "v"); + let audioOut: string | undefined; + if (timeline.audio) { + timeline.audio.filters.push(AUDIO_FORMAT); + audioOut = graph.flush(timeline.audio, "a"); + } + return { + args: [ + ...inputs.flat(), + "-filter_complex", + graph.toString(), + "-map", + `[${videoOut}]`, + ...(audioOut ? ["-map", `[${audioOut}]`, ...AUDIO_ENCODE_ARGS] : []), + ...VIDEO_ENCODE_ARGS, + ...STRIP_METADATA_ARGS, + "-f", + "mp4", + output, + ], + kind: "video", + durationMs, + width: timeline.width, + height: timeline.height, + ...(posterAt !== undefined ? { posterAtMs: Math.round(posterAt * 1000) } : {}), + }; +} + +interface Segment { + start: number; + end: number; +} + +/** Validates ranges against the timeline, clamps ends to it, sorts and merges overlaps. */ +function normalizeSegments( + ranges: readonly Segment[], + duration: number, + op: "trim" | "keep", +): Segment[] { + const clamped = ranges.map((range) => { + if (range.start >= duration) { + throw new Error( + `${op} ${seconds(range.start)}–${seconds(range.end)} s is outside the ${seconds(duration)} s video.`, + ); + } + const end = Math.min(range.end, duration); + if (end - range.start < MIN_SEGMENT_SECONDS) { + throw new Error( + `${op} ${seconds(range.start)}–${seconds(end)} s is shorter than ${MIN_SEGMENT_SECONDS} s.`, + ); + } + return { start: range.start, end }; + }); + clamped.sort((a, b) => a.start - b.start); + const merged: Segment[] = []; + for (const segment of clamped) { + const last = merged.at(-1); + if (last && segment.start <= last.end) last.end = Math.max(last.end, segment.end); + else merged.push({ ...segment }); + } + return merged; +} + +/** Keeps sorted, disjoint segments of the timeline: one trim, or split + trims + concat. */ +function keepSegments(graph: FilterGraph, timeline: Timeline, segments: readonly Segment[]) { + const [only] = segments; + if (segments.length === 1 && only) { + const range = `start=${ffmpegSeconds(only.start)}:end=${ffmpegSeconds(only.end)}`; + timeline.video?.filters.push(`trim=${range}`, "setpts=PTS-STARTPTS"); + timeline.audio?.filters.push(`atrim=${range}`, "asetpts=PTS-STARTPTS"); + timeline.seconds = only.end - only.start; + return; + } + const count = segments.length; + const branch = (track: Track | null, kind: "v" | "a"): string[] => { + if (!track) return []; + const input = graph.flush(track, kind); + const splits = segments.map(() => graph.label(kind)); + graph.add([input], [`${kind === "v" ? "split" : "asplit"}=${count}`], splits); + return segments.map((segment, index) => { + const output = graph.label(kind); + const range = `start=${ffmpegSeconds(segment.start)}:end=${ffmpegSeconds(segment.end)}`; + graph.add( + [splits[index]!], + kind === "v" + ? [`trim=${range}`, "setpts=PTS-STARTPTS"] + : [`atrim=${range}`, "asetpts=PTS-STARTPTS"], + [output], + ); + return output; + }); + }; + const videoParts = branch(timeline.video, "v"); + const audioParts = branch(timeline.audio, "a"); + concatParts(graph, timeline, videoParts, audioParts, count); + timeline.seconds = segments.reduce((total, segment) => total + segment.end - segment.start, 0); +} + +/** Appends clips normalized to the timeline: frame size, fit, fps and 48 kHz stereo audio. */ +function appendClips( + graph: FilterGraph, + timeline: Timeline, + clips: readonly { clip: PlanInput; index: number }[], +) { + const fps = timeline.fps || FALLBACK_FPS; + const needsAudio = timeline.audio !== null || clips.some(({ clip }) => clip.probe.hasAudio); + const videoParts: string[] = []; + const audioParts: string[] = []; + if (timeline.video) { + timeline.video.filters.push("setsar=1", "format=yuv420p"); + videoParts.push(graph.flush(timeline.video, "v")); + } + if (needsAudio) audioParts.push(audioPart(graph, timeline.audio, timeline.seconds)); + let seconds = timeline.seconds; + for (const { clip, index } of clips) { + const video = clip.probe.video; + if (!video) throw new Error(`${clip.name} has no video.`); + const clipSeconds = clip.probe.durationMs / 1000; + if (clipSeconds <= 0) throw new Error(`Could not read the length of ${clip.name}.`); + if (timeline.video) { + const track = fitFrame( + graph, + { label: `${index}:v:0`, filters: sourceFilters(video) }, + displaySize(video), + timeline, + timeline.fit, + ); + track.filters.push(`fps=${number(fps)}`, "format=yuv420p"); + videoParts.push(graph.flush(track, "v")); + } + if (needsAudio) { + const track = clip.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null; + audioParts.push(audioPart(graph, track, clipSeconds)); + } + seconds += clipSeconds; + } + concatParts(graph, timeline, videoParts, audioParts, clips.length + 1); + timeline.seconds = seconds; +} + +/** A clip's audio in the common format, or silence as long as the clip when it has none. */ +function audioPart(graph: FilterGraph, track: Track | null, seconds: number): string { + if (track) { + track.filters.push(AUDIO_FORMAT); + return graph.flush(track, "a"); + } + const output = graph.label("a"); + graph.add( + [], + [ + "anullsrc=channel_layout=stereo:sample_rate=48000", + `atrim=end=${ffmpegSeconds(seconds)}`, + AUDIO_FORMAT, + ], + [output], + ); + return output; +} + +function concatParts( + graph: FilterGraph, + timeline: Timeline, + videoParts: readonly string[], + audioParts: readonly string[], + count: number, +) { + const inputs = Array.from({ length: count }, (_, index) => [ + ...(videoParts[index] ? [videoParts[index]] : []), + ...(audioParts[index] ? [audioParts[index]] : []), + ]).flat(); + const video = videoParts.length > 0 ? graph.label("v") : undefined; + const audio = audioParts.length > 0 ? graph.label("a") : undefined; + graph.add( + inputs, + [`concat=n=${count}:v=${video ? 1 : 0}:a=${audio ? 1 : 0}`], + [...(video ? [video] : []), ...(audio ? [audio] : [])], + ); + timeline.video = video ? { label: video, filters: [] } : null; + timeline.audio = audio ? { label: audio, filters: [] } : null; +} + +interface Size { + width: number; + height: number; +} + +/** Scales a frame of size `from` into `to`: center crop, bars, or a blurred copy behind it. */ +function fitFrame(graph: FilterGraph, track: Track, from: Size, to: Size, fit: MediaFit): Track { + const { width, height } = to; + if (Math.abs(from.width / from.height - width / height) < 0.01) { + track.filters.push(`scale=${width}:${height}`, "setsar=1"); + return track; + } + const contain = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`; + if (fit === "crop") { + track.filters.push( + `scale=${width}:${height}:force_original_aspect_ratio=increase`, + `crop=${width}:${height}`, + "setsar=1", + ); + return track; + } + if (fit === "pad") { + track.filters.push( + contain, + `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, + "setsar=1", + ); + return track; + } + // Blur a small copy and scale it up: as smooth as a large blur at a fraction of the cost. + const input = graph.flush(track, "v"); + const [background, foreground, blurred, contained, output] = [ + graph.label("v"), + graph.label("v"), + graph.label("v"), + graph.label("v"), + graph.label("v"), + ]; + const smallWidth = even(width / 4); + const smallHeight = even(height / 4); + const radius = Math.max(2, Math.floor(Math.min(smallWidth, smallHeight) / 10)); + graph.add([input], ["split=2"], [background, foreground]); + graph.add( + [background], + [ + `scale=${smallWidth}:${smallHeight}:force_original_aspect_ratio=increase`, + `crop=${smallWidth}:${smallHeight}`, + `boxblur=${radius}:2`, + `scale=${width}:${height}`, + "setsar=1", + ], + [blurred], + ); + graph.add([foreground], [contain, "setsar=1"], [contained]); + graph.add([blurred, contained], ["overlay=(W-w)/2:(H-h)/2"], [output]); + return { label: output, filters: [] }; +} + +/** Square pixels and even dimensions, which yuv420p H.264 requires. */ +function sourceFilters(video: NonNullable): string[] { + const size = displaySize(video); + if (Math.abs(video.sampleAspectRatio - 1) > 0.01) { + return [`scale=${size.width}:${size.height}`, "setsar=1"]; + } + if (size.width !== video.width || size.height !== video.height) { + return [`crop=${size.width}:${size.height}:0:0`]; + } + return []; +} + +function displaySize(video: NonNullable): Size { + if (Math.abs(video.sampleAspectRatio - 1) > 0.01) { + return { + width: even(Math.round(video.width * video.sampleAspectRatio)), + height: even(video.height), + }; + } + return { width: even(video.width), height: even(video.height) }; +} + +/** atempo takes 0.5–2 per instance on older ffmpeg builds, so larger changes chain it. */ +export function atempoFilters(factor: number): string[] { + const filters: string[] = []; + let rest = factor; + while (rest > 2) { + filters.push("atempo=2"); + rest /= 2; + } + while (rest < 0.5) { + filters.push("atempo=0.5"); + rest /= 0.5; + } + filters.push(`atempo=${number(rest)}`); + return filters; +} + +/** + * Readable burned-in subtitles: bold white text with a dark outline, sized to the frame, and + * lifted above the bottom fifth of tall frames where 9:16 apps place their own controls. + */ +export function subtitleStyle(width: number, height: number): string { + const tall = height >= width * 1.5; + const fontSize = Math.round(Math.min(width, height) * 0.06); + const marginX = Math.round(width * (tall ? 0.1 : 0.05)); + const marginY = Math.round(height * (tall ? 0.2 : 0.06)); + return [ + `PlayResX=${width}`, + `PlayResY=${height}`, + "FontName=DejaVu Sans", + `FontSize=${fontSize}`, + "Bold=1", + "PrimaryColour=&H00FFFFFF", + "OutlineColour=&H00000000", + "BackColour=&H80000000", + "BorderStyle=1", + `Outline=${Math.max(2, Math.round(fontSize / 16))}`, + "Shadow=1", + "Alignment=2", + `MarginL=${marginX}`, + `MarginR=${marginX}`, + `MarginV=${marginY}`, + ].join(","); +} + +function even(value: number): number { + return Math.max(2, 2 * Math.floor(value / 2)); +} + +function number(value: number): string { + return String(Number(value.toFixed(6))); +} + +function seconds(value: number): string { + return String(Number(value.toFixed(2))); +} diff --git a/packages/adapters/src/media/video-operations.postgres.test.ts b/packages/adapters/src/media/video-operations.postgres.test.ts new file mode 100644 index 0000000..7f5592d --- /dev/null +++ b/packages/adapters/src/media/video-operations.postgres.test.ts @@ -0,0 +1,214 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { JobPublisher } from "@rakazo/adapter-kit"; +import type { MediaJobBlock, MessageBlock } from "@rakazo/contracts"; +import type { PrismaClient } from "@rakazo/db"; +import { createDb } from "@rakazo/db"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { storeArtifactFromFile } from "../artifact-files.js"; +import { LocalArtifactStore } from "../artifacts.js"; +import { createMediaRuntime, MediaJobRejectedError } from "../media-runtime.js"; +import { detectFfmpeg } from "./ffmpeg.js"; +import type { TestClips } from "./media-test-support.js"; +import { createTestClips } from "./media-test-support.js"; +import { createVideoOperations } from "./video-operations.js"; + +// CI runners may not have ffmpeg; the suite runs wherever both it and Postgres are available. +const describePostgres = + process.env.VERIFY_DATABASE && process.env.DATABASE_URL && (await detectFfmpeg()) + ? describe.sequential + : describe.skip; + +describePostgres("video operations in the media runtime (PostgreSQL + ffmpeg)", () => { + let db: ReturnType; + let prisma: PrismaClient; + let root: string; + let clips: TestClips; + const users: string[] = []; + + beforeAll(async () => { + db = createDb(process.env.DATABASE_URL!); + prisma = db.prisma; + root = await mkdtemp(join(tmpdir(), "video-operations-pg-")); + clips = await createTestClips(); + }); + + afterAll(async () => { + if (!db) return; + await prisma.mediaJob.deleteMany({ where: { userId: { in: users } } }); + await prisma.artifact.deleteMany({ where: { userId: { in: users } } }); + await prisma.organization.deleteMany({ where: { id: { in: users } } }); + await prisma.user.deleteMany({ where: { id: { in: users } } }); + await prisma.$disconnect(); + await db.pool.end(); + await rm(root, { recursive: true, force: true }); + await rm(clips.dir, { recursive: true, force: true }); + }); + + async function setup() { + const id = randomUUID(); + users.push(id); + await prisma.user.create({ + data: { id, name: "Video test", email: `${id}@example.test`, emailVerified: true }, + }); + await prisma.organization.create({ + data: { id, name: "Video test", slug: id, createdAt: new Date() }, + }); + await prisma.space.create({ data: { id, organizationId: id, name: "Media", isDefault: true } }); + await prisma.member.create({ + data: { id, organizationId: id, userId: id, role: "owner", createdAt: new Date() }, + }); + const bot = await prisma.bot.create({ + data: { spaceId: id, userId: id, name: "Studio", color: "test-color" }, + }); + const thread = await prisma.thread.create({ data: { spaceId: id, userId: id, botId: bot.id } }); + const jobs: JobPublisher = { + enqueue: async () => undefined, + cancel: async () => undefined, + close: async () => undefined, + }; + const artifacts = new LocalArtifactStore(join(root, id, "artifacts")); + const runtime = createMediaRuntime( + { + prisma, + events: { notify: async () => undefined }, + jobs, + artifacts, + registrations: createVideoOperations(), + }, + { tempRoot: join(root, id, "jobs") }, + ); + const scope = { + spaceId: id, + userId: id, + botId: bot.id, + groupId: null, + threadId: thread.id, + runId: null, + }; + const store = async (path: string, name: string, mimeType: string) => + ( + await storeArtifactFromFile( + { prisma, artifacts }, + { + path, + name, + mimeType, + spaceId: id, + userId: id, + botId: bot.id, + groupId: null, + runId: null, + }, + ) + ).id; + return { runtime, scope, store }; + } + + async function card(messageId: string, jobId: string): Promise { + const message = await prisma.message.findUniqueOrThrow({ where: { id: messageId } }); + return (message.blocks as MessageBlock[]).find( + (block): block is MediaJobBlock => block.kind === "media" && block.jobId === jobId, + ); + } + + it("edits a stored video into a linked 9:16 version with a poster on the card", async () => { + const { runtime, scope, store } = await setup(); + const source = await store(clips.landscape, "holiday.mp4", "video/mp4"); + const subtitles = await store(clips.srt, "holiday.srt", "application/x-subrip"); + const started = await runtime.start({ + ...scope, + operation: "edit_video", + input: { + artifact_id: source, + title: "Urlaub 9:16", + steps: [ + { op: "trim", start: 0.5, end: 2.5 }, + { op: "reframe", preset: "vertical", fit: "blur" }, + { op: "subtitles", artifact_id: subtitles }, + ], + }, + }); + await runtime.runJob(started.jobId); + const job = await runtime.get(started.jobId); + expect(job).toMatchObject({ status: "done", title: "Urlaub 9:16", progress: 1 }); + const [output] = job!.outputs; + expect(output).toMatchObject({ + kind: "video", + mimeType: "video/mp4", + name: "Urlaub 9 16.mp4", + width: 1080, + height: 1920, + parentArtifactId: source, + }); + expect(output?.posterArtifactId).toBeDefined(); + const poster = await prisma.artifact.findUniqueOrThrow({ + where: { id: output!.posterArtifactId! }, + }); + expect(poster.mimeType).toBe("image/jpeg"); + const stored = await prisma.artifact.findUniqueOrThrow({ where: { id: output!.artifactId } }); + expect(stored).toMatchObject({ parentArtifactId: source, width: 1080, height: 1920 }); + expect(Math.abs((stored.durationMs ?? 0) - 2000)).toBeLessThan(150); + expect(await card(started.messageId, started.jobId)).toMatchObject({ + status: "done", + outputs: job?.outputs, + }); + }); + + it("exports one stored version per preset", async () => { + const { runtime, scope, store } = await setup(); + const source = await store(clips.landscape, "holiday.mp4", "video/mp4"); + const started = await runtime.start({ + ...scope, + operation: "export_video", + input: { artifact_id: source, presets: ["square", "portrait"] }, + }); + await runtime.runJob(started.jobId); + const job = await runtime.get(started.jobId); + expect(job).toMatchObject({ status: "done", title: "1:1 · 4:5" }); + expect( + job?.outputs.map((output) => [output.name, output.preset, output.width, output.height]), + ).toEqual([ + ["holiday-1x1.mp4", "square", 1080, 1080], + ["holiday-4x5.mp4", "portrait", 1080, 1350], + ]); + }); + + it("refuses appended clips and subtitles the user cannot read", async () => { + const { runtime, scope, store } = await setup(); + const other = await setup(); + const source = await store(clips.landscape, "mine.mp4", "video/mp4"); + const foreignClip = await other.store(clips.silentPortrait, "theirs.mp4", "video/mp4"); + const foreignSubtitles = await other.store(clips.srt, "theirs.srt", "application/x-subrip"); + for (const step of [ + { op: "concat", artifact_ids: [foreignClip] }, + { op: "subtitles", artifact_id: foreignSubtitles }, + ]) { + await expect( + runtime.start({ + ...scope, + operation: "edit_video", + input: { artifact_id: source, title: "x", steps: [step] }, + }), + ).rejects.toThrow(MediaJobRejectedError); + } + }); + + it("fails the job with a readable error when a range is outside the video", async () => { + const { runtime, scope, store } = await setup(); + const source = await store(clips.landscape, "short.mp4", "video/mp4"); + const started = await runtime.start({ + ...scope, + operation: "edit_video", + input: { artifact_id: source, title: "x", steps: [{ op: "trim", start: 10, end: 20 }] }, + }); + await runtime.runJob(started.jobId); + expect(await runtime.get(started.jobId)).toMatchObject({ + status: "failed", + error: "trim 10–20 s is outside the 3 s video.", + outputs: [], + }); + }); +}); diff --git a/packages/adapters/src/media/video-operations.test.ts b/packages/adapters/src/media/video-operations.test.ts new file mode 100644 index 0000000..1d8d6f9 --- /dev/null +++ b/packages/adapters/src/media/video-operations.test.ts @@ -0,0 +1,289 @@ +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { builtinAgentTools } from "../builtin-tools.js"; +import { createMediaOperations } from "../media-operations.js"; +import { MEDIA_CONTINUE_AFTER_PROPERTY } from "../media-tools.js"; +import { parametersFor } from "../pi-runtime.js"; +import { detectFfmpeg, probeMedia, runFfmpeg } from "./ffmpeg.js"; +import type { TestArtifact, TestClips } from "./media-test-support.js"; +import { createTestClips, fakeMediaJobContext } from "./media-test-support.js"; +import { cleanFileName, createVideoOperations, subtitleFormat } from "./video-operations.js"; + +const ffmpeg = await detectFfmpeg(); + +function operation(name: "edit_video" | "export_video", deps = {}) { + const registration = createVideoOperations(deps).find((entry) => entry.operation === name); + if (!registration) throw new Error(`missing ${name}`); + return registration; +} + +describe("video operation registrations", () => { + it("are registered for the runtime and unavailable without ffmpeg", async () => { + const names = createMediaOperations({ loadSettings: async () => ({}) as never }).map( + (entry) => entry.operation, + ); + expect(names).toEqual(expect.arrayContaining(["edit_video", "export_video"])); + const edit = operation("edit_video", { detect: async () => null }); + expect(await edit.available({ spaceId: "space-1" })).toBe(false); + const { context } = await fakeMediaJobContext({}, {}); + await expect(edit.handler.run(context)).rejects.toThrow( + "ffmpeg and ffprobe are not installed on this server.", + ); + }); + + it("names every artifact the input reads so start() can authorize them", () => { + const edit = operation("edit_video"); + const input = edit.inputSchema.parse({ + artifact_id: "src", + title: "Reel", + steps: [ + { op: "concat", artifact_ids: ["clip-1", "clip-2"] }, + { op: "subtitles", artifact_id: "subs" }, + ], + }); + expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs"]); + const exported = operation("export_video"); + const exportInput = exported.inputSchema.parse({ artifact_id: "src", presets: ["vertical"] }); + expect(exported.artifactIds?.(exportInput)).toEqual(["src"]); + }); + + it("titles exports by aspect ratio unless the bot gave a title", () => { + const exported = operation("export_video"); + expect( + exported.title( + exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical", "square"] }), + ), + ).toBe("9:16 · 1:1"); + expect( + exported.title( + exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical"], title: "Reel" }), + ), + ).toBe("Reel"); + }); + + it("defines tools named like the operations, with continue_after, that the runtime can encode", () => { + for (const name of ["edit_video", "export_video"]) { + const tool = builtinAgentTools.find((entry) => entry.name === name); + expect(tool).toBeDefined(); + expect(tool?.inputSchema.properties).toMatchObject(MEDIA_CONTINUE_AFTER_PROPERTY); + expect(() => parametersFor(tool!)).not.toThrow(); + } + }); +}); + +describe("subtitle and file name helpers", () => { + it("recognizes SRT and VTT by content only", () => { + expect(subtitleFormat("WEBVTT\n\n00:00.000 --> 00:01.000\nHi")).toBe("vtt"); + expect(subtitleFormat("1\r\n00:00:01,000 --> 00:00:02,000\r\nHi")).toBe("srt"); + expect(subtitleFormat("00:00:01.000 --> 00:00:02.000\nHi")).toBe("srt"); + expect(subtitleFormat("#EXTM3U\n#EXTINF:1,\nfile:/etc/passwd")).toBeNull(); + expect(subtitleFormat("WEBVTTX")).toBeNull(); + }); + + it("cleans file names without losing the user's language", () => { + expect(cleanFileName("Reel: Sommer/Urlaub 2026", "video")).toBe("Reel Sommer Urlaub 2026"); + expect(cleanFileName("../..\u0000", "video")).toBe("video"); + expect(cleanFileName("x".repeat(200), "video")).toHaveLength(80); + }); +}); + +// CI runners may not have ffmpeg; these run wherever it is installed. +describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => { + let clips: TestClips; + let artifacts: Record; + + beforeAll(async () => { + clips = await createTestClips(); + artifacts = { + land: { path: clips.landscape, name: "Landscape clip.mp4", mimeType: "video/mp4" }, + silent: { path: clips.silentPortrait, name: "silent.mp4", mimeType: "video/mp4" }, + srt: { path: clips.srt, name: "captions.srt", mimeType: "application/x-subrip" }, + vtt: { path: clips.vtt, name: "captions.vtt", mimeType: "text/vtt" }, + notes: { path: clips.landscape, name: "notes.txt", mimeType: "text/plain" }, + }; + }); + + afterAll(async () => { + if (clips) await rm(clips.dir, { recursive: true, force: true }); + }); + + async function run(name: "edit_video" | "export_video", input: Record) { + const registration = operation(name); + const job = await fakeMediaJobContext(registration.inputSchema.parse(input), artifacts); + try { + const result = await registration.handler.run(job.context); + return { ...job, result }; + } finally { + await rm(job.tempDir, { recursive: true, force: true }); + } + } + + it("cuts, reframes to 9:16 with a blurred fill, burns in subtitles and speeds up", async () => { + const { outputs, progress, result } = await run("edit_video", { + artifact_id: "land", + title: "Sommer-Reel", + steps: [ + { op: "trim", start: 0.5, end: 2.5 }, + { op: "reframe", preset: "vertical", fit: "blur" }, + { op: "subtitles", artifact_id: "srt" }, + { op: "speed", factor: 2 }, + { op: "poster", at: 0.5 }, + ], + }); + expect(outputs).toHaveLength(1); + const [output] = outputs; + expect(output).toMatchObject({ + name: "Sommer-Reel.mp4", + mimeType: "video/mp4", + width: 1080, + height: 1920, + parentArtifactId: "land", + }); + expect(output!.posterSize).toBeGreaterThan(0); + expect(Math.abs(output!.durationMs! - 1000)).toBeLessThan(150); + expect(progress.at(-1)).toBe(1); + expect(progress).toEqual([...progress].sort((a, b) => a - b)); + expect(result.summary).toContain("Sommer-Reel.mp4 (1080×1920, "); + }); + + it("keeps segments, appends a silent clip with generated silence, and changes volume", async () => { + const registration = operation("edit_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ + artifact_id: "land", + title: "Cut", + steps: [ + { + op: "keep", + segments: [ + { start: 0, end: 0.5 }, + { start: 2, end: 2.8 }, + ], + }, + { op: "concat", artifact_ids: ["silent"] }, + { op: "volume", gain: 0.5 }, + { op: "subtitles", artifact_id: "vtt" }, + ], + }), + artifacts, + ); + await registration.handler.run(job.context); + const [output] = job.outputs; + const probe = await probeMedia(output!.path, { mimeType: "video/mp4" }); + expect(probe.hasAudio).toBe(true); + expect(probe.video).toMatchObject({ width: 320, height: 180, rotation: 0 }); + expect(Math.abs(probe.durationMs - 3300)).toBeLessThan(200); + await rm(job.tempDir, { recursive: true, force: true }); + }); + + it("plays rotated phone video upright without rotating it twice", async (context) => { + const rotated = join(clips.dir, "rotated.mp4"); + try { + await runFfmpeg([ + ...["-display_rotation", "90", "-i", `file:${clips.landscape}`], + ...["-c", "copy", `file:${rotated}`], + ]); + } catch { + context.skip("this ffmpeg cannot write rotation metadata"); + } + const registration = operation("edit_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ + artifact_id: "rotated", + title: "Upright", + steps: [{ op: "trim", start: 0, end: 1 }], + }), + { rotated: { path: rotated, name: "rotated.mp4", mimeType: "video/mp4" } }, + ); + await registration.handler.run(job.context); + const probe = await probeMedia(job.outputs[0]!.path, { mimeType: "video/mp4" }); + expect(probe.video).toMatchObject({ width: 180, height: 320, rotation: 0 }); + await rm(job.tempDir, { recursive: true, force: true }); + }); + + it("extracts the audio as M4A", async () => { + const { outputs } = await run("edit_video", { + artifact_id: "land", + title: "Ton", + steps: [{ op: "trim", start: 1, end: 2 }, { op: "extract_audio" }], + }); + expect(outputs).toEqual([ + expect.objectContaining({ name: "Ton.m4a", mimeType: "audio/mp4", parentArtifactId: "land" }), + ]); + expect(outputs[0]?.posterPath).toBeUndefined(); + expect(Math.abs(outputs[0]!.durationMs! - 1000)).toBeLessThan(100); + }); + + it("saves a single frame when poster is the only step", async () => { + const { outputs } = await run("edit_video", { + artifact_id: "land", + title: "Cover", + steps: [{ op: "poster", at: 1 }], + }); + expect(outputs).toEqual([ + expect.objectContaining({ + name: "Cover.jpg", + mimeType: "image/jpeg", + width: 320, + height: 180, + }), + ]); + }); + + it("exports one output per distinct preset with posters and batch progress", async () => { + const { outputs, progress } = await run("export_video", { + artifact_id: "land", + presets: ["square", "portrait", "square"], + fit: "pad", + }); + expect( + outputs.map((output) => [output.name, output.preset, output.width, output.height]), + ).toEqual([ + ["Landscape clip-1x1.mp4", "square", 1080, 1080], + ["Landscape clip-4x5.mp4", "portrait", 1080, 1350], + ]); + for (const output of outputs) { + expect(output.parentArtifactId).toBe("land"); + expect(output.posterSize).toBeGreaterThan(0); + } + expect(progress).toContain(0.5); + expect(progress.at(-1)).toBe(1); + }); + + it("fails clearly on ranges past the end, non-subtitle files and non-videos", async () => { + await expect( + run("edit_video", { + artifact_id: "land", + title: "x", + steps: [{ op: "trim", start: 5, end: 6 }], + }), + ).rejects.toThrow("trim 5–6 s is outside the 3 s video."); + await expect( + run("edit_video", { + artifact_id: "land", + title: "x", + steps: [{ op: "subtitles", artifact_id: "land" }], + }), + ).rejects.toThrow("Landscape clip.mp4 is not an SRT or VTT file."); + await expect( + run("export_video", { artifact_id: "notes", presets: ["square"] }), + ).rejects.toThrow("notes.txt is not a video."); + }); + + it("stops ffmpeg and stores nothing when the job is cancelled", async () => { + const registration = operation("export_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ artifact_id: "land", presets: ["cinema"], fit: "blur" }), + artifacts, + ); + const progress = job.context.progress; + job.context.progress = async (fraction) => { + await progress(fraction); + job.controller.abort(new Error("cancelled")); + }; + await expect(registration.handler.run(job.context)).rejects.toThrow("cancelled"); + expect(job.outputs).toEqual([]); + await rm(job.tempDir, { recursive: true, force: true }); + }); +}); diff --git a/packages/adapters/src/media/video-operations.ts b/packages/adapters/src/media/video-operations.ts new file mode 100644 index 0000000..e4e1e6d --- /dev/null +++ b/packages/adapters/src/media/video-operations.ts @@ -0,0 +1,324 @@ +import { copyFile, mkdtemp, open, stat } from "node:fs/promises"; +import { join } from "node:path"; +import type { MediaJobContext, MediaJobInputFile, MediaOperationResult } from "@rakazo/adapter-kit"; +import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts"; +import { + ARTIFACT_MAX_BYTES_BY_KIND, + EditVideoInputSchema, + ExportVideoInputSchema, + MEDIA_PRESETS, +} from "@rakazo/contracts"; +import type { MediaOperationRegistration } from "../media-runtime.js"; +import type { FfmpegInfo } from "./ffmpeg.js"; +import { + detectFfmpeg, + outputTooLargeMessage, + probeMedia, + renderPoster, + runFfmpeg, +} from "./ffmpeg.js"; +import type { PlanInput, VideoEditPlan } from "./video-edit-plan.js"; +import { planVideoEdit } from "./video-edit-plan.js"; + +export interface VideoOperationsDeps { + /** Finds ffmpeg; defaults to PATH lookup. */ + detect?: () => Promise; +} + +/** Share of each output's progress spent encoding; the rest is probing and the poster. */ +const ENCODE_SHARE = 0.95; +const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024; + +/** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */ +export function createVideoOperations( + deps: VideoOperationsDeps = {}, +): MediaOperationRegistration[] { + const detect = deps.detect ?? detectFfmpeg; + const available = async () => (await detect()) !== null; + const requireTools = async () => { + if (!(await available())) + throw new Error("ffmpeg and ffprobe are not installed on this server."); + }; + const editVideo: MediaOperationRegistration = { + operation: "edit_video", + inputSchema: EditVideoInputSchema, + title: (input) => input.title, + available, + artifactIds: (input) => [ + input.artifact_id, + ...input.steps.flatMap((step) => + step.op === "concat" + ? step.artifact_ids + : step.op === "subtitles" + ? [step.artifact_id] + : [], + ), + ], + handler: { + operation: "edit_video", + run: async (context) => { + await requireTools(); + return runEditVideo(context); + }, + }, + }; + const exportVideo: MediaOperationRegistration = { + operation: "export_video", + inputSchema: ExportVideoInputSchema, + title: (input) => + input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "), + available, + artifactIds: (input) => [input.artifact_id], + handler: { + operation: "export_video", + run: async (context) => { + await requireTools(); + return runExportVideo(context); + }, + }, + }; + return [editVideo, exportVideo]; +} + +async function runEditVideo(context: MediaJobContext): Promise { + const input = EditVideoInputSchema.parse(context.input); + const workDir = await mkdtemp(join(context.tempDir, "edit-")); + const progress = createProgressReporter(context); + const source = await loadVideo(context, input.artifact_id); + const name = cleanFileName(input.title, cleanFileName(withoutExtension(source.name), "video")); + + const [first] = input.steps; + if (input.steps.length === 1 && first?.op === "poster") { + const at = first.at; + if (at * 1000 >= source.probe.durationMs) { + throw new Error( + `poster at ${at} s is past the end of the ${source.probe.durationMs / 1000} s video.`, + ); + } + const posterPath = join(workDir, "poster.jpg"); + await renderPoster(source.path, posterPath, { + signal: context.signal, + mimeType: source.mimeType, + atMs: at * 1000, + }); + const output = await context.addOutput({ + path: posterPath, + name: `${name}.jpg`, + mimeType: "image/jpeg", + width: source.probe.video?.width, + height: source.probe.video?.height, + parentArtifactId: source.artifactId, + }); + await progress.done(); + return { summary: `Saved the frame at ${at} s of ${source.name} as ${describe(output)}.` }; + } + + const clips = new Map(); + const subtitleFiles = new Map(); + for (const step of input.steps) { + if (step.op === "concat") { + for (const id of step.artifact_ids) { + if (!clips.has(id)) clips.set(id, await loadVideo(context, id)); + } + } else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) { + subtitleFiles.set( + step.artifact_id, + await prepareSubtitles(context, step.artifact_id, workDir, subtitleFiles.size), + ); + } + } + const audioOnly = input.steps.at(-1)?.op === "extract_audio"; + const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4"); + const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath }); + const output = await encode(context, plan, outputPath, workDir, progress.range(0, 1), { + name: `${name}${audioOnly ? ".m4a" : ".mp4"}`, + parentArtifactId: source.artifactId, + }); + await progress.done(); + return { + summary: `Edited ${source.name} into ${describe(output)}.`, + }; +} + +async function runExportVideo(context: MediaJobContext): Promise { + const input = ExportVideoInputSchema.parse(context.input); + const presets = [...new Set(input.presets)]; + const workDir = await mkdtemp(join(context.tempDir, "export-")); + const progress = createProgressReporter(context); + const source = await loadVideo(context, input.artifact_id); + const base = cleanFileName(withoutExtension(source.name), "video"); + const outputs: MediaOutput[] = []; + for (const [index, preset] of presets.entries()) { + const outputPath = join(workDir, `${preset}.mp4`); + const plan = planVideoEdit({ + source, + steps: [{ op: "reframe", preset, fit: input.fit }], + outputPath, + }); + const share = 1 / presets.length; + outputs.push( + await encode( + context, + plan, + outputPath, + workDir, + progress.range(index * share, (index + 1) * share), + { + name: `${base}-${MEDIA_PRESETS[preset].aspect.replace(":", "x")}.mp4`, + parentArtifactId: source.artifactId, + preset, + }, + ), + ); + } + await progress.done(); + return { summary: `Exported ${source.name} as ${outputs.map(describe).join(", ")}.` }; +} + +/** Runs a plan, checks the result, renders its poster and stores it as an output. */ +async function encode( + context: MediaJobContext, + plan: VideoEditPlan, + outputPath: string, + workDir: string, + report: (fraction: number) => void, + file: Pick[0], "name" | "parentArtifactId" | "preset">, +): Promise { + const maxBytes = ARTIFACT_MAX_BYTES_BY_KIND[plan.kind]; + await runFfmpeg(plan.args, { + signal: context.signal, + cwd: workDir, + durationMs: plan.durationMs, + onProgress: (fraction) => report(fraction * ENCODE_SHARE), + maxOutputBytes: maxBytes, + outputs: [outputPath], + }); + if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes)); + const probe = await probeMedia(outputPath, { + signal: context.signal, + mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4", + }); + if (plan.kind === "audio") { + const output = await context.addOutput({ + ...file, + path: outputPath, + mimeType: "audio/mp4", + durationMs: probe.durationMs, + }); + report(1); + return output; + } + const posterPath = outputPath.replace(/\.mp4$/, ".jpg"); + await renderPoster(outputPath, posterPath, { + signal: context.signal, + mimeType: "video/mp4", + atMs: plan.posterAtMs ?? Math.min(1_000, probe.durationMs / 4), + }); + const output = await context.addOutput({ + ...file, + path: outputPath, + mimeType: "video/mp4", + width: probe.video?.width ?? plan.width, + height: probe.video?.height ?? plan.height, + durationMs: probe.durationMs, + posterPath, + }); + report(1); + return output; +} + +async function loadVideo( + context: MediaJobContext, + artifactId: string, +): Promise { + const file = await context.fetchArtifact(artifactId); + if (file.kind !== "video") throw new Error(`${file.name} is not a video.`); + const probe = await probeMedia(file.path, { + signal: context.signal, + mimeType: file.mimeType, + }); + if (!probe.video) throw new Error(`${file.name} has no video.`); + if (probe.durationMs <= 0) throw new Error(`Could not read the length of ${file.name}.`); + return { ...file, probe }; +} + +/** Copies an SRT/VTT artifact to a generated name, so no user text reaches the filter graph. */ +async function prepareSubtitles( + context: MediaJobContext, + artifactId: string, + workDir: string, + index: number, +): Promise { + const file = await context.fetchArtifact(artifactId); + const format = file.size <= SUBTITLE_MAX_BYTES ? subtitleFormat(await readHead(file.path)) : null; + if (!format) throw new Error(`${file.name} is not an SRT or VTT file.`); + const name = `subtitles-${index}.${format}`; + await copyFile(file.path, join(workDir, name)); + return name; +} + +async function readHead(path: string): Promise { + const handle = await open(path, "r"); + try { + const buffer = Buffer.alloc(1024); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + return buffer.subarray(0, bytesRead).toString("utf8"); + } finally { + await handle.close(); + } +} + +/** + * Recognizes WebVTT and SubRip by content. Anything else is refused, so ffmpeg never probes + * an arbitrary file (a playlist, say) handed in as subtitles. + */ +export function subtitleFormat(head: string): "srt" | "vtt" | null { + const text = head.replace(/^/, "").trimStart(); + if (/^WEBVTT(?:[ \t\r\n]|$)/.test(text)) return "vtt"; + if (/^(?:\d+[ \t]*\r?\n)?\d{1,2}:\d{2}:\d{2}[,.]\d{1,3}[ \t]*-->/.test(text)) return "srt"; + return null; +} + +/** A name for produced files: no path separators or control characters, at most 80 chars. */ +export function cleanFileName(text: string, fallback: string): string { + const name = text + .replaceAll(/[\p{Cc}\\/:*?"<>|]+/gu, " ") + .replaceAll(/\s+/g, " ") + .replace(/^[\s.]+/, "") + .slice(0, 80) + .trim(); + return name || fallback; +} + +function withoutExtension(name: string): string { + return name.replace(/\.[A-Za-z0-9]{1,5}$/, ""); +} + +/** "name (1080×1920, 12.5 s)" for the bot's summary; the runtime adds the artifact ids. */ +function describe(output: MediaOutput): string { + const details = [ + output.width && output.height ? `${output.width}×${output.height}` : "", + output.durationMs !== undefined ? `${Number((output.durationMs / 1000).toFixed(1))} s` : "", + ].filter(Boolean); + return details.length ? `${output.name} (${details.join(", ")})` : output.name; +} + +/** Maps ffmpeg progress onto the card: monotonic, written in order, never failing the job. */ +function createProgressReporter(context: MediaJobContext) { + let reported = 0; + let writes = Promise.resolve(); + const write = (fraction: number) => { + reported = fraction; + writes = writes.then(() => context.progress(fraction)).catch(() => undefined); + }; + return { + range: (from: number, to: number) => (fraction: number) => { + const value = Math.min(1, from + (to - from) * Math.min(1, Math.max(0, fraction))); + if (value > reported) write(value); + }, + done: async () => { + if (reported < 1) write(1); + await writes; + }, + }; +} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index e1982e1..ec03561 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -13,6 +13,7 @@ export * from "./media.js"; export * from "./media-generation.js"; export * from "./media-jobs.js"; export * from "./media-settings.js"; +export * from "./media-video.js"; export * from "./model-tiers.js"; export * from "./openai-compatible-ui.js"; export * from "./project-goals.js"; diff --git a/packages/contracts/src/media-generation.ts b/packages/contracts/src/media-generation.ts index 2e61c42..cbec033 100644 --- a/packages/contracts/src/media-generation.ts +++ b/packages/contracts/src/media-generation.ts @@ -1,6 +1,7 @@ import * as z from "zod"; import { Id } from "./ids.js"; import { MediaFitSchema, MediaPresetIdSchema } from "./media.js"; +import { MediaTitleSchema } from "./media-video.js"; export const MEDIA_IMAGE_MAX_COUNT = 4; export const MEDIA_VIDEO_MAX_SECONDS = 10; @@ -55,6 +56,9 @@ export type ImageFormat = z.infer; export const EditImageInputSchema = z .object({ artifactId: Id.describe("Image artifact to edit."), + title: MediaTitleSchema.optional().describe( + "A few words in the user's language for the card and file name.", + ), crop: z .object({ left: Pixels, top: Pixels, width: Pixels.min(1), height: Pixels.min(1) }) .optional() diff --git a/packages/contracts/src/media-video.test.ts b/packages/contracts/src/media-video.test.ts new file mode 100644 index 0000000..eb3fc17 --- /dev/null +++ b/packages/contracts/src/media-video.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { EditVideoInputSchema, ExportVideoInputSchema } from "./media-video.js"; + +const edit = (steps: unknown[]) => + EditVideoInputSchema.safeParse({ artifact_id: "a", title: "Reel", steps }); + +describe("edit_video input", () => { + it("accepts ordered steps and fills defaults", () => { + const parsed = EditVideoInputSchema.parse({ + artifact_id: "a", + title: " Reel ", + steps: [ + { op: "trim", start: 1, end: 5 }, + { op: "reframe", preset: "vertical" }, + { op: "extract_audio" }, + ], + }); + expect(parsed.title).toBe("Reel"); + expect(parsed.steps[1]).toEqual({ op: "reframe", preset: "vertical", fit: "crop" }); + // The job stores the parsed input and parses it again when it runs. + expect(EditVideoInputSchema.parse(parsed)).toEqual(parsed); + }); + + it("rejects malformed step lists", () => { + expect(edit([]).success).toBe(false); + expect(edit([{ op: "trim", start: 5, end: 5 }]).success).toBe(false); + expect(edit([{ op: "keep", segments: [{ start: 3, end: 1 }] }]).success).toBe(false); + expect(edit([{ op: "speed", factor: 8 }]).success).toBe(false); + expect(edit([{ op: "extract_audio" }, { op: "mute" }]).success).toBe(false); + expect(edit([{ op: "poster", at: 1 }, { op: "extract_audio" }]).success).toBe(false); + expect( + edit([ + { op: "poster", at: 1 }, + { op: "poster", at: 2 }, + ]).success, + ).toBe(false); + expect(edit([{ op: "blur" }]).success).toBe(false); + expect( + EditVideoInputSchema.safeParse({ artifact_id: "a", steps: [{ op: "mute" }] }).success, + ).toBe(false); + }); +}); + +describe("export_video input", () => { + it("needs at least one preset and defaults to cropping", () => { + expect(ExportVideoInputSchema.parse({ artifact_id: "a", presets: ["square"] })).toEqual({ + artifact_id: "a", + presets: ["square"], + fit: "crop", + }); + expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: [] }).success).toBe(false); + expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: ["imax"] }).success).toBe( + false, + ); + }); +}); diff --git a/packages/contracts/src/media-video.ts b/packages/contracts/src/media-video.ts new file mode 100644 index 0000000..a12bf51 --- /dev/null +++ b/packages/contracts/src/media-video.ts @@ -0,0 +1,80 @@ +import * as z from "zod"; +import { Id } from "./ids.js"; +import { MediaFitSchema, MediaPresetIdSchema } from "./media.js"; + +/** + * Inputs of the `edit_video` and `export_video` media operations, which are also the arguments + * of the tools with those names. Times are seconds on the timeline left by the previous steps. + */ + +const Seconds = z.number().min(0).max(86_400); + +const TimeRangeSchema = z + .object({ start: Seconds, end: Seconds }) + .refine((range) => range.end > range.start, "end must be after start"); + +/** Written by the bot in the user's language; shown on the media card and used as file name. */ +export const MediaTitleSchema = z.string().trim().min(1).max(120); + +export const EditVideoStepSchema = z.discriminatedUnion("op", [ + z + .object({ op: z.literal("trim"), start: Seconds, end: Seconds }) + .refine((step) => step.end > step.start, "end must be after start"), + /** Keeps these parts and cuts everything between them. */ + z.object({ op: z.literal("keep"), segments: z.array(TimeRangeSchema).min(1).max(50) }), + /** Appends these videos, fitted to the current frame. */ + z.object({ op: z.literal("concat"), artifact_ids: z.array(Id).min(1).max(20) }), + z.object({ + op: z.literal("reframe"), + preset: MediaPresetIdSchema, + fit: MediaFitSchema.default("crop"), + }), + z.object({ op: z.literal("speed"), factor: z.number().min(0.25).max(4) }), + /** Linear gain: 0.5 halves the loudness, 2 doubles it. */ + z.object({ op: z.literal("volume"), gain: z.number().min(0).max(8) }), + z.object({ op: z.literal("mute") }), + /** Burns in an SRT or VTT artifact. */ + z.object({ op: z.literal("subtitles"), artifact_id: Id }), + /** Last step only: the result is the audio as M4A. */ + z.object({ op: z.literal("extract_audio") }), + /** Cover frame at this second of the result; as the only kind of step, the result is that image. */ + z.object({ op: z.literal("poster"), at: Seconds }), +]); +export type EditVideoStep = z.infer; + +export const EditVideoInputSchema = z + .object({ + artifact_id: Id, + steps: z.array(EditVideoStepSchema).min(1).max(20), + title: MediaTitleSchema, + }) + .superRefine((input, context) => { + const ops = input.steps.map((step) => step.op); + const extract = ops.indexOf("extract_audio"); + if (extract >= 0 && extract !== ops.length - 1) { + context.addIssue({ + code: "custom", + path: ["steps", extract], + message: "extract_audio must be the last step", + }); + } + if (extract >= 0 && ops.includes("poster")) { + context.addIssue({ + code: "custom", + path: ["steps"], + message: "poster cannot be combined with extract_audio", + }); + } + if (ops.filter((op) => op === "poster").length > 1) { + context.addIssue({ code: "custom", path: ["steps"], message: "use at most one poster step" }); + } + }); +export type EditVideoInput = z.infer; + +export const ExportVideoInputSchema = z.object({ + artifact_id: Id, + presets: z.array(MediaPresetIdSchema).min(1).max(6), + fit: MediaFitSchema.default("crop"), + title: MediaTitleSchema.optional(), +}); +export type ExportVideoInput = z.infer; diff --git a/packages/testkit/src/cli/harness.ts b/packages/testkit/src/cli/harness.ts index fc38326..ce36780 100644 --- a/packages/testkit/src/cli/harness.ts +++ b/packages/testkit/src/cli/harness.ts @@ -118,6 +118,7 @@ async function main() { "packages/adapters/src/job-reconciler.postgres.test.ts", "packages/adapters/src/cloud-agent.postgres.test.ts", "packages/adapters/src/media/image-operations.postgres.test.ts", + "packages/adapters/src/media/video-operations.postgres.test.ts", "packages/adapters/src/media-runtime.postgres.test.ts", "packages/adapters/src/builtin-memory-provider.postgres.test.ts", "packages/adapters/src/profile-consolidation.postgres.test.ts",