import type { EditVideoStep, MediaFit } from "@rakazo/contracts"; import { MEDIA_PRESETS } from "@rakazo/contracts"; import type { LoudnessMeasurement, MediaProbe } from "./ffmpeg.js"; import { ffmpegInputArgs, ffmpegSeconds, loudnormFilter } from "./ffmpeg.js"; /** * Turns an ordered list of edit steps into one ffmpeg invocation with a single filter_complex * pass. Pure: the caller probes the inputs and runs the arguments. */ export interface PlanInput { path: string; mimeType: string; name: string; probe: MediaProbe; } export interface VideoEditPlanRequest { source: PlanInput; steps: readonly EditVideoStep[]; /** Videos named by concat steps, by artifact id. */ clips?: ReadonlyMap; /** Images named by overlay steps, by artifact id. */ images?: ReadonlyMap; /** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */ subtitleFiles?: ReadonlyMap; /** * What each remove_silence step keeps, on its own input timeline: speech segments, or null * when nothing is silent. Detected beforehand by an analysis plan (see silence.ts). */ speech?: ReadonlyMap; /** * Analysis instead of an encode: the audio left by the steps runs through these filters * into a null output, so they can report on it (silencedetect writing a file, say). */ analyzeAudio?: readonly string[]; /** A platform's limits, applied after the steps. */ target?: ExportTarget; outputPath: string; } /** The frame, frame rate cap, bitrate cap and loudness a platform export is rendered to. */ export interface ExportTarget { width: number; height: number; fit: MediaFit; maxFps: number; maxBitrateKbps: number; integratedLufs: number; truePeakDb: number; } export interface VideoEditPlan { args: string[]; kind: "video" | "audio"; durationMs: number; width?: number; height?: number; posterAtMs?: number; } export const VIDEO_ENCODE_ARGS = [ "-c:v", "libx264", "-preset", "veryfast", "-crf", "21", "-pix_fmt", "yuv420p", "-movflags", "+faststart", ]; export const AUDIO_BITRATE_KBPS = 160; const AUDIO_ENCODE_ARGS = ["-c:a", "aac", "-b:a", `${AUDIO_BITRATE_KBPS}k`]; const AUDIO_ONLY_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"]; /** Location and other camera metadata stay private; chapters do not survive cuts anyway. */ const STRIP_METADATA_ARGS = ["-map_metadata", "-1", "-map_chapters", "-1"]; const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo"; const SAFE_FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; const MIN_SEGMENT_SECONDS = 0.1; const FALLBACK_FPS = 30; interface Track { label: string; filters: string[]; } interface Timeline { /** Null when the result is audio only: video work is skipped. */ video: Track | null; audio: Track | null; width: number; height: number; fps: number; seconds: number; /** How appended clips fill the frame: the last reframe's fit, otherwise bars. */ fit: MediaFit; } class FilterGraph { private readonly chains: string[] = []; private count = 0; label(kind: "v" | "a"): string { this.count += 1; return `${kind}${this.count}`; } add(inputs: readonly string[], filters: readonly string[], outputs: readonly string[]) { const pads = (labels: readonly string[]) => labels.map((label) => `[${label}]`).join(""); this.chains.push(`${pads(inputs)}${filters.join(",")}${pads(outputs)}`); } /** Ends a track's pending filters in a labelled pad other chains can consume. */ flush(track: Track, kind: "v" | "a"): string { if (track.filters.length === 0) return track.label; const output = this.label(kind); this.add([track.label], track.filters, [output]); track.label = output; track.filters = []; return output; } toString(): string { return this.chains.join(";"); } } export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { const { source } = request; const video = source.probe.video; if (!video) throw new Error(`${source.name} has no video.`); const sourceSeconds = source.probe.durationMs / 1000; const audioOnly = request.steps.at(-1)?.op === "extract_audio" || request.analyzeAudio !== undefined; const graph = new FilterGraph(); const inputs: string[][] = []; const addInput = (input: PlanInput, seek?: { start: number; end: number }) => { inputs.push( ffmpegInputArgs(input.path, { mimeType: input.mimeType, ...(seek ? { startSeconds: seek.start, durationSeconds: seek.end - seek.start } : {}), }), ); return inputs.length - 1; }; let steps = request.steps; // A leading cut seeks the input instead of decoding everything before it. const first = steps[0]; let segments: Segment[] | undefined; if (first?.op === "trim" || first?.op === "keep") { segments = normalizeSegments( first.op === "trim" ? [first] : first.segments, sourceSeconds, first.op, ); steps = steps.slice(1); } const seek = segments && { start: segments[0]!.start, end: segments.at(-1)!.end }; const index = addInput(source, seek); const size = displaySize(video); const timeline: Timeline = { video: audioOnly ? null : { label: `${index}:v:0`, filters: sourceFilters(video) }, audio: source.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null, width: size.width, height: size.height, fps: video.fps, seconds: seek ? seek.end - seek.start : sourceSeconds, fit: "pad", }; if (segments && seek && segments.length > 1) { keepSegments( graph, timeline, segments.map((segment) => ({ start: segment.start - seek.start, end: segment.end - seek.start, })), ); } let posterAt: number | undefined; for (const step of steps) { switch (step.op) { case "trim": case "keep": keepSegments( graph, timeline, normalizeSegments(step.op === "trim" ? [step] : step.segments, timeline.seconds, step.op), ); break; case "concat": addClips( graph, timeline, step.artifact_ids.map((id) => { const clip = request.clips?.get(id); if (!clip) throw new Error(`Video ${id} is not available.`); return { clip, index: addInput(clip) }; }), step.at, ); break; case "reframe": { const preset = MEDIA_PRESETS[step.preset]; if (timeline.video) { timeline.video = fitFrame( graph, timeline.video, timeline, { width: preset.width, height: preset.height }, step.fit, ); } timeline.width = preset.width; timeline.height = preset.height; timeline.fit = step.fit; break; } case "speed": timeline.video?.filters.push( `setpts=PTS/${number(step.factor)}`, ...(timeline.fps ? [`fps=${number(timeline.fps)}`] : []), ); timeline.audio?.filters.push(...atempoFilters(step.factor)); timeline.seconds /= step.factor; break; case "volume": timeline.audio?.filters.push(`volume=${number(step.gain)}`); break; case "mute": timeline.audio = null; break; case "subtitles": { const file = request.subtitleFiles?.get(step.artifact_id); if (!file) throw new Error(`Subtitles ${step.artifact_id} are not available.`); // Only generated names reach the filter graph, so nothing here needs escaping. if (!SAFE_FILE_NAME.test(file)) throw new Error("Unsafe subtitle file name."); timeline.video?.filters.push( `subtitles=filename=${file}:force_style='${subtitleStyle(timeline.width, timeline.height)}'`, ); break; } case "extract_audio": break; case "poster": posterAt = step.at; break; case "remove_silence": { const speech = request.speech?.get(step); if (speech === undefined) throw new Error("Silence was not detected before planning."); if (speech) keepSegments(graph, timeline, speech); break; } case "overlay": { if (!timeline.video) break; const image = request.images?.get(step.artifact_id); if (!image) throw new Error(`Image ${step.artifact_id} is not available.`); if (step.start !== undefined && step.start >= timeline.seconds) { throw new Error( `overlay from ${seconds(step.start)} s starts after the end of the ${seconds(timeline.seconds)} s video.`, ); } overlayImage(graph, timeline, timeline.video, step, addInput(image)); break; } } } if (posterAt !== undefined && posterAt >= timeline.seconds) { throw new Error( `poster at ${seconds(posterAt)} s is past the end of the ${seconds(timeline.seconds)} s result.`, ); } const { target } = request; if (target && !audioOnly) applyTarget(graph, timeline, target); const durationMs = Math.round(timeline.seconds * 1000); const output = `file:${request.outputPath}`; if (audioOnly || !timeline.video) { if (!timeline.audio) throw new Error(`${source.name} has no audio.`); if (request.analyzeAudio) { timeline.audio.filters.push(...request.analyzeAudio); const audio = graph.flush(timeline.audio, "a"); return { args: [ ...inputs.flat(), "-filter_complex", graph.toString(), "-map", `[${audio}]`, "-f", "null", "-", ], kind: "audio", durationMs, }; } timeline.audio.filters.push(AUDIO_FORMAT); const audio = graph.flush(timeline.audio, "a"); return { args: [ ...inputs.flat(), "-filter_complex", graph.toString(), "-map", `[${audio}]`, ...AUDIO_ONLY_ENCODE_ARGS, ...STRIP_METADATA_ARGS, "-f", "mp4", output, ], kind: "audio", durationMs, }; } timeline.video.filters.push("format=yuv420p"); const videoOut = graph.flush(timeline.video, "v"); let audioOut: string | undefined; if (timeline.audio) { timeline.audio.filters.push(AUDIO_FORMAT); audioOut = graph.flush(timeline.audio, "a"); } return { args: [ ...inputs.flat(), "-filter_complex", graph.toString(), "-map", `[${videoOut}]`, ...(audioOut ? ["-map", `[${audioOut}]`, ...AUDIO_ENCODE_ARGS] : []), ...VIDEO_ENCODE_ARGS, ...(target && !audioOnly ? bitrateCapArgs(target.maxBitrateKbps) : []), ...STRIP_METADATA_ARGS, "-f", "mp4", output, ], kind: "video", durationMs, width: timeline.width, height: timeline.height, ...(posterAt !== undefined ? { posterAtMs: Math.round(posterAt * 1000) } : {}), }; } /** * Fits the result to a platform: frame rate cap, frame size and loudness. Loudness is one * dynamic loudnorm pass, which lands within about 1 LU of the target on most material without * decoding the source twice; the export measures the result and corrects it when it misses. */ function applyTarget(graph: FilterGraph, timeline: Timeline, target: ExportTarget) { if (timeline.video) { if (!timeline.fps || timeline.fps > target.maxFps + 0.01) { timeline.video.filters.push(`fps=${number(target.maxFps)}`); timeline.fps = target.maxFps; } timeline.video = fitFrame(graph, timeline.video, timeline, target, target.fit); } timeline.width = target.width; timeline.height = target.height; // loudnorm pads its last frame; trim the audio back to the video's length. timeline.audio?.filters.push( loudnormFilter(target), `atrim=end=${ffmpegSeconds(timeline.seconds)}`, ); } /** CRF quality, capped so no stretch of the video runs above the platform's bitrate. */ function bitrateCapArgs(maxBitrateKbps: number): string[] { const kbps = Math.max(1, Math.round(maxBitrateKbps)); return ["-maxrate", `${kbps}k`, "-bufsize", `${kbps * 2}k`]; } /** * Second loudness pass over a rendered MP4: linear loudnorm with the measured values on the * audio, the video copied as it is. */ export function planLoudnessFix(request: { inputPath: string; outputPath: string; durationMs: number; target: { integratedLufs: number; truePeakDb: number }; measured: LoudnessMeasurement; }): string[] { const filters = [ loudnormFilter(request.target, request.measured), `atrim=end=${ffmpegSeconds(request.durationMs / 1000)}`, AUDIO_FORMAT, ]; return [ ...ffmpegInputArgs(request.inputPath, { mimeType: "video/mp4" }), "-filter_complex", `[0:a:0]${filters.join(",")}[a1]`, ...["-map", "0:v:0", "-map", "[a1]", "-c:v", "copy"], ...AUDIO_ENCODE_ARGS, ...["-movflags", "+faststart"], ...STRIP_METADATA_ARGS, "-f", "mp4", `file:${request.outputPath}`, ]; } interface Segment { start: number; end: number; } /** Validates ranges against the timeline, clamps ends to it, sorts and merges overlaps. */ function normalizeSegments( ranges: readonly Segment[], duration: number, op: "trim" | "keep", ): Segment[] { const clamped = ranges.map((range) => { if (range.start >= duration) { throw new Error( `${op} ${seconds(range.start)}–${seconds(range.end)} s is outside the ${seconds(duration)} s video.`, ); } const end = Math.min(range.end, duration); if (end - range.start < MIN_SEGMENT_SECONDS) { throw new Error( `${op} ${seconds(range.start)}–${seconds(end)} s is shorter than ${MIN_SEGMENT_SECONDS} s.`, ); } return { start: range.start, end }; }); clamped.sort((a, b) => a.start - b.start); const merged: Segment[] = []; for (const segment of clamped) { const last = merged.at(-1); if (last && segment.start <= last.end) last.end = Math.max(last.end, segment.end); else merged.push({ ...segment }); } return merged; } /** Keeps sorted, disjoint segments of the timeline: one trim, or split + trims + concat. */ function keepSegments(graph: FilterGraph, timeline: Timeline, segments: readonly Segment[]) { const [only] = segments; if (segments.length === 1 && only) { const range = `start=${ffmpegSeconds(only.start)}:end=${ffmpegSeconds(only.end)}`; timeline.video?.filters.push(`trim=${range}`, "setpts=PTS-STARTPTS"); timeline.audio?.filters.push(`atrim=${range}`, "asetpts=PTS-STARTPTS"); timeline.seconds = only.end - only.start; return; } const count = segments.length; const branch = (track: Track | null, kind: "v" | "a"): string[] => { if (!track) return []; const input = graph.flush(track, kind); const splits = segments.map(() => graph.label(kind)); graph.add([input], [`${kind === "v" ? "split" : "asplit"}=${count}`], splits); return segments.map((segment, index) => { const output = graph.label(kind); const range = `start=${ffmpegSeconds(segment.start)}:end=${ffmpegSeconds(segment.end)}`; graph.add( [splits[index]!], kind === "v" ? [`trim=${range}`, "setpts=PTS-STARTPTS"] : [`atrim=${range}`, "asetpts=PTS-STARTPTS"], [output], ); return output; }); }; const videoParts = branch(timeline.video, "v"); const audioParts = branch(timeline.audio, "a"); concatParts(graph, timeline, videoParts, audioParts, count); timeline.seconds = segments.reduce((total, segment) => total + segment.end - segment.start, 0); } /** * Adds clips at the end or the start of the timeline, normalized to it: frame size, fit, fps * and 48 kHz stereo audio. */ function addClips( graph: FilterGraph, timeline: Timeline, clips: readonly { clip: PlanInput; index: number }[], at: "end" | "start", ) { const fps = timeline.fps || FALLBACK_FPS; const needsAudio = timeline.audio !== null || clips.some(({ clip }) => clip.probe.hasAudio); const videoParts: string[] = []; const audioParts: string[] = []; let mainVideo: string | undefined; let mainAudio: string | undefined; if (timeline.video) { timeline.video.filters.push("setsar=1", "format=yuv420p"); mainVideo = graph.flush(timeline.video, "v"); } if (needsAudio) mainAudio = audioPart(graph, timeline.audio, timeline.seconds); let seconds = timeline.seconds; for (const { clip, index } of clips) { const video = clip.probe.video; if (!video) throw new Error(`${clip.name} has no video.`); const clipSeconds = clip.probe.durationMs / 1000; if (clipSeconds <= 0) throw new Error(`Could not read the length of ${clip.name}.`); if (timeline.video) { const track = fitFrame( graph, { label: `${index}:v:0`, filters: sourceFilters(video) }, displaySize(video), timeline, timeline.fit, ); track.filters.push(`fps=${number(fps)}`, "format=yuv420p"); videoParts.push(graph.flush(track, "v")); } if (needsAudio) { const track = clip.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null; audioParts.push(audioPart(graph, track, clipSeconds)); } seconds += clipSeconds; } const add = at === "start" ? "push" : "unshift"; if (mainVideo) videoParts[add](mainVideo); if (mainAudio) audioParts[add](mainAudio); concatParts(graph, timeline, videoParts, audioParts, clips.length + 1); timeline.seconds = seconds; } /** * Scales an image to its share of the frame width, within the margins, and lays it over the * video at a corner or the center. Margins are wider on tall frames, where 9:16 apps put their * own buttons and caption, so the top corners stay visible there. */ function overlayImage( graph: FilterGraph, timeline: Timeline, video: Track, step: Extract, index: number, ) { const { width, height } = timeline; const tall = height >= width * 1.5; const marginX = Math.round(width * (tall ? 0.06 : 0.04)); const top = Math.round(height * (tall ? 0.15 : 0.04)); const bottom = Math.round(height * (tall ? 0.2 : 0.04)); const image = graph.label("v"); graph.add( [`${index}:v:0`], [ `scale=w=${even(width * step.width)}:h=${even(height - top - bottom)}:force_original_aspect_ratio=decrease:force_divisible_by=2`, "format=rgba", ...(step.opacity < 1 ? [`colorchannelmixer=aa=${number(step.opacity)}`] : []), ], [image], ); const x = step.position.endsWith("left") ? `${marginX}` : step.position.endsWith("right") ? `W-w-${marginX}` : "(W-w)/2"; const y = step.position.startsWith("top") ? `${top}` : step.position.startsWith("bottom") ? `H-h-${bottom}` : "(H-h)/2"; // A still image ends after its one frame; overlay repeats it for as long as the video runs. const timed = step.start !== undefined || step.end !== undefined; const enable = timed ? `:enable='between(t,${ffmpegSeconds(step.start ?? 0)},${ffmpegSeconds(step.end ?? timeline.seconds)})'` : ""; const output = graph.label("v"); graph.add([graph.flush(video, "v"), image], [`overlay=x=${x}:y=${y}${enable}`], [output]); timeline.video = { label: output, filters: [] }; } /** A clip's audio in the common format, or silence as long as the clip when it has none. */ function audioPart(graph: FilterGraph, track: Track | null, seconds: number): string { if (track) { track.filters.push(AUDIO_FORMAT); return graph.flush(track, "a"); } const output = graph.label("a"); graph.add( [], [ "anullsrc=channel_layout=stereo:sample_rate=48000", `atrim=end=${ffmpegSeconds(seconds)}`, AUDIO_FORMAT, ], [output], ); return output; } function concatParts( graph: FilterGraph, timeline: Timeline, videoParts: readonly string[], audioParts: readonly string[], count: number, ) { const inputs = Array.from({ length: count }, (_, index) => [ ...(videoParts[index] ? [videoParts[index]] : []), ...(audioParts[index] ? [audioParts[index]] : []), ]).flat(); const video = videoParts.length > 0 ? graph.label("v") : undefined; const audio = audioParts.length > 0 ? graph.label("a") : undefined; graph.add( inputs, [`concat=n=${count}:v=${video ? 1 : 0}:a=${audio ? 1 : 0}`], [...(video ? [video] : []), ...(audio ? [audio] : [])], ); timeline.video = video ? { label: video, filters: [] } : null; timeline.audio = audio ? { label: audio, filters: [] } : null; } interface Size { width: number; height: number; } /** Scales a frame of size `from` into `to`: center crop, bars, or a blurred copy behind it. */ function fitFrame(graph: FilterGraph, track: Track, from: Size, to: Size, fit: MediaFit): Track { const { width, height } = to; if (Math.abs(from.width / from.height - width / height) < 0.01) { track.filters.push(`scale=${width}:${height}`, "setsar=1"); return track; } const contain = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`; if (fit === "crop") { track.filters.push( `scale=${width}:${height}:force_original_aspect_ratio=increase`, `crop=${width}:${height}`, "setsar=1", ); return track; } if (fit === "pad") { track.filters.push( contain, `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, "setsar=1", ); return track; } // Blur a small copy and scale it up: as smooth as a large blur at a fraction of the cost. const input = graph.flush(track, "v"); const [background, foreground, blurred, contained, output] = [ graph.label("v"), graph.label("v"), graph.label("v"), graph.label("v"), graph.label("v"), ]; const smallWidth = even(width / 4); const smallHeight = even(height / 4); const radius = Math.max(2, Math.floor(Math.min(smallWidth, smallHeight) / 10)); graph.add([input], ["split=2"], [background, foreground]); graph.add( [background], [ `scale=${smallWidth}:${smallHeight}:force_original_aspect_ratio=increase`, `crop=${smallWidth}:${smallHeight}`, `boxblur=${radius}:2`, `scale=${width}:${height}`, "setsar=1", ], [blurred], ); graph.add([foreground], [contain, "setsar=1"], [contained]); graph.add([blurred, contained], ["overlay=(W-w)/2:(H-h)/2"], [output]); return { label: output, filters: [] }; } /** Square pixels and even dimensions, which yuv420p H.264 requires. */ function sourceFilters(video: NonNullable): string[] { const size = displaySize(video); if (Math.abs(video.sampleAspectRatio - 1) > 0.01) { return [`scale=${size.width}:${size.height}`, "setsar=1"]; } if (size.width !== video.width || size.height !== video.height) { return [`crop=${size.width}:${size.height}:0:0`]; } return []; } function displaySize(video: NonNullable): Size { if (Math.abs(video.sampleAspectRatio - 1) > 0.01) { return { width: even(Math.round(video.width * video.sampleAspectRatio)), height: even(video.height), }; } return { width: even(video.width), height: even(video.height) }; } /** atempo takes 0.5–2 per instance on older ffmpeg builds, so larger changes chain it. */ export function atempoFilters(factor: number): string[] { const filters: string[] = []; let rest = factor; while (rest > 2) { filters.push("atempo=2"); rest /= 2; } while (rest < 0.5) { filters.push("atempo=0.5"); rest /= 0.5; } filters.push(`atempo=${number(rest)}`); return filters; } /** * Readable burned-in subtitles: bold white text with a dark outline, sized to the frame, and * lifted above the bottom fifth of tall frames where 9:16 apps place their own controls. */ export function subtitleStyle(width: number, height: number): string { const tall = height >= width * 1.5; const fontSize = Math.round(Math.min(width, height) * 0.06); const marginX = Math.round(width * (tall ? 0.1 : 0.05)); const marginY = Math.round(height * (tall ? 0.2 : 0.06)); return [ `PlayResX=${width}`, `PlayResY=${height}`, "FontName=DejaVu Sans", `FontSize=${fontSize}`, "Bold=1", "PrimaryColour=&H00FFFFFF", "OutlineColour=&H00000000", "BackColour=&H80000000", "BorderStyle=1", `Outline=${Math.max(2, Math.round(fontSize / 16))}`, "Shadow=1", "Alignment=2", `MarginL=${marginX}`, `MarginR=${marginX}`, `MarginV=${marginY}`, ].join(","); } function even(value: number): number { return Math.max(2, 2 * Math.floor(value / 2)); } function number(value: number): string { return String(Number(value.toFixed(6))); } function seconds(value: number): string { return String(Number(value.toFixed(2))); }