import type { BrandKit, CaptionStylePreset, Edl, EdlAudioClip, EdlOverlay, EdlVideoClip, MediaRenderQuality, MotionTemplateRef, NormalizedBox, PlatformProfile, ReframeKeyframe, RenderPlan, RenderStep, } from "@rakazo/contracts"; import { EdlSchema } from "@rakazo/contracts"; import { BLENDED_TRANSITIONS, clipLengthMs, edlDurationMs, introMs, mainProgramMs, sortedVideoClips, } from "./edl.js"; export interface CompileSource { /** Redacted intermediate when redactions apply, otherwise original or proxy. */ path: string; /** * Coded size as stored. Rotation 90/270 swaps it for display, because * ffmpeg auto-rotates while decoding. */ width: number; height: number; durationMs: number; fps: number; hasAudio: boolean; rotation: 0 | 90 | 180 | 270; /** A still image (photo). Also assumed when durationMs is 0. */ still?: boolean; } export interface CaptionWord { startMs: number; endMs: number; text: string; } export interface CompileInput { edl: Edl; profile: PlatformProfile; brandKit: BrandKit | null; quality: MediaRenderQuality; /** Absolute work directory for intermediates and outputs. */ workDir: string; sources: Map; /** Source-time words per asset for captions. */ transcripts: Map; /** Saliency or subject points per asset for "auto" reframe. */ subjects: Map>; /** * Pre-rendered motion clips keyed by overlay id, "intro", "outro". Missing * ones become `motion` steps that the renderer runs first. */ motionClips: Map; files: { lutPath: string | null; watermarkPath: string | null; /** Brand-kit font artifact id (or family name) → font file. */ fonts: Map; /** artifactId → absolute path for audio clips. */ audio: Map; /** First brand-kit logo, for logo overlays without a motion clip. */ logoPath?: string | null; }; /** ffmpeg video encoder name, e.g. libx264, h264_vaapi, h264_videotoolbox. */ videoEncoder: string; /** Output time of the poster frame. Defaults to a third into the main program. */ posterAtMs?: number; } // --------------------------------------------------------------------------- // Escaping // --------------------------------------------------------------------------- /* * A filtergraph value passes three parsers: * 1. the graph parser splits filters on `[ ] , ;` and honours `\` and `'`, * 2. the option parser splits options on `:` (and `=`), again with `\` and `'`, * 3. drawtext expands `%{...}` and treats `\x` as a literal `x`. * Each level is escaped with a backslash, innermost first, so any user text * arrives verbatim and can never open a new option, filter, or expansion. */ const escapeDrawtextLevel = (value: string) => value.replace(/[\\%]/g, "\\$&"); const escapeOptionLevel = (value: string) => value.replace(/[\\':=]/g, "\\$&"); const escapeGraphLevel = (value: string) => value.replace(/[\\'[\],;]/g, "\\$&"); /** Escape text for ffmpeg drawtext/filtergraph values. */ export function escapeFilterText(text: string): string { const normalized = text .replace(/\r\n?/g, "\n") // biome-ignore lint/suspicious/noControlCharactersInRegex: strip control characters on purpose .replace(/[\u0000-\u0009\u000b-\u001f\u007f]/g, "") .trim(); return escapeGraphLevel(escapeOptionLevel(escapeDrawtextLevel(normalized))); } /** Escape a file path or expression used as a filter option value. */ export function escapeFilterValue(value: string): string { if (value.includes("\0")) throw new Error("filter value contains NUL"); return escapeGraphLevel(escapeOptionLevel(value)); } // --------------------------------------------------------------------------- // Small helpers // --------------------------------------------------------------------------- /** Fixed-point number without exponent notation. */ function num(value: number, digits = 3): string { const fixed = value.toFixed(digits); return fixed.includes(".") ? fixed.replace(/\.?0+$/, "") || "0" : fixed; } const sec = (ms: number) => num(ms / 1000); function joinPath(dir: string, name: string): string { return `${dir.replace(/[\\/]+$/, "")}/${name}`; } function dirName(file: string): string { const index = Math.max(file.lastIndexOf("/"), file.lastIndexOf("\\")); return index > 0 ? file.slice(0, index) : "."; } function brandColor(hex: string | undefined, fallback: string): string { return hex ? `0x${hex.slice(1).toUpperCase()}` : fallback; } function even(value: number): number { return Math.max(2, Math.floor(value / 2) * 2); } const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value)); function enableBetween(startMs: number, endMs: number): string { return escapeFilterValue(`between(t,${sec(startMs)},${sec(endMs)})`); } /** atempo accepts 0.5..2 per instance for clean results; chain for the rest. */ export function atempoChain(speed: number): string[] { const factors: number[] = []; let rest = speed; while (rest > 2) { factors.push(2); rest /= 2; } while (rest < 0.5) { factors.push(0.5); rest /= 0.5; } if (Math.abs(rest - 1) > 1e-9 || factors.length === 0) factors.push(rest); return factors.filter((f) => Math.abs(f - 1) > 1e-9).map((f) => `atempo=${num(f, 6)}`); } // --------------------------------------------------------------------------- // Reframing // --------------------------------------------------------------------------- interface FrameGeometry { sourceWidth: number; sourceHeight: number; outWidth: number; outHeight: number; durationMs: number; } /** Largest crop with the output aspect inside the source, shrunk by scale. */ function cropSize(geometry: FrameGeometry, scale: number): { w: number; h: number } { const outAspect = geometry.outWidth / geometry.outHeight; let w: number; let h: number; if (geometry.sourceWidth / geometry.sourceHeight > outAspect) { h = geometry.sourceHeight; w = h * outAspect; } else { w = geometry.sourceWidth; h = w / outAspect; } const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2); return { w: Math.min(geometry.sourceWidth, roundEven(w / scale)), h: Math.min(geometry.sourceHeight, roundEven(h / scale)), }; } const AUTO_STEP_MS = 500; const AUTO_WINDOW_MS = 750; const AUTO_SMOOTHING = 0.35; const AUTO_TOLERANCE = 0.01; const MAX_KEYFRAMES = 64; function smooth(values: number[]): number[] { const forward = [...values]; for (let i = 1; i < forward.length; i++) { forward[i] = forward[i - 1]! + AUTO_SMOOTHING * (forward[i]! - forward[i - 1]!); } const backward = [...forward]; for (let i = backward.length - 2; i >= 0; i--) { backward[i] = backward[i + 1]! + AUTO_SMOOTHING * (backward[i]! - backward[i + 1]!); } return backward; } /** Keep only keyframes that linear interpolation cannot reproduce, at most MAX_KEYFRAMES. */ function simplify(keyframes: ReframeKeyframe[]): ReframeKeyframe[] { if (keyframes.length <= 2) return keyframes; const kept: ReframeKeyframe[] = [keyframes[0]!]; for (let i = 1; i < keyframes.length - 1; i++) { const last = kept.at(-1)!; const next = keyframes[i + 1]!; const point = keyframes[i]!; const f = (point.tMs - last.tMs) / (next.tMs - last.tMs); const cx = last.cx + (next.cx - last.cx) * f; const cy = last.cy + (next.cy - last.cy) * f; if (Math.abs(cx - point.cx) > AUTO_TOLERANCE || Math.abs(cy - point.cy) > AUTO_TOLERANCE) { kept.push(point); } } kept.push(keyframes.at(-1)!); if (kept.length <= MAX_KEYFRAMES) return kept; const step = (kept.length - 1) / (MAX_KEYFRAMES - 1); return Array.from({ length: MAX_KEYFRAMES }, (_, i) => kept[Math.round(i * step)]!); } /** * Smoothed crop path that keeps the subject in frame for the output aspect. * Subjects are weighted by strength in a ±750 ms window every 500 ms, the * path is smoothed forwards and backwards (no lag), clamped so the crop stays * inside the source, and reduced to the keyframes that matter. Times are the * subjects' time base, from 0 to durationMs. */ export function autoReframe( subjects: Array<{ tMs: number; cx: number; cy: number; strength: number }>, input: FrameGeometry, ): ReframeKeyframe[] { const crop = cropSize(input, 1); const halfW = crop.w / input.sourceWidth / 2; const halfH = crop.h / input.sourceHeight / 2; const clampX = (x: number) => clamp(x, halfW, 1 - halfW); const clampY = (y: number) => clamp(y, halfH, 1 - halfH); const usable = subjects .filter((s) => s.strength > 0 && Number.isFinite(s.cx) && Number.isFinite(s.cy)) .sort((a, b) => a.tMs - b.tMs); if (usable.length === 0) return [{ tMs: 0, cx: clampX(0.5), cy: clampY(0.5), scale: 1 }]; const times: number[] = []; for (let t = 0; t < input.durationMs; t += AUTO_STEP_MS) times.push(t); times.push(Math.max(0, input.durationMs)); const xs: number[] = []; const ys: number[] = []; for (const t of times) { let weight = 0; let x = 0; let y = 0; for (const subject of usable) { const distance = Math.abs(subject.tMs - t); if (distance > AUTO_WINDOW_MS) continue; const w = subject.strength * (1 - distance / (AUTO_WINDOW_MS * 2)); weight += w; x += subject.cx * w; y += subject.cy * w; } if (weight > 0) { xs.push(x / weight); ys.push(y / weight); } else if (xs.length > 0) { xs.push(xs.at(-1)!); ys.push(ys.at(-1)!); } else { const nearest = usable.reduce((best, s) => Math.abs(s.tMs - t) < Math.abs(best.tMs - t) ? s : best, ); xs.push(nearest.cx); ys.push(nearest.cy); } } const sx = smooth(xs); const sy = smooth(ys); const round = (v: number) => Math.round(v * 10_000) / 10_000; return simplify( times.map((tMs, i) => ({ tMs: Math.round(tMs), cx: round(clampX(sx[i]!)), cy: round(clampY(sy[i]!)), scale: 1, })), ); } const REFRAME_SCALES = [1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4]; const FACE_MARGIN = 0.02; /** * Crop path that keeps every given face box outside the output frame, or * null if impossible. The window is static for the whole shot (a moving * window could sweep across a face between keyframes): it avoids the union of * all face boxes plus a 2 % margin, prefers the widest crop, and then the * position closest to the frame centre. */ export function reframeExcluding( faces: Array<{ tMs: number; box: NormalizedBox }>, input: FrameGeometry, ): ReframeKeyframe[] | null { const tMs = faces.reduce((min, face) => Math.min(min, face.tMs), Number.POSITIVE_INFINITY); const boxes = faces.map(({ box }) => ({ x0: clamp(box.x - FACE_MARGIN, 0, 1), y0: clamp(box.y - FACE_MARGIN, 0, 1), x1: clamp(box.x + box.w + FACE_MARGIN, 0, 1), y1: clamp(box.y + box.h + FACE_MARGIN, 0, 1), })); for (const scale of REFRAME_SCALES) { const crop = cropSize(input, scale); const w = crop.w / input.sourceWidth; const h = crop.h / input.sourceHeight; const xs = new Set([0.5, w / 2, 1 - w / 2]); const ys = new Set([0.5, h / 2, 1 - h / 2]); for (const b of boxes) { xs.add(b.x0 - w / 2); xs.add(b.x1 + w / 2); ys.add(b.y0 - h / 2); ys.add(b.y1 + h / 2); } const candidates: Array<{ cx: number; cy: number; distance: number }> = []; for (const rawX of xs) { for (const rawY of ys) { const cx = clamp(rawX, w / 2, 1 - w / 2); const cy = clamp(rawY, h / 2, 1 - h / 2); const clear = boxes.every( (b) => cx + w / 2 <= b.x0 + 1e-9 || cx - w / 2 >= b.x1 - 1e-9 || cy + h / 2 <= b.y0 + 1e-9 || cy - h / 2 >= b.y1 - 1e-9, ); if (clear) candidates.push({ cx, cy, distance: Math.hypot(cx - 0.5, cy - 0.5) }); } } candidates.sort((a, b) => a.distance - b.distance || a.cx - b.cx || a.cy - b.cy); const best = candidates[0]; if (best) { const round = (v: number) => Math.round(v * 10_000) / 10_000; return [ { tMs: Number.isFinite(tMs) ? Math.round(tMs) : 0, cx: round(best.cx), cy: round(best.cy), scale, }, ]; } } return null; } /** Piecewise-linear expression of `t` (seconds) through the given points. */ function piecewise(points: Array<{ t: number; v: number }>): string { if (points.length === 1) return num(points[0]!.v, 2); let expression = num(points.at(-1)!.v, 2); for (let i = points.length - 2; i >= 0; i--) { const a = points[i]!; const b = points[i + 1]!; const span = Math.max(b.t - a.t, 0.001); const segment = `${num(a.v, 2)}+(${num(b.v - a.v, 2)})*(t-${num(a.t)})/${num(span)}`; expression = `if(lt(t,${num(b.t)}),${segment},${expression})`; } return `if(lt(t,${num(points[0]!.t)}),${num(points[0]!.v, 2)},${expression})`; } /** crop+scale filters for keyframes in source time; uses the tightest scale of the path. */ function cropFilters( keyframes: ReframeKeyframe[], clip: EdlVideoClip, geometry: FrameGeometry, ): string[] { const scale = Math.max(...keyframes.map((k) => k.scale)); const crop = cropSize(geometry, scale); const points = [...keyframes] .sort((a, b) => a.tMs - b.tMs) .map((k) => ({ t: Math.max(0, (k.tMs - clip.inMs) / 1000), x: clamp(k.cx * geometry.sourceWidth - crop.w / 2, 0, geometry.sourceWidth - crop.w), y: clamp(k.cy * geometry.sourceHeight - crop.h / 2, 0, geometry.sourceHeight - crop.h), })); const x = piecewise(points.map((p) => ({ t: p.t, v: p.x }))); const y = piecewise(points.map((p) => ({ t: p.t, v: p.y }))); return [ `crop=w=${crop.w}:h=${crop.h}:x=${escapeFilterValue(x)}:y=${escapeFilterValue(y)}`, `scale=${geometry.outWidth}:${geometry.outHeight}`, ]; } function fitFilters(width: number, height: number): string[] { return [ `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`, `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, ]; } // --------------------------------------------------------------------------- // Captions // --------------------------------------------------------------------------- export interface CaptionCue { startMs: number; endMs: number; text: string; /** Word timings inside the cue for karaoke highlight. */ words: CaptionWord[]; } const CUE_PAUSE_MS = 1000; const CUE_MAX_MS = 7000; const CUE_MIN_MS = 500; function cleanWord(word: CaptionWord): CaptionWord | null { const text = word.text.replace(/\s+/g, " ").trim(); if (!text || !(word.endMs >= word.startMs)) return null; return { startMs: word.startMs, endMs: word.endMs, text }; } /** Greedy line wrap; a single word longer than the limit gets its own line. */ function wrapWords(words: CaptionWord[], maxChars: number): CaptionWord[][] { const lines: CaptionWord[][] = []; let line: CaptionWord[] = []; let length = 0; for (const word of words) { const extra = line.length === 0 ? word.text.length : length + 1 + word.text.length; if (line.length > 0 && extra > maxChars) { lines.push(line); line = [word]; length = word.text.length; } else { line.push(word); length = extra; } } if (line.length > 0) lines.push(line); return lines; } /** * Group words into cues of at most maxLines lines of maxCharsPerLine. A cue * also ends at a pause over 1 s, after sentence punctuation, or at 7 s. Each * cue is shown at least 500 ms unless the next one starts earlier. */ export function buildCaptionCues( words: CaptionWord[], options: { maxCharsPerLine: number; maxLines: number }, ): CaptionCue[] { const clean = words .map(cleanWord) .filter((w): w is CaptionWord => w !== null) .sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs); const groups: CaptionWord[][] = []; let current: CaptionWord[] = []; for (const word of clean) { const last = current.at(-1); const first = current[0]; const breaks = last !== undefined && first !== undefined && (word.startMs - last.endMs > CUE_PAUSE_MS || /[.!?…]["»”)]?$/.test(last.text) || word.endMs - first.startMs > CUE_MAX_MS || wrapWords([...current, word], options.maxCharsPerLine).length > options.maxLines); if (breaks) { groups.push(current); current = []; } current.push(word); } if (current.length > 0) groups.push(current); return groups.map((group, index) => { const startMs = group[0]!.startMs; const next = groups[index + 1]?.[0]?.startMs ?? Number.POSITIVE_INFINITY; const spoken = Math.max(...group.map((w) => w.endMs)); const endMs = Math.max(spoken, Math.min(startMs + CUE_MIN_MS, next)); return { startMs, endMs: Math.max(endMs, startMs + 1), text: wrapWords(group, options.maxCharsPerLine) .map((line) => line.map((w) => w.text).join(" ")) .join("\n"), words: group, }; }); } function clock(ms: number, separator: "," | "."): string { const total = Math.max(0, Math.round(ms)); const hours = Math.floor(total / 3_600_000); const minutes = Math.floor((total % 3_600_000) / 60_000); const seconds = Math.floor((total % 60_000) / 1000); const millis = total % 1000; const pad = (v: number, n = 2) => String(v).padStart(n, "0"); return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}${separator}${pad(millis, 3)}`; } /** Cue text without blank lines, which would end a cue early in both formats. */ function cueLines(text: string): string { return text .split("\n") .map((line) => line.trim()) .filter(Boolean) .join("\n"); } export function toSrt(cues: CaptionCue[]): string { return cues .map( (cue, index) => `${index + 1}\n${clock(cue.startMs, ",")} --> ${clock(cue.endMs, ",")}\n${cueLines( cue.text, ).replace(/-->/g, "->")}\n`, ) .join("\n"); } export function toVtt(cues: CaptionCue[]): string { const escapeCue = (text: string) => text.replace(/&/g, "&").replace(//g, ">"); return `WEBVTT\n\n${cues .map( (cue) => `${clock(cue.startMs, ".")} --> ${clock(cue.endMs, ".")}\n${escapeCue(cueLines(cue.text))}\n`, ) .join("\n")}`; } /** ASS colour &HAABBGGRR from #RRGGBB. */ function assColor(hex: string, alpha = 0): string { const r = hex.slice(1, 3); const g = hex.slice(3, 5); const b = hex.slice(5, 7); return `&H${alpha.toString(16).padStart(2, "0")}${b}${g}${r}`.toUpperCase(); } function assTime(ms: number): string { const total = Math.max(0, Math.round(ms / 10)); const hours = Math.floor(total / 360_000); const minutes = Math.floor((total % 360_000) / 6000); const seconds = Math.floor((total % 6000) / 100); const centis = total % 100; const pad = (v: number) => String(v).padStart(2, "0"); return `${hours}:${pad(minutes)}:${pad(seconds)}.${pad(centis)}`; } /** Plain text for an ASS event: no override blocks, no escape sequences. */ function assText(text: string): string { return text.replace(/\\/g, "\\⁠").replace(/\{/g, "{").replace(/\}/g, "}"); } export function toAss( cues: CaptionCue[], options: { width: number; height: number; profile: PlatformProfile; preset: CaptionStylePreset | null; fallbackTextColor: string; position: "lower" | "middle" | "upper"; maxCharsPerLine: number; wordHighlight: boolean; language?: string; }, ): string { const { width, height, profile, preset } = options; const zones = profile.safeZones; const margin = Math.round(height * 0.02); const alignment = { lower: 2, middle: 5, upper: 8 }[options.position]; const marginV = options.position === "upper" ? Math.round((zones.topPct / 100) * height) + margin : options.position === "lower" ? Math.round((zones.bottomPct / 100) * height) + margin : 0; const marginL = Math.round((zones.leftPct / 100) * width) + margin; const marginR = Math.round((zones.rightPct / 100) * width) + margin; const textColor = preset?.textColor ?? options.fallbackTextColor; const highlight = preset?.highlightColor ?? textColor; const karaoke = options.wordHighlight || preset?.template === "karaoke"; const bold = preset?.template === "bold_social"; const fontName = (preset?.fontFamily ?? "Sans").replace(/[,\r\n]/g, " "); const fontSize = Math.round(((preset?.sizePct ?? 5) / 100) * height); const background = preset?.backgroundColor; const style = [ "Default", fontName, fontSize, assColor(karaoke ? highlight : textColor), assColor(textColor), background ? assColor(background) : "&H00000000", background ? assColor(background, 0x40) : "&H80000000", bold ? -1 : 0, 0, 0, 0, 100, 100, 0, 0, background ? 3 : 1, bold ? 4 : 2, 0, alignment, marginL, marginR, marginV, 1, ].join(","); const transform = (text: string) => preset?.uppercase ? text.toLocaleUpperCase(options.language) : text; const events = cues.map((cue) => { let body: string; if (karaoke && cue.words.length > 0) { let cursor = cue.startMs; body = wrapWords(cue.words, options.maxCharsPerLine) .map((line) => line .map((word) => { const gap = Math.max(0, Math.round((word.startMs - cursor) / 10)); const length = Math.max(1, Math.round((word.endMs - word.startMs) / 10)); cursor = Math.max(cursor, word.endMs); return `${gap > 0 ? `{\\k${gap}}` : ""}{\\k${length}}${assText(transform(word.text))}`; }) .join(" "), ) .join("\\N"); } else { body = cueLines(transform(cue.text)).split("\n").map(assText).join("\\N"); } return `Dialogue: 0,${assTime(cue.startMs)},${assTime(cue.endMs)},Default,,0,0,0,,${body}`; }); return [ "[Script Info]", "ScriptType: v4.00+", `PlayResX: ${width}`, `PlayResY: ${height}`, "WrapStyle: 2", "ScaledBorderAndShadow: yes", "", "[V4+ Styles]", "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding", `Style: ${style}`, "", "[Events]", "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text", ...events, "", ].join("\n"); } /** Caption words on the output timeline, from each unmuted clip's transcript. */ export function outputCaptionWords( edl: Edl, transcripts: Map, ): CaptionWord[] { const offset = introMs(edl); const words: CaptionWord[] = []; for (const clip of sortedVideoClips(edl)) { if (clip.muted) continue; for (const word of transcripts.get(clip.assetId) ?? []) { const start = Math.max(word.startMs, clip.inMs); const end = Math.min(word.endMs, clip.outMs); if (word.startMs < clip.inMs || start >= clip.outMs || end < start) continue; words.push({ startMs: Math.round(offset + clip.atMs + (start - clip.inMs) / clip.speed), endMs: Math.round(offset + clip.atMs + (end - clip.inMs) / clip.speed), text: word.text, }); } } return words; } // --------------------------------------------------------------------------- // Compiler // --------------------------------------------------------------------------- const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo"; const XFADE_KIND = { crossfade: "fade", dip: "fadeblack", whip: "slideleft" } as const; const MAX_INLINE_GRAPH = 60_000; function encoderFamily(encoder: string): "h264" | "hevc" | null { if (/^(libx264|h264_)/.test(encoder)) return "h264"; if (/^(libx265|hevc_)/.test(encoder)) return "hevc"; return null; } function encodingArgs(input: CompileInput): string[] { const { profile, quality, videoEncoder } = input; const family = encoderFamily(videoEncoder); if (family !== profile.video.codec) { throw new Error(`encoder ${videoEncoder} does not produce ${profile.video.codec}`); } const draft = quality === "draft"; const maxKbps = draft ? Math.min(profile.video.maxBitrateKbps, 3000) : profile.video.maxBitrateKbps; const args = ["-c:v", videoEncoder]; if (videoEncoder === "libx264" || videoEncoder === "libx265") { args.push("-preset", draft ? "veryfast" : "medium", "-crf", draft ? "28" : "20"); if (videoEncoder === "libx264") args.push("-profile:v", "high"); } else { args.push("-b:v", `${Math.round(maxKbps * 0.8)}k`); } args.push("-maxrate", `${maxKbps}k`, "-bufsize", `${maxKbps * 2}k`); if (!videoEncoder.endsWith("_vaapi")) args.push("-pix_fmt", "yuv420p"); args.push( "-colorspace", "bt709", "-color_primaries", "bt709", "-color_trc", "bt709", "-color_range", "tv", ); // Drop SEI messages, which carry encoder name, version and settings. args.push( "-bsf:v", family === "h264" ? "filter_units=remove_types=6" : "filter_units=remove_types=39|40", ); if (family === "hevc") args.push("-tag:v", "hvc1"); args.push("-r", num(input.edl.output.fps, 3)); args.push("-c:a", "aac", "-b:a", draft ? "128k" : "192k", "-ar", "48000", "-ac", "2"); return args; } /** Strip every inherited tag, chapter, and encoder string; only the plan's title is written. */ function metadataArgs(title: string | undefined): string[] { const args = [ "-map_metadata", "-1", "-map_chapters", "-1", "-fflags", "+bitexact", "-flags:v", "+bitexact", "-flags:a", "+bitexact", ]; const clean = title?.replace(/[\0\r\n]+/g, " ").trim(); if (clean) args.push("-metadata", `title=${clean}`); return args; } class Inputs { readonly args: string[] = []; private count = 0; add(...args: string[]): number { if (args.some((arg) => arg.includes("\0"))) throw new Error("argument contains NUL"); this.args.push(...args); return this.count++; } } function captionPreset(edl: Edl, kit: BrandKit | null): CaptionStylePreset | null { const presets = kit?.captionPresets ?? []; return presets.find((p) => p.id === edl.captions?.presetId) ?? presets[0] ?? null; } function pickFont(kit: BrandKit | null, fonts: Map, role: string): string | null { const preferred = kit?.fonts.find((font) => font.role === role) ?? kit?.fonts[0]; if (preferred) { const path = fonts.get(preferred.artifactId) ?? fonts.get(preferred.family); if (path) return path; } return fonts.values().next().value ?? null; } /** * Pure compiler from a validated EDL to ffmpeg argv steps plus caption files. * No string ever reaches a shell; text is escaped for filtergraph and ASS, * every label comes from SafeId values. * * Loudness: a single loudnorm pass with linear=true targets the profile (or * plan) loudness and true peak. Without measured values loudnorm runs in its * dynamic mode; a renderer may run a measurement pass first and append the * measured_* options to that filter. * * Face, mark, and region redactions are applied before compilation: sources * already point at redacted intermediates. */ export function compileRenderPlan(input: CompileInput): RenderPlan { const edl = EdlSchema.parse(input.edl); const { profile, brandKit: kit, workDir } = input; const W = edl.output.width; const H = edl.output.height; const F = edl.output.fps; const totalMs = edlDurationMs(edl); const intro = introMs(edl); const mainMs = mainProgramMs(edl); const steps: RenderStep[] = []; const outputs: RenderPlan["outputs"] = []; const inputs = new Inputs(); const graph: string[] = []; let motionCount = 0; const motionInput = (key: string, template: MotionTemplateRef, width: number, height: number) => { let path = input.motionClips.get(key); if (!path) { motionCount += 1; path = joinPath(workDir, `motion-${motionCount}.mov`); steps.push({ kind: "motion", id: `motion-${motionCount}`, template, width, height, fps: F, outPath: path, }); } return inputs.add("-i", path); }; const finishVideo = (lengthMs: number) => [ `fps=${num(F, 3)}`, "format=yuv420p", "tpad=stop_mode=clone:stop_duration=2", `trim=duration=${sec(lengthMs)}`, "setpts=PTS-STARTPTS", ]; const finishAudio = (lengthMs: number) => [ AUDIO_FORMAT, "apad", `atrim=duration=${sec(lengthMs)}`, "asetpts=PTS-STARTPTS", ]; const silence = (label: string, lengthMs: number) => graph.push(`anullsrc=r=48000:cl=stereo,${finishAudio(lengthMs).join(",")}[${label}]`); // Per-clip chains. interface Part { v: string; a: string; lengthMs: number; } const clipParts = new Map(); for (const clip of sortedVideoClips(edl)) { const source = input.sources.get(clip.assetId); if (!source) throw new Error(`no source for asset in clip ${clip.id}`); const still = source.still === true || source.durationMs === 0; const sourceLength = clip.outMs - clip.inMs; const lengthMs = clipLengthMs(clip); const index = still ? inputs.add( "-loop", "1", "-framerate", num(F, 3), "-t", sec(sourceLength), "-i", source.path, ) : inputs.add("-ss", sec(clip.inMs), "-t", sec(sourceLength), "-i", source.path); const rotated = source.rotation === 90 || source.rotation === 270; const geometry: FrameGeometry = { sourceWidth: rotated ? source.height : source.width, sourceHeight: rotated ? source.width : source.height, outWidth: W, outHeight: H, durationMs: sourceLength, }; const video: string[] = []; if (clip.stabilize && !still) video.push("deshake"); if (clip.reframe === "none") { video.push(...fitFilters(W, H)); } else { const keyframes = clip.reframe === "auto" ? autoReframe( (input.subjects.get(clip.assetId) ?? []) .filter( (s) => s.tMs >= clip.inMs - AUTO_WINDOW_MS && s.tMs <= clip.outMs + AUTO_WINDOW_MS, ) .map((s) => ({ ...s, tMs: s.tMs - clip.inMs })), geometry, ).map((k) => ({ ...k, tMs: k.tMs + clip.inMs })) : clip.reframe; video.push(...cropFilters(keyframes, clip, geometry)); } video.push("setsar=1", `setpts=(PTS-STARTPTS)/${num(clip.speed, 6)}`); const color = clip.color; if (color) { const brightness = clamp(color.exposure * 0.1, -1, 1); if (brightness !== 0 || color.contrast !== 1 || color.saturation !== 1) { video.push( `eq=brightness=${num(brightness)}:contrast=${num(color.contrast)}:saturation=${num(color.saturation)}`, ); } if (color.temperature !== 0) { const shift = num(color.temperature * 0.1); const inverse = num(-color.temperature * 0.1); video.push(`colorbalance=rm=${shift}:bm=${inverse}`); } if (color.useBrandLut) { if (!input.files.lutPath) throw new Error(`clip ${clip.id} needs the brand LUT file`); video.push(`lut3d=file=${escapeFilterValue(input.files.lutPath)}`); } } video.push(...finishVideo(lengthMs)); graph.push(`[${index}:v]${video.join(",")}[v_${clip.id}]`); if (source.hasAudio && !clip.muted && !still) { const audio = ["asetpts=PTS-STARTPTS", ...atempoChain(clip.speed), "aresample=48000"]; graph.push(`[${index}:a]${[...audio, ...finishAudio(lengthMs)].join(",")}[a_${clip.id}]`); } else { silence(`a_${clip.id}`, lengthMs); } clipParts.set(clip.id, { v: `v_${clip.id}`, a: `a_${clip.id}`, lengthMs }); } // Main program: concatenate runs, blend where transitions ask for it, fill gaps with black. const blends = new Map( edl.transitions .filter((t) => BLENDED_TRANSITIONS.has(t.kind) && t.durationMs > 0) .map((t) => [`${t.fromClipId}>${t.toClipId}`, t] as const), ); let mixCount = 0; const nextLabel = (prefix: string) => `${prefix}${++mixCount}`; let accumulated: Part | null = null; let queue: Part[] = []; const flush = () => { const parts = accumulated ? [accumulated, ...queue] : queue; queue = []; if (parts.length <= 1) { accumulated = parts[0] ?? accumulated; return; } const v = nextLabel("cv"); const a = nextLabel("ca"); graph.push( `${parts.map((p) => `[${p.v}][${p.a}]`).join("")}concat=n=${parts.length}:v=1:a=1[${v}][${a}]`, ); accumulated = { v, a, lengthMs: parts.reduce((sum, p) => sum + p.lengthMs, 0) }; }; let cursor = 0; let previous: EdlVideoClip | null = null; for (const clip of sortedVideoClips(edl)) { const part = clipParts.get(clip.id)!; const blend = previous ? blends.get(`${previous.id}>${clip.id}`) : undefined; if (blend) { flush(); const base = accumulated as Part | null; if (!base) throw new Error("transition without a preceding clip"); const v = nextLabel("xv"); const a = nextLabel("xa"); const kind = XFADE_KIND[blend.kind as keyof typeof XFADE_KIND]; graph.push( `[${base.v}][${part.v}]xfade=transition=${kind}:duration=${sec(blend.durationMs)}:offset=${sec( base.lengthMs - blend.durationMs, )}[${v}]`, `[${base.a}][${part.a}]acrossfade=d=${sec(blend.durationMs)}:c1=tri:c2=tri[${a}]`, ); accumulated = { v, a, lengthMs: base.lengthMs + part.lengthMs - blend.durationMs }; } else { const gap = clip.atMs - cursor; if (gap > 1) { const v = nextLabel("gv"); const a = nextLabel("ga"); graph.push(`color=c=black:s=${W}x${H}:r=${num(F, 3)},${finishVideo(gap).join(",")}[${v}]`); silence(a, gap); queue.push({ v, a, lengthMs: gap }); } queue.push(part); } cursor = Math.max(cursor, clip.atMs + part.lengthMs); previous = clip; } flush(); const main = accumulated as Part | null; if (!main) throw new Error("plan has no video clips"); let mainAudio = main.a; const voice = edl.redactions.voice; if (voice) { const ratio = 2 ** (voice.semitones / 12); const label = nextLabel("vd"); const filter = voice.method === "pitch" ? `rubberband=pitch=${num(ratio, 6)}:formant=preserved` : `asetrate=${num(48000 * ratio, 2)},aresample=48000,atempo=${num(1 / ratio, 6)}`; graph.push(`[${mainAudio}]${filter},${finishAudio(main.lengthMs).join(",")}[${label}]`); mainAudio = label; } // Intro and outro. const program: Part[] = []; const bookend = (key: "intro" | "outro", ref: MotionTemplateRef | undefined) => { if (!ref || ref.durationMs <= 0) return; const index = motionInput(key, ref, W, H); const v = `${key}_v`; const a = `${key}_a`; graph.push( `[${index}:v]${[...fitFilters(W, H), "setsar=1", ...finishVideo(ref.durationMs)].join(",")}[${v}]`, ); silence(a, ref.durationMs); program.push({ v, a, lengthMs: ref.durationMs }); }; bookend("intro", edl.intro); program.push({ v: main.v, a: mainAudio, lengthMs: main.lengthMs }); bookend("outro", edl.outro); let videoLabel = main.v; let programAudio = mainAudio; if (program.length > 1) { videoLabel = "prog_v"; programAudio = "prog_a"; graph.push( `${program.map((p) => `[${p.v}][${p.a}]`).join("")}concat=n=${program.length}:v=1:a=1[prog_v][prog_a]`, ); } // Overlays on the output timeline. const zones = profile.safeZones; const margin = Math.round(W * 0.02); const safeLeft = Math.round((zones.leftPct / 100) * W) + margin; const safeRight = Math.round((zones.rightPct / 100) * W) + margin; const safeTop = Math.round((zones.topPct / 100) * H) + margin; const safeBottom = Math.round((zones.bottomPct / 100) * H) + margin; const textColor = brandColor(kit?.colors.text, "white"); const overlays = [...edl.overlays].sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)); const imageOverlay = ( path: string, overlay: EdlOverlay, widthPx: number, opacity: number, position: { x: string; y: string }, ) => { const index = inputs.add("-i", path); const label = nextLabel("img"); const out = nextLabel("ov"); const alpha = opacity < 1 ? `,colorchannelmixer=aa=${num(opacity)}` : ""; graph.push( `[${index}:v]format=rgba,scale=${even(widthPx)}:-2${alpha}[${label}]`, `[${videoLabel}][${label}]overlay=x=${escapeFilterValue(position.x)}:y=${escapeFilterValue( position.y, )}:enable=${enableBetween(overlay.atMs, overlay.atMs + overlay.durationMs)}[${out}]`, ); videoLabel = out; }; const boxPosition = (box: NormalizedBox) => ({ x: String(Math.round(box.x * W)), y: String(Math.round(box.y * H)), }); const cornerPosition = (corner: string) => ({ x: corner.endsWith("left") ? String(safeLeft) : corner === "center" ? "(main_w-overlay_w)/2" : `main_w-overlay_w-${safeRight}`, y: corner.startsWith("top") ? String(safeTop) : corner === "center" ? "(main_h-overlay_h)/2" : `main_h-overlay_h-${safeBottom}`, }); for (const overlay of overlays) { const endMs = overlay.atMs + overlay.durationMs; const motionPath = input.motionClips.get(overlay.id); if (overlay.motion || motionPath) { const box = overlay.box; const width = box ? even(box.w * W) : W; const height = box ? even(box.h * H) : H; const template = overlay.motion ?? { templateId: "overlay", params: {}, durationMs: overlay.durationMs, }; const index = motionInput(overlay.id, template, width, height); const label = nextLabel("mo"); const out = nextLabel("ov"); const position = box ? boxPosition(box) : { x: "0", y: "0" }; graph.push( `[${index}:v]format=yuva420p,scale=${width}:${height},setpts=PTS-STARTPTS+${sec(overlay.atMs)}/TB[${label}]`, `[${videoLabel}][${label}]overlay=x=${position.x}:y=${position.y}:eof_action=pass:enable=${enableBetween( overlay.atMs, endMs, )}[${out}]`, ); videoLabel = out; continue; } if (overlay.kind === "watermark") { const watermark = kit?.watermark; if (!watermark || !input.files.watermarkPath) { throw new Error(`overlay ${overlay.id} needs the brand watermark file`); } const width = overlay.box ? overlay.box.w * W : watermark.scale * W; imageOverlay( input.files.watermarkPath, overlay, width, watermark.opacity, overlay.box ? boxPosition(overlay.box) : cornerPosition(watermark.position), ); continue; } if (overlay.kind === "logo") { if (!input.files.logoPath) throw new Error(`overlay ${overlay.id} needs a logo file`); const width = overlay.box ? overlay.box.w * W : 0.15 * W; imageOverlay( input.files.logoPath, overlay, width, 1, overlay.box ? boxPosition(overlay.box) : cornerPosition("top_right"), ); continue; } if (overlay.kind === "progress_bar") { const barHeight = Math.max(4, even(H * 0.006)); const label = nextLabel("pb"); const out = nextLabel("ov"); const y = overlay.box ? Math.round(overlay.box.y * H) : H - safeBottom - barHeight; const x = `-overlay_w+main_w*min(1,(t-${sec(overlay.atMs)})/${sec(Math.max(1, overlay.durationMs))})`; graph.push( `color=c=${brandColor(kit?.colors.accent, "white")}:s=${W}x${barHeight}:r=${num(F, 3)}:d=${sec(totalMs)}[${label}]`, `[${videoLabel}][${label}]overlay=x=${escapeFilterValue(x)}:y=${y}:enable=${enableBetween( overlay.atMs, endMs, )}[${out}]`, ); videoLabel = out; continue; } // text, lower_third, cta_end_card const text = overlay.text?.trim(); if (!text) throw new Error(`overlay ${overlay.id} has no text and no motion clip`); const lines = text.split(/\r?\n/).length; const fontSize = overlay.box ? Math.max(8, Math.round((overlay.box.h * H * 0.8) / lines)) : Math.round(H * (overlay.kind === "cta_end_card" ? 0.05 : 0.04)); const position = overlay.box ? boxPosition(overlay.box) : { x: "(w-text_w)/2", y: overlay.kind === "lower_third" ? `h-text_h-${safeBottom + Math.round(H * 0.08)}` : overlay.kind === "cta_end_card" ? "(h-text_h)/2" : String(safeTop + Math.round(H * 0.05)), }; const font = pickFont( kit, input.files.fonts, overlay.kind === "cta_end_card" ? "heading" : "body", ); const options = [ ...(font ? [`fontfile=${escapeFilterValue(font)}`] : []), `text=${escapeFilterText(text)}`, `fontsize=${fontSize}`, `fontcolor=${textColor}`, `x=${escapeFilterValue(position.x)}`, `y=${escapeFilterValue(position.y)}`, ]; if (overlay.kind !== "text") { options.push( "box=1", `boxcolor=${brandColor(kit?.colors.primary, "black")}@0.85`, `boxborderw=${Math.round(fontSize * 0.4)}`, ); } options.push(`enable=${enableBetween(overlay.atMs, endMs)}`); const out = nextLabel("ov"); graph.push(`[${videoLabel}]drawtext=${options.join(":")}[${out}]`); videoLabel = out; } // Captions. const captions = edl.captions; if (captions && captions.mode !== "none") { const cues = buildCaptionCues(outputCaptionWords(edl, input.transcripts), { maxCharsPerLine: captions.maxCharsPerLine, maxLines: profile.captions.maxLines, }); if (cues.length > 0) { if (captions.mode === "burned" || captions.mode === "both") { const preset = captionPreset(edl, kit); const assPath = joinPath(workDir, "captions.ass"); steps.push({ kind: "write_file", id: "captions-ass", path: assPath, content: toAss(cues, { width: W, height: H, profile, preset, fallbackTextColor: kit?.colors.text ?? "#FFFFFF", position: captions.position, maxCharsPerLine: captions.maxCharsPerLine, wordHighlight: captions.wordHighlight, language: captions.language, }), }); const font = pickFont(kit, input.files.fonts, "caption"); const fontsDir = font ? `:fontsdir=${escapeFilterValue(dirName(font))}` : ""; const out = nextLabel("cap"); graph.push(`[${videoLabel}]ass=filename=${escapeFilterValue(assPath)}${fontsDir}[${out}]`); videoLabel = out; } if (captions.mode === "sidecar" || captions.mode === "both") { const srt = joinPath(workDir, "captions.srt"); const vtt = joinPath(workDir, "captions.vtt"); steps.push( { kind: "write_file", id: "captions-srt", path: srt, content: toSrt(cues) }, { kind: "write_file", id: "captions-vtt", path: vtt, content: toVtt(cues) }, ); outputs.push({ kind: "subtitles_srt", path: srt }, { kind: "subtitles_vtt", path: vtt }); } } } const finalVideo = input.videoEncoder.endsWith("_vaapi") ? "format=nv12,hwupload" : "format=yuv420p"; graph.push(`[${videoLabel}]${finalVideo}[vout]`); // Audio: program (+ voice-over) is speech; clips marked duckUnderSpeech are compressed under it. const speech: string[] = [programAudio]; const ducked: string[] = []; const beds: string[] = []; const audioClips = [...edl.tracks.audio].sort( (a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id), ); for (const clip of audioClips) { const path = input.files.audio.get(clip.artifactId); if (!path) throw new Error(`no file for audio clip ${clip.id}`); const length = clip.outMs - clip.inMs; const index = inputs.add("-ss", sec(clip.inMs), "-t", sec(length), "-i", path); graph.push(`[${index}:a]${audioClipFilters(clip, length).join(",")}[ac_${clip.id}]`); const label = `ac_${clip.id}`; if (clip.kind === "voiceover") speech.push(label); else if (clip.duckUnderSpeech) ducked.push(label); else beds.push(label); } const mix = (labels: string[], out: string) => { if (labels.length === 1) { graph.push(`[${labels[0]}]anull[${out}]`); } else { graph.push( `${labels.map((l) => `[${l}]`).join("")}amix=inputs=${labels.length}:duration=first:normalize=0[${out}]`, ); } }; const finalInputs: string[] = []; if (ducked.length > 0) { mix(speech, "speech"); graph.push("[speech]asplit=2[speech_mix][speech_key]"); mix(ducked, "duck_bus"); graph.push( "[duck_bus][speech_key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400:makeup=1[ducked]", ); finalInputs.push("speech_mix", "ducked"); } else { finalInputs.push(...speech); } finalInputs.push(...beds); const targetLufs = edl.targetLufs ?? profile.audio.targetLufs; const loudnorm = `loudnorm=I=${num(targetLufs, 1)}:TP=${num(profile.audio.truePeakDb, 1)}:LRA=11:linear=true`; const mixed = finalInputs.length > 1 ? "mixed" : finalInputs[0]!; if (finalInputs.length > 1) mix(finalInputs, "mixed"); graph.push(`[${mixed}]${loudnorm},aresample=48000,${AUDIO_FORMAT}[aout]`); // Render step. const graphText = graph.join(";"); const graphArgs = graphText.length > MAX_INLINE_GRAPH ? ["-filter_complex_script", joinPath(workDir, "filtergraph.txt")] : ["-filter_complex", graphText]; if (graphText.length > MAX_INLINE_GRAPH) { steps.push({ kind: "write_file", id: "filtergraph", path: joinPath(workDir, "filtergraph.txt"), content: graphText, }); } const videoPath = joinPath(workDir, "output.mp4"); steps.push({ kind: "ffmpeg", id: "render", argv: [ "-hide_banner", "-nostdin", "-y", ...inputs.args, ...graphArgs, "-map", "[vout]", "-map", "[aout]", ...encodingArgs(input), ...metadataArgs(edl.metadata.title), "-t", sec(totalMs), "-movflags", "+faststart", videoPath, ], produces: [videoPath], durationMs: totalMs, }); outputs.unshift({ kind: "video", path: videoPath }); if (profile.export.poster) { const posterPath = joinPath(workDir, "poster.jpg"); const at = clamp(input.posterAtMs ?? intro + mainMs / 3, 0, Math.max(0, totalMs - 1)); steps.push({ kind: "ffmpeg", id: "poster", argv: [ "-hide_banner", "-nostdin", "-y", "-ss", sec(at), "-i", videoPath, "-frames:v", "1", "-map_metadata", "-1", "-fflags", "+bitexact", "-flags:v", "+bitexact", "-q:v", "2", posterPath, ], produces: [posterPath], durationMs: 0, }); outputs.push({ kind: "poster", path: posterPath }); } return { quality: input.quality, workDir, steps, outputs }; } function audioClipFilters(clip: EdlAudioClip, lengthMs: number): string[] { const filters = ["asetpts=PTS-STARTPTS", "aresample=48000", AUDIO_FORMAT]; if (clip.gainDb !== 0) filters.push(`volume=${num(clip.gainDb, 2)}dB`); if (clip.fadeInMs > 0) filters.push(`afade=t=in:st=0:d=${sec(clip.fadeInMs)}`); if (clip.fadeOutMs > 0) { filters.push( `afade=t=out:st=${sec(Math.max(0, lengthMs - clip.fadeOutMs))}:d=${sec(clip.fadeOutMs)}`, ); } if (clip.atMs > 0) filters.push(`adelay=${clip.atMs}|${clip.atMs}`); return filters; }