evOo-local/packages/adapters/src/media-ffmpeg-renderer.test.ts
Claude 3ebe0fe47b
feat(media): ffmpeg renderer, render job and real render integration test
Fail closed when anonymization cannot prepare every clip, retry a failing
hardware encoder in software, verify each ffmpeg output exists, and fail
renders whose video or poster still carries source metadata. Cover parsing,
encoder detection, two-pass loudness, abort and cleanup, the render job
(auto-fix, cancel, outputs) on memory and Postgres, and a synthetic
16:9 to 9:16 render plus a region blur check with ffprobe.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w
2026-10-03 18:44:58 +00:00

585 lines
20 KiB
TypeScript

import {
chmod,
mkdir,
mkdtemp,
readdir,
readFile,
realpath,
rm,
stat,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import type { MediaRenderProgress } from "@rakazo/adapter-kit";
import type { RenderPlan } from "@rakazo/contracts";
import { EdlSchema } from "@rakazo/contracts";
import type { CompileInput } from "@rakazo/media";
import { compileRenderPlan, findMetadataLeaks } from "@rakazo/media";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import {
FfmpegMediaRenderer,
findTwoPassLoudnorm,
parseEbur128Summary,
parseFrameRate,
parseLoudnormJson,
parseProbeReport,
} from "./media-ffmpeg-renderer.js";
import { MediaToolMissingError } from "./media-folder-source-probe.js";
import { hasMediaTools, renderTestProfile, writeSyntheticClip } from "./media-render.fixtures.js";
describe("ffmpeg output parsing", () => {
it("parses frame rates", () => {
expect(parseFrameRate("30000/1001")).toBe(29.97);
expect(parseFrameRate("25/1")).toBe(25);
expect(parseFrameRate("0/0")).toBe(0);
expect(parseFrameRate(undefined)).toBe(0);
expect(parseFrameRate("abc")).toBe(0);
});
it("flattens format and stream tags and falls back to the file size", () => {
const report = parseProbeReport(
{
streams: [
{
codec_type: "video",
codec_name: "h264",
width: 640,
height: 360,
r_frame_rate: "30/1",
},
{ codec_type: "audio", codec_name: "aac", tags: { language: "und" } },
],
format: { duration: "2.5", tags: { title: "Example", location: "+00+000/" } },
},
1234,
);
expect(report).toEqual({
durationMs: 2500,
width: 640,
height: 360,
fps: 30,
videoCodec: "h264",
audioCodec: "aac",
sizeBytes: 1234,
integratedLufs: null,
truePeakDb: null,
tags: {
"format.title": "Example",
"format.location": "+00+000/",
"stream.1.language": "und",
},
});
});
it("reads the ebur128 summary and treats the silence gate as no programme", () => {
const stderr = `[Parsed_ebur128_0 @ 0x1] Summary:
Integrated loudness:
I: -14.2 LUFS
Threshold: -24.3 LUFS
True peak:
Peak: -1.4 dBFS`;
expect(parseEbur128Summary(stderr)).toEqual({ integratedLufs: -14.2, truePeakDb: -1.4 });
expect(parseEbur128Summary("Summary:\n I: -70.0 LUFS\n True peak:\n Peak: -inf dBFS")).toEqual({
integratedLufs: null,
truePeakDb: null,
});
expect(parseEbur128Summary("no summary")).toEqual({ integratedLufs: null, truePeakDb: null });
});
it("finds exactly one linear loudnorm without measured values", () => {
const graph = "[a]anull[b];[b]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]";
const site = findTwoPassLoudnorm(graph);
expect(site?.options).toBe("I=-14:TP=-1:LRA=11:linear=true");
expect(graph.slice(site!.start, site!.end)).toBe(site!.options);
expect(findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true:measured_I=-20[b]")).toBeNull();
expect(findTwoPassLoudnorm("[a]loudnorm=I=-14[b]")).toBeNull();
expect(
findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true[b];[c]loudnorm=I=-14:linear=true[d]"),
).toBeNull();
expect(findTwoPassLoudnorm("[a]anull[b]")).toBeNull();
});
it("parses the loudnorm measurement block", () => {
const stderr = `[Parsed_loudnorm_3 @ 0x55]
{
\t"input_i" : "-31.20",
\t"input_tp" : "-20.11",
\t"input_lra" : "0.00",
\t"input_thresh" : "-41.20",
\t"output_i" : "-14.02",
\t"output_tp" : "-1.00",
\t"output_lra" : "0.00",
\t"output_thresh" : "-24.02",
\t"normalization_type" : "dynamic",
\t"target_offset" : "0.02"
}`;
expect(parseLoudnormJson(stderr)).toEqual({
inputI: -31.2,
inputTp: -20.11,
inputLra: 0,
inputThresh: -41.2,
targetOffset: 0.02,
});
expect(parseLoudnormJson(stderr.replace('"-31.20"', '"-inf"'))).toBeNull();
expect(parseLoudnormJson("nothing")).toBeNull();
});
});
describe("hardware encoder detection", () => {
let dir: string;
beforeAll(async () => {
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-hw-")));
});
afterAll(async () => {
await rm(dir, { recursive: true, force: true });
});
/** A fake ffmpeg that lists encoders and succeeds only for the given test encodes. */
async function fakeFfmpeg(name: string, working: string[]): Promise<string> {
const file = path.join(dir, name);
const cases = working.map((encoder) => `*" ${encoder} "*) exit 0 ;;`).join("\n ");
await writeFile(
file,
`#!/bin/sh
case " $* " in
*" -encoders "*)
echo " V....D libx264 libx264"
echo " V....D h264_nvenc NVIDIA NVENC H.264 encoder"
echo " V....D hevc_nvenc NVIDIA NVENC hevc encoder"
echo " V....D h264_vaapi H.264/AVC (VAAPI)"
exit 0 ;;
${cases}
esac
echo "No device available" >&2
exit 1
`,
);
await chmod(file, 0o755);
return file;
}
it("keeps only listed encoders whose one-frame test encode works", async () => {
const renderer = new FfmpegMediaRenderer({
ffmpegPath: await fakeFfmpeg("ffmpeg-nvenc", ["h264_nvenc"]),
vaapiDevice: path.join(dir, "no-render-node"),
});
expect(await renderer.hardwareEncoders()).toEqual(["h264_nvenc"]);
});
it("falls back to software when no hardware encoder works or ffmpeg is missing", async () => {
const broken = new FfmpegMediaRenderer({ ffmpegPath: await fakeFfmpeg("ffmpeg-none", []) });
expect(await broken.hardwareEncoders()).toEqual([]);
const missing = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") });
expect(await missing.hardwareEncoders()).toEqual([]);
});
it("skips VAAPI encoders without a render node", async () => {
const renderer = new FfmpegMediaRenderer({
ffmpegPath: await fakeFfmpeg("ffmpeg-vaapi", ["h264_vaapi"]),
vaapiDevice: path.join(dir, "no-render-node"),
});
expect(await renderer.hardwareEncoders()).toEqual([]);
});
it.skipIf(!hasMediaTools)("never reports an encoder this host cannot run", async () => {
const renderer = new FfmpegMediaRenderer();
for (const encoder of await renderer.hardwareEncoders()) {
expect(encoder).toMatch(/^(h264|hevc)_(vaapi|videotoolbox|nvenc)$/);
}
});
});
describe("plan safety", () => {
let dir: string;
beforeAll(async () => {
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-plan-")));
});
afterAll(async () => {
await rm(dir, { recursive: true, force: true });
});
it("refuses writes outside the work directory", async () => {
const renderer = new FfmpegMediaRenderer({ root: dir });
const workDir = path.join(dir, "work");
const plan: RenderPlan = {
quality: "draft",
workDir,
steps: [
{ kind: "write_file", id: "escape", path: path.join(dir, "outside.txt"), content: "x" },
],
outputs: [],
};
await expect(renderer.render(plan, {})).rejects.toThrow("outside");
await expect(stat(path.join(dir, "outside.txt"))).rejects.toThrow();
});
it("refuses a work directory outside the media root", async () => {
const renderer = new FfmpegMediaRenderer({ root: path.join(dir, "root") });
await mkdir(path.join(dir, "root"), { recursive: true });
const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] };
await expect(renderer.render(plan, {})).rejects.toThrow("outside");
});
it("requires motion, inpaint and identity steps to be prepared first", async () => {
const renderer = new FfmpegMediaRenderer();
const plan: RenderPlan = {
quality: "final",
workDir: path.join(dir, "motion"),
steps: [
{
kind: "motion",
id: "intro",
template: { templateId: "intro-minimal", params: {}, durationMs: 1000 },
width: 320,
height: 180,
fps: 30,
outPath: path.join(dir, "motion", "intro.mov"),
},
],
outputs: [],
};
await expect(renderer.render(plan, {})).rejects.toThrow("was not prepared");
});
it("reports a missing ffmpeg instead of pretending to render", async () => {
const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") });
const out = path.join(dir, "missing", "out.mp4");
const plan: RenderPlan = {
quality: "draft",
workDir: path.join(dir, "missing"),
steps: [{ kind: "ffmpeg", id: "render", argv: ["-version"], produces: [out], durationMs: 1 }],
outputs: [{ kind: "video", path: out }],
};
await expect(renderer.render(plan, {})).rejects.toBeInstanceOf(MediaToolMissingError);
});
it("rejects an already aborted render without spawning anything", async () => {
const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") });
const controller = new AbortController();
controller.abort();
const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] };
await expect(renderer.render(plan, { signal: controller.signal })).rejects.toMatchObject({
name: "AbortError",
});
});
});
describe("two-pass loudness", () => {
let dir: string;
let ffmpegPath: string;
let log: string;
const GRAPH = "[0:a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]";
beforeAll(async () => {
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-loudnorm-")));
log = path.join(dir, "calls.log");
ffmpegPath = path.join(dir, "ffmpeg");
// Logs each call; the measurement pass prints loudnorm's JSON, the render
// pass writes its output file (the last argument).
await writeFile(
ffmpegPath,
`#!/bin/sh
echo "$*" >> "${log}"
case " $* " in
*print_format=json*|*.measure.txt*)
printf '[Parsed_loudnorm_0 @ 0x1] \\n{\\n "input_i" : "-31.20",\\n "input_tp" : "-20.11",\\n "input_lra" : "0.00",\\n "input_thresh" : "-41.20",\\n "target_offset" : "0.02"\\n}\\n' >&2
exit 0 ;;
esac
for last; do :; done
[ "$last" = "-" ] || echo rendered > "$last"
`,
);
await chmod(ffmpegPath, 0o755);
});
afterAll(async () => {
await rm(dir, { recursive: true, force: true });
});
async function calls(): Promise<string[]> {
return (await readFile(log, "utf8")).trim().split("\n");
}
it("measures first, then renders with the measured values", async () => {
await rm(log, { force: true });
const workDir = path.join(dir, "inline");
const out = path.join(workDir, "out.m4a");
const plan: RenderPlan = {
quality: "final",
workDir,
steps: [
{
kind: "ffmpeg",
id: "render",
argv: ["-i", "in.wav", "-filter_complex", GRAPH, "-map", "[aout]", "-t", "2", out],
produces: [out],
durationMs: 2000,
},
],
outputs: [],
};
await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {});
const [measure, render] = await calls();
expect(measure).toContain("linear=true:print_format=json");
expect(measure).toContain("-map [aout] -t 2 -f null -");
expect(measure).not.toContain(out);
expect(render).toContain(
"linear=true:measured_I=-31.20:measured_TP=-20.11:measured_LRA=0.00:measured_thresh=-41.20:offset=0.02,aresample=48000",
);
expect(render?.endsWith(out)).toBe(true);
});
it("rewrites a filtergraph script and removes the measurement script", async () => {
await rm(log, { force: true });
const workDir = path.join(dir, "script");
const script = path.join(workDir, "filtergraph.txt");
const out = path.join(workDir, "out.m4a");
const plan: RenderPlan = {
quality: "final",
workDir,
steps: [
{ kind: "write_file", id: "filtergraph", path: script, content: GRAPH },
{
kind: "ffmpeg",
id: "render",
argv: ["-i", "in.wav", "-filter_complex_script", script, "-map", "[aout]", out],
produces: [out],
durationMs: 2000,
},
],
outputs: [],
};
await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {});
const [measure] = await calls();
expect(measure).toContain(path.join(workDir, "filtergraph.measure.txt"));
expect(await readFile(script, "utf8")).toContain("measured_I=-31.20");
expect((await readdir(workDir)).sort()).toEqual(["filtergraph.txt", "out.m4a"]);
});
it("fails a step whose output was never written and removes what it wrote", async () => {
const workDir = path.join(dir, "silent");
const out = path.join(workDir, "out.m4a");
const script = path.join(workDir, "notes.txt");
const plan: RenderPlan = {
quality: "draft",
workDir,
steps: [
{ kind: "write_file", id: "notes", path: script, content: "x" },
{
kind: "ffmpeg",
id: "render",
// The fake writes its last argument; the declared output stays missing.
argv: ["-i", "in.wav", path.join(workDir, "elsewhere.m4a")],
produces: [out],
durationMs: 1000,
},
],
outputs: [],
};
await expect(new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {})).rejects.toThrow(
"wrote no out.m4a",
);
await expect(stat(script)).rejects.toThrow();
});
});
describe.skipIf(!hasMediaTools)("real ffmpeg renders", () => {
let dir: string;
let source: string;
const renderer = () => new FfmpegMediaRenderer({ root: dir });
const profile = renderTestProfile();
beforeAll(async () => {
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-ffmpeg-")));
source = path.join(dir, "assets", "source.mp4");
// Deliberately quiet (-36 dBFS sine) so loudness normalization has real work to do.
await writeSyntheticClip(source, { seconds: 6, width: 320, height: 180, volumeDb: -36 });
});
afterAll(async () => {
await rm(dir, { recursive: true, force: true });
});
function compile(workDir: string, overrides: Partial<CompileInput> = {}): RenderPlan {
const edl = EdlSchema.parse({
edlVersion: 1,
output: {
profileId: profile.id,
width: 180,
height: 320,
fps: 30,
durationTargetMs: 4000,
durationHardMinMs: 1000,
durationHardMaxMs: 120_000,
},
tracks: {
video: [
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" },
{ id: "c2", assetId: "asset-a", inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" },
],
},
captions: { mode: "both" },
overlays: [{ id: "t1", kind: "text", atMs: 500, durationMs: 1500, text: "Example: 50%" }],
metadata: { title: "Example teaser" },
});
return compileRenderPlan({
edl,
profile,
brandKit: null,
quality: "draft",
workDir,
sources: new Map([
[
"asset-a",
{
path: source,
width: 320,
height: 180,
durationMs: 6000,
fps: 30,
hasAudio: true,
rotation: 0,
},
],
]),
transcripts: new Map([
[
"asset-a",
[
{ startMs: 100, endMs: 600, text: "Hello" },
{ startMs: 650, endMs: 1200, text: "there." },
{ startMs: 3100, endMs: 3800, text: "Bye." },
],
],
]),
subjects: new Map([
[
"asset-a",
[
{ tMs: 0, cx: 0.3, cy: 0.5, strength: 1 },
{ tMs: 5000, cx: 0.7, cy: 0.5, strength: 1 },
],
],
]),
motionClips: new Map(),
files: { lutPath: null, watermarkPath: null, fonts: new Map(), audio: new Map() },
videoEncoder: "libx264",
...overrides,
});
}
it("renders 16:9 to 9:16 with captions, overlay, two-pass loudness and clean metadata", async () => {
const workDir = path.join(dir, "render-ok");
const plan = compile(workDir);
const progress: MediaRenderProgress[] = [];
await renderer().render(plan, { onProgress: (p) => progress.push(p) });
const video = plan.outputs.find((output) => output.kind === "video")!.path;
const report = await renderer().probe(video, { measureLoudness: true });
expect(report.width).toBe(180);
expect(report.height).toBe(320);
expect(report.fps).toBe(30);
expect(report.videoCodec).toBe("h264");
expect(report.audioCodec).toBe("aac");
expect(Math.abs(report.durationMs - 4000)).toBeLessThanOrEqual(1000 / 30 + 1);
expect(report.integratedLufs).not.toBeNull();
expect(Math.abs(report.integratedLufs! - profile.audio.targetLufs)).toBeLessThanOrEqual(1);
expect(report.truePeakDb!).toBeLessThanOrEqual(profile.audio.truePeakDb + 0.5);
expect(findMetadataLeaks(report.tags)).toEqual([]);
expect(report.tags["format.title"]).toBe("Example teaser");
const keys = Object.keys(report.tags).join(" ").toLowerCase();
for (const leak of ["location", "creation_time", "make", "model"]) {
expect(keys).not.toContain(leak);
}
expect(Object.values(report.tags).join(" ")).not.toContain("ExampleCam");
const poster = plan.outputs.find((output) => output.kind === "poster");
if (poster) expect(findMetadataLeaks((await renderer().probe(poster.path)).tags)).toEqual([]);
expect(progress.length).toBeGreaterThan(0);
expect(progress.at(-1)!.progress).toBeCloseTo(1, 5);
for (let i = 1; i < progress.length; i++) {
expect(progress[i]!.progress).toBeGreaterThanOrEqual(progress[i - 1]!.progress - 1e-9);
}
// The measurement pass leaves nothing behind.
expect((await readdir(workDir)).sort()).toEqual(
["captions.ass", "captions.srt", "captions.vtt", "output.mp4", "poster.jpg"].filter(
(name) => name !== "poster.jpg" || poster,
),
);
});
it("extracts an EXIF-free PNG frame", async () => {
const out = path.join(dir, "frames", "frame.png");
await mkdir(path.dirname(out), { recursive: true });
await renderer().extractFrame({ path: source, tMs: 1000, outPath: out, width: 161 });
const report = await renderer().probe(out);
expect(report.width).toBe(160);
expect(report.videoCodec).toBe("png");
expect(findMetadataLeaks(report.tags)).toEqual([]);
await expect(
renderer().extractFrame({ path: source, tMs: 1000, outPath: path.join(dir, "frame.jpg") }),
).rejects.toThrow("PNG");
await expect(
renderer().extractFrame({ path: source, tMs: 60_000, outPath: path.join(dir, "late.png") }),
).rejects.toThrow();
});
it("refuses to probe outside the media root", async () => {
await expect(renderer().probe("/etc/hostname")).rejects.toThrow("outside");
await expect(renderer().probe("relative.mp4")).rejects.toThrow("absolute");
});
it("aborts a running render, kills ffmpeg and removes partial outputs", async () => {
const workDir = path.join(dir, "render-abort");
const long = await mkdtemp(path.join(dir, "long-"));
const longSource = path.join(long, "long.mp4");
await writeSyntheticClip(longSource, { seconds: 30, width: 1280, height: 720 });
const plan = compile(workDir, {
edl: EdlSchema.parse({
edlVersion: 1,
output: {
profileId: profile.id,
width: 1080,
height: 1920,
fps: 30,
durationTargetMs: 30_000,
durationHardMinMs: 1000,
durationHardMaxMs: 120_000,
},
tracks: {
video: [
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 30_000, atMs: 0, reframe: "none" },
],
},
metadata: {},
}),
sources: new Map([
[
"asset-a",
{
path: longSource,
width: 1280,
height: 720,
durationMs: 30_000,
fps: 30,
hasAudio: true,
rotation: 0,
},
],
]),
});
const controller = new AbortController();
const started = Date.now();
const rendering = renderer().render(plan, {
signal: controller.signal,
onProgress: (p) => {
if (p.progress > 0.01) controller.abort();
},
});
await expect(rendering).rejects.toMatchObject({ name: "AbortError" });
expect(Date.now() - started).toBeLessThan(25_000);
await expect(stat(path.join(workDir, "output.mp4"))).rejects.toThrow();
});
});