feat(media): ffmpeg renderer, render job and real render integration test
Fail closed when anonymization cannot prepare every clip, retry a failing hardware encoder in software, verify each ffmpeg output exists, and fail renders whose video or poster still carries source metadata. Cover parsing, encoder detection, two-pass loudness, abort and cleanup, the render job (auto-fix, cancel, outputs) on memory and Postgres, and a synthetic 16:9 to 9:16 render plus a region blur check with ffprobe. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w
This commit is contained in:
Claude
parent
393137be9a
commit
3ebe0fe47b
584
packages/adapters/src/media-ffmpeg-renderer.test.ts
Normal file
584
packages/adapters/src/media-ffmpeg-renderer.test.ts
Normal file
@ -0,0 +1,584 @@
|
||||
import {
|
||||
chmod,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readdir,
|
||||
readFile,
|
||||
realpath,
|
||||
rm,
|
||||
stat,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { MediaRenderProgress } from "@rakazo/adapter-kit";
|
||||
import type { RenderPlan } from "@rakazo/contracts";
|
||||
import { EdlSchema } from "@rakazo/contracts";
|
||||
import type { CompileInput } from "@rakazo/media";
|
||||
import { compileRenderPlan, findMetadataLeaks } from "@rakazo/media";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
FfmpegMediaRenderer,
|
||||
findTwoPassLoudnorm,
|
||||
parseEbur128Summary,
|
||||
parseFrameRate,
|
||||
parseLoudnormJson,
|
||||
parseProbeReport,
|
||||
} from "./media-ffmpeg-renderer.js";
|
||||
import { MediaToolMissingError } from "./media-folder-source-probe.js";
|
||||
import { hasMediaTools, renderTestProfile, writeSyntheticClip } from "./media-render.fixtures.js";
|
||||
|
||||
describe("ffmpeg output parsing", () => {
|
||||
it("parses frame rates", () => {
|
||||
expect(parseFrameRate("30000/1001")).toBe(29.97);
|
||||
expect(parseFrameRate("25/1")).toBe(25);
|
||||
expect(parseFrameRate("0/0")).toBe(0);
|
||||
expect(parseFrameRate(undefined)).toBe(0);
|
||||
expect(parseFrameRate("abc")).toBe(0);
|
||||
});
|
||||
|
||||
it("flattens format and stream tags and falls back to the file size", () => {
|
||||
const report = parseProbeReport(
|
||||
{
|
||||
streams: [
|
||||
{
|
||||
codec_type: "video",
|
||||
codec_name: "h264",
|
||||
width: 640,
|
||||
height: 360,
|
||||
r_frame_rate: "30/1",
|
||||
},
|
||||
{ codec_type: "audio", codec_name: "aac", tags: { language: "und" } },
|
||||
],
|
||||
format: { duration: "2.5", tags: { title: "Example", location: "+00+000/" } },
|
||||
},
|
||||
1234,
|
||||
);
|
||||
expect(report).toEqual({
|
||||
durationMs: 2500,
|
||||
width: 640,
|
||||
height: 360,
|
||||
fps: 30,
|
||||
videoCodec: "h264",
|
||||
audioCodec: "aac",
|
||||
sizeBytes: 1234,
|
||||
integratedLufs: null,
|
||||
truePeakDb: null,
|
||||
tags: {
|
||||
"format.title": "Example",
|
||||
"format.location": "+00+000/",
|
||||
"stream.1.language": "und",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("reads the ebur128 summary and treats the silence gate as no programme", () => {
|
||||
const stderr = `[Parsed_ebur128_0 @ 0x1] Summary:
|
||||
|
||||
Integrated loudness:
|
||||
I: -14.2 LUFS
|
||||
Threshold: -24.3 LUFS
|
||||
|
||||
True peak:
|
||||
Peak: -1.4 dBFS`;
|
||||
expect(parseEbur128Summary(stderr)).toEqual({ integratedLufs: -14.2, truePeakDb: -1.4 });
|
||||
expect(parseEbur128Summary("Summary:\n I: -70.0 LUFS\n True peak:\n Peak: -inf dBFS")).toEqual({
|
||||
integratedLufs: null,
|
||||
truePeakDb: null,
|
||||
});
|
||||
expect(parseEbur128Summary("no summary")).toEqual({ integratedLufs: null, truePeakDb: null });
|
||||
});
|
||||
|
||||
it("finds exactly one linear loudnorm without measured values", () => {
|
||||
const graph = "[a]anull[b];[b]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]";
|
||||
const site = findTwoPassLoudnorm(graph);
|
||||
expect(site?.options).toBe("I=-14:TP=-1:LRA=11:linear=true");
|
||||
expect(graph.slice(site!.start, site!.end)).toBe(site!.options);
|
||||
expect(findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true:measured_I=-20[b]")).toBeNull();
|
||||
expect(findTwoPassLoudnorm("[a]loudnorm=I=-14[b]")).toBeNull();
|
||||
expect(
|
||||
findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true[b];[c]loudnorm=I=-14:linear=true[d]"),
|
||||
).toBeNull();
|
||||
expect(findTwoPassLoudnorm("[a]anull[b]")).toBeNull();
|
||||
});
|
||||
|
||||
it("parses the loudnorm measurement block", () => {
|
||||
const stderr = `[Parsed_loudnorm_3 @ 0x55]
|
||||
{
|
||||
\t"input_i" : "-31.20",
|
||||
\t"input_tp" : "-20.11",
|
||||
\t"input_lra" : "0.00",
|
||||
\t"input_thresh" : "-41.20",
|
||||
\t"output_i" : "-14.02",
|
||||
\t"output_tp" : "-1.00",
|
||||
\t"output_lra" : "0.00",
|
||||
\t"output_thresh" : "-24.02",
|
||||
\t"normalization_type" : "dynamic",
|
||||
\t"target_offset" : "0.02"
|
||||
}`;
|
||||
expect(parseLoudnormJson(stderr)).toEqual({
|
||||
inputI: -31.2,
|
||||
inputTp: -20.11,
|
||||
inputLra: 0,
|
||||
inputThresh: -41.2,
|
||||
targetOffset: 0.02,
|
||||
});
|
||||
expect(parseLoudnormJson(stderr.replace('"-31.20"', '"-inf"'))).toBeNull();
|
||||
expect(parseLoudnormJson("nothing")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("hardware encoder detection", () => {
|
||||
let dir: string;
|
||||
beforeAll(async () => {
|
||||
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-hw-")));
|
||||
});
|
||||
afterAll(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** A fake ffmpeg that lists encoders and succeeds only for the given test encodes. */
|
||||
async function fakeFfmpeg(name: string, working: string[]): Promise<string> {
|
||||
const file = path.join(dir, name);
|
||||
const cases = working.map((encoder) => `*" ${encoder} "*) exit 0 ;;`).join("\n ");
|
||||
await writeFile(
|
||||
file,
|
||||
`#!/bin/sh
|
||||
case " $* " in
|
||||
*" -encoders "*)
|
||||
echo " V....D libx264 libx264"
|
||||
echo " V....D h264_nvenc NVIDIA NVENC H.264 encoder"
|
||||
echo " V....D hevc_nvenc NVIDIA NVENC hevc encoder"
|
||||
echo " V....D h264_vaapi H.264/AVC (VAAPI)"
|
||||
exit 0 ;;
|
||||
${cases}
|
||||
esac
|
||||
echo "No device available" >&2
|
||||
exit 1
|
||||
`,
|
||||
);
|
||||
await chmod(file, 0o755);
|
||||
return file;
|
||||
}
|
||||
|
||||
it("keeps only listed encoders whose one-frame test encode works", async () => {
|
||||
const renderer = new FfmpegMediaRenderer({
|
||||
ffmpegPath: await fakeFfmpeg("ffmpeg-nvenc", ["h264_nvenc"]),
|
||||
vaapiDevice: path.join(dir, "no-render-node"),
|
||||
});
|
||||
expect(await renderer.hardwareEncoders()).toEqual(["h264_nvenc"]);
|
||||
});
|
||||
|
||||
it("falls back to software when no hardware encoder works or ffmpeg is missing", async () => {
|
||||
const broken = new FfmpegMediaRenderer({ ffmpegPath: await fakeFfmpeg("ffmpeg-none", []) });
|
||||
expect(await broken.hardwareEncoders()).toEqual([]);
|
||||
const missing = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") });
|
||||
expect(await missing.hardwareEncoders()).toEqual([]);
|
||||
});
|
||||
|
||||
it("skips VAAPI encoders without a render node", async () => {
|
||||
const renderer = new FfmpegMediaRenderer({
|
||||
ffmpegPath: await fakeFfmpeg("ffmpeg-vaapi", ["h264_vaapi"]),
|
||||
vaapiDevice: path.join(dir, "no-render-node"),
|
||||
});
|
||||
expect(await renderer.hardwareEncoders()).toEqual([]);
|
||||
});
|
||||
|
||||
it.skipIf(!hasMediaTools)("never reports an encoder this host cannot run", async () => {
|
||||
const renderer = new FfmpegMediaRenderer();
|
||||
for (const encoder of await renderer.hardwareEncoders()) {
|
||||
expect(encoder).toMatch(/^(h264|hevc)_(vaapi|videotoolbox|nvenc)$/);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("plan safety", () => {
|
||||
let dir: string;
|
||||
beforeAll(async () => {
|
||||
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-plan-")));
|
||||
});
|
||||
afterAll(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("refuses writes outside the work directory", async () => {
|
||||
const renderer = new FfmpegMediaRenderer({ root: dir });
|
||||
const workDir = path.join(dir, "work");
|
||||
const plan: RenderPlan = {
|
||||
quality: "draft",
|
||||
workDir,
|
||||
steps: [
|
||||
{ kind: "write_file", id: "escape", path: path.join(dir, "outside.txt"), content: "x" },
|
||||
],
|
||||
outputs: [],
|
||||
};
|
||||
await expect(renderer.render(plan, {})).rejects.toThrow("outside");
|
||||
await expect(stat(path.join(dir, "outside.txt"))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("refuses a work directory outside the media root", async () => {
|
||||
const renderer = new FfmpegMediaRenderer({ root: path.join(dir, "root") });
|
||||
await mkdir(path.join(dir, "root"), { recursive: true });
|
||||
const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] };
|
||||
await expect(renderer.render(plan, {})).rejects.toThrow("outside");
|
||||
});
|
||||
|
||||
it("requires motion, inpaint and identity steps to be prepared first", async () => {
|
||||
const renderer = new FfmpegMediaRenderer();
|
||||
const plan: RenderPlan = {
|
||||
quality: "final",
|
||||
workDir: path.join(dir, "motion"),
|
||||
steps: [
|
||||
{
|
||||
kind: "motion",
|
||||
id: "intro",
|
||||
template: { templateId: "intro-minimal", params: {}, durationMs: 1000 },
|
||||
width: 320,
|
||||
height: 180,
|
||||
fps: 30,
|
||||
outPath: path.join(dir, "motion", "intro.mov"),
|
||||
},
|
||||
],
|
||||
outputs: [],
|
||||
};
|
||||
await expect(renderer.render(plan, {})).rejects.toThrow("was not prepared");
|
||||
});
|
||||
|
||||
it("reports a missing ffmpeg instead of pretending to render", async () => {
|
||||
const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") });
|
||||
const out = path.join(dir, "missing", "out.mp4");
|
||||
const plan: RenderPlan = {
|
||||
quality: "draft",
|
||||
workDir: path.join(dir, "missing"),
|
||||
steps: [{ kind: "ffmpeg", id: "render", argv: ["-version"], produces: [out], durationMs: 1 }],
|
||||
outputs: [{ kind: "video", path: out }],
|
||||
};
|
||||
await expect(renderer.render(plan, {})).rejects.toBeInstanceOf(MediaToolMissingError);
|
||||
});
|
||||
|
||||
it("rejects an already aborted render without spawning anything", async () => {
|
||||
const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") });
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] };
|
||||
await expect(renderer.render(plan, { signal: controller.signal })).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("two-pass loudness", () => {
|
||||
let dir: string;
|
||||
let ffmpegPath: string;
|
||||
let log: string;
|
||||
const GRAPH = "[0:a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]";
|
||||
|
||||
beforeAll(async () => {
|
||||
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-loudnorm-")));
|
||||
log = path.join(dir, "calls.log");
|
||||
ffmpegPath = path.join(dir, "ffmpeg");
|
||||
// Logs each call; the measurement pass prints loudnorm's JSON, the render
|
||||
// pass writes its output file (the last argument).
|
||||
await writeFile(
|
||||
ffmpegPath,
|
||||
`#!/bin/sh
|
||||
echo "$*" >> "${log}"
|
||||
case " $* " in
|
||||
*print_format=json*|*.measure.txt*)
|
||||
printf '[Parsed_loudnorm_0 @ 0x1] \\n{\\n "input_i" : "-31.20",\\n "input_tp" : "-20.11",\\n "input_lra" : "0.00",\\n "input_thresh" : "-41.20",\\n "target_offset" : "0.02"\\n}\\n' >&2
|
||||
exit 0 ;;
|
||||
esac
|
||||
for last; do :; done
|
||||
[ "$last" = "-" ] || echo rendered > "$last"
|
||||
`,
|
||||
);
|
||||
await chmod(ffmpegPath, 0o755);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function calls(): Promise<string[]> {
|
||||
return (await readFile(log, "utf8")).trim().split("\n");
|
||||
}
|
||||
|
||||
it("measures first, then renders with the measured values", async () => {
|
||||
await rm(log, { force: true });
|
||||
const workDir = path.join(dir, "inline");
|
||||
const out = path.join(workDir, "out.m4a");
|
||||
const plan: RenderPlan = {
|
||||
quality: "final",
|
||||
workDir,
|
||||
steps: [
|
||||
{
|
||||
kind: "ffmpeg",
|
||||
id: "render",
|
||||
argv: ["-i", "in.wav", "-filter_complex", GRAPH, "-map", "[aout]", "-t", "2", out],
|
||||
produces: [out],
|
||||
durationMs: 2000,
|
||||
},
|
||||
],
|
||||
outputs: [],
|
||||
};
|
||||
await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {});
|
||||
const [measure, render] = await calls();
|
||||
expect(measure).toContain("linear=true:print_format=json");
|
||||
expect(measure).toContain("-map [aout] -t 2 -f null -");
|
||||
expect(measure).not.toContain(out);
|
||||
expect(render).toContain(
|
||||
"linear=true:measured_I=-31.20:measured_TP=-20.11:measured_LRA=0.00:measured_thresh=-41.20:offset=0.02,aresample=48000",
|
||||
);
|
||||
expect(render?.endsWith(out)).toBe(true);
|
||||
});
|
||||
|
||||
it("rewrites a filtergraph script and removes the measurement script", async () => {
|
||||
await rm(log, { force: true });
|
||||
const workDir = path.join(dir, "script");
|
||||
const script = path.join(workDir, "filtergraph.txt");
|
||||
const out = path.join(workDir, "out.m4a");
|
||||
const plan: RenderPlan = {
|
||||
quality: "final",
|
||||
workDir,
|
||||
steps: [
|
||||
{ kind: "write_file", id: "filtergraph", path: script, content: GRAPH },
|
||||
{
|
||||
kind: "ffmpeg",
|
||||
id: "render",
|
||||
argv: ["-i", "in.wav", "-filter_complex_script", script, "-map", "[aout]", out],
|
||||
produces: [out],
|
||||
durationMs: 2000,
|
||||
},
|
||||
],
|
||||
outputs: [],
|
||||
};
|
||||
await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {});
|
||||
const [measure] = await calls();
|
||||
expect(measure).toContain(path.join(workDir, "filtergraph.measure.txt"));
|
||||
expect(await readFile(script, "utf8")).toContain("measured_I=-31.20");
|
||||
expect((await readdir(workDir)).sort()).toEqual(["filtergraph.txt", "out.m4a"]);
|
||||
});
|
||||
|
||||
it("fails a step whose output was never written and removes what it wrote", async () => {
|
||||
const workDir = path.join(dir, "silent");
|
||||
const out = path.join(workDir, "out.m4a");
|
||||
const script = path.join(workDir, "notes.txt");
|
||||
const plan: RenderPlan = {
|
||||
quality: "draft",
|
||||
workDir,
|
||||
steps: [
|
||||
{ kind: "write_file", id: "notes", path: script, content: "x" },
|
||||
{
|
||||
kind: "ffmpeg",
|
||||
id: "render",
|
||||
// The fake writes its last argument; the declared output stays missing.
|
||||
argv: ["-i", "in.wav", path.join(workDir, "elsewhere.m4a")],
|
||||
produces: [out],
|
||||
durationMs: 1000,
|
||||
},
|
||||
],
|
||||
outputs: [],
|
||||
};
|
||||
await expect(new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {})).rejects.toThrow(
|
||||
"wrote no out.m4a",
|
||||
);
|
||||
await expect(stat(script)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasMediaTools)("real ffmpeg renders", () => {
|
||||
let dir: string;
|
||||
let source: string;
|
||||
const renderer = () => new FfmpegMediaRenderer({ root: dir });
|
||||
const profile = renderTestProfile();
|
||||
|
||||
beforeAll(async () => {
|
||||
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-ffmpeg-")));
|
||||
source = path.join(dir, "assets", "source.mp4");
|
||||
// Deliberately quiet (-36 dBFS sine) so loudness normalization has real work to do.
|
||||
await writeSyntheticClip(source, { seconds: 6, width: 320, height: 180, volumeDb: -36 });
|
||||
});
|
||||
afterAll(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function compile(workDir: string, overrides: Partial<CompileInput> = {}): RenderPlan {
|
||||
const edl = EdlSchema.parse({
|
||||
edlVersion: 1,
|
||||
output: {
|
||||
profileId: profile.id,
|
||||
width: 180,
|
||||
height: 320,
|
||||
fps: 30,
|
||||
durationTargetMs: 4000,
|
||||
durationHardMinMs: 1000,
|
||||
durationHardMaxMs: 120_000,
|
||||
},
|
||||
tracks: {
|
||||
video: [
|
||||
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" },
|
||||
{ id: "c2", assetId: "asset-a", inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" },
|
||||
],
|
||||
},
|
||||
captions: { mode: "both" },
|
||||
overlays: [{ id: "t1", kind: "text", atMs: 500, durationMs: 1500, text: "Example: 50%" }],
|
||||
metadata: { title: "Example teaser" },
|
||||
});
|
||||
return compileRenderPlan({
|
||||
edl,
|
||||
profile,
|
||||
brandKit: null,
|
||||
quality: "draft",
|
||||
workDir,
|
||||
sources: new Map([
|
||||
[
|
||||
"asset-a",
|
||||
{
|
||||
path: source,
|
||||
width: 320,
|
||||
height: 180,
|
||||
durationMs: 6000,
|
||||
fps: 30,
|
||||
hasAudio: true,
|
||||
rotation: 0,
|
||||
},
|
||||
],
|
||||
]),
|
||||
transcripts: new Map([
|
||||
[
|
||||
"asset-a",
|
||||
[
|
||||
{ startMs: 100, endMs: 600, text: "Hello" },
|
||||
{ startMs: 650, endMs: 1200, text: "there." },
|
||||
{ startMs: 3100, endMs: 3800, text: "Bye." },
|
||||
],
|
||||
],
|
||||
]),
|
||||
subjects: new Map([
|
||||
[
|
||||
"asset-a",
|
||||
[
|
||||
{ tMs: 0, cx: 0.3, cy: 0.5, strength: 1 },
|
||||
{ tMs: 5000, cx: 0.7, cy: 0.5, strength: 1 },
|
||||
],
|
||||
],
|
||||
]),
|
||||
motionClips: new Map(),
|
||||
files: { lutPath: null, watermarkPath: null, fonts: new Map(), audio: new Map() },
|
||||
videoEncoder: "libx264",
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
it("renders 16:9 to 9:16 with captions, overlay, two-pass loudness and clean metadata", async () => {
|
||||
const workDir = path.join(dir, "render-ok");
|
||||
const plan = compile(workDir);
|
||||
const progress: MediaRenderProgress[] = [];
|
||||
await renderer().render(plan, { onProgress: (p) => progress.push(p) });
|
||||
|
||||
const video = plan.outputs.find((output) => output.kind === "video")!.path;
|
||||
const report = await renderer().probe(video, { measureLoudness: true });
|
||||
expect(report.width).toBe(180);
|
||||
expect(report.height).toBe(320);
|
||||
expect(report.fps).toBe(30);
|
||||
expect(report.videoCodec).toBe("h264");
|
||||
expect(report.audioCodec).toBe("aac");
|
||||
expect(Math.abs(report.durationMs - 4000)).toBeLessThanOrEqual(1000 / 30 + 1);
|
||||
expect(report.integratedLufs).not.toBeNull();
|
||||
expect(Math.abs(report.integratedLufs! - profile.audio.targetLufs)).toBeLessThanOrEqual(1);
|
||||
expect(report.truePeakDb!).toBeLessThanOrEqual(profile.audio.truePeakDb + 0.5);
|
||||
expect(findMetadataLeaks(report.tags)).toEqual([]);
|
||||
expect(report.tags["format.title"]).toBe("Example teaser");
|
||||
const keys = Object.keys(report.tags).join(" ").toLowerCase();
|
||||
for (const leak of ["location", "creation_time", "make", "model"]) {
|
||||
expect(keys).not.toContain(leak);
|
||||
}
|
||||
expect(Object.values(report.tags).join(" ")).not.toContain("ExampleCam");
|
||||
|
||||
const poster = plan.outputs.find((output) => output.kind === "poster");
|
||||
if (poster) expect(findMetadataLeaks((await renderer().probe(poster.path)).tags)).toEqual([]);
|
||||
expect(progress.length).toBeGreaterThan(0);
|
||||
expect(progress.at(-1)!.progress).toBeCloseTo(1, 5);
|
||||
for (let i = 1; i < progress.length; i++) {
|
||||
expect(progress[i]!.progress).toBeGreaterThanOrEqual(progress[i - 1]!.progress - 1e-9);
|
||||
}
|
||||
// The measurement pass leaves nothing behind.
|
||||
expect((await readdir(workDir)).sort()).toEqual(
|
||||
["captions.ass", "captions.srt", "captions.vtt", "output.mp4", "poster.jpg"].filter(
|
||||
(name) => name !== "poster.jpg" || poster,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it("extracts an EXIF-free PNG frame", async () => {
|
||||
const out = path.join(dir, "frames", "frame.png");
|
||||
await mkdir(path.dirname(out), { recursive: true });
|
||||
await renderer().extractFrame({ path: source, tMs: 1000, outPath: out, width: 161 });
|
||||
const report = await renderer().probe(out);
|
||||
expect(report.width).toBe(160);
|
||||
expect(report.videoCodec).toBe("png");
|
||||
expect(findMetadataLeaks(report.tags)).toEqual([]);
|
||||
await expect(
|
||||
renderer().extractFrame({ path: source, tMs: 1000, outPath: path.join(dir, "frame.jpg") }),
|
||||
).rejects.toThrow("PNG");
|
||||
await expect(
|
||||
renderer().extractFrame({ path: source, tMs: 60_000, outPath: path.join(dir, "late.png") }),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("refuses to probe outside the media root", async () => {
|
||||
await expect(renderer().probe("/etc/hostname")).rejects.toThrow("outside");
|
||||
await expect(renderer().probe("relative.mp4")).rejects.toThrow("absolute");
|
||||
});
|
||||
|
||||
it("aborts a running render, kills ffmpeg and removes partial outputs", async () => {
|
||||
const workDir = path.join(dir, "render-abort");
|
||||
const long = await mkdtemp(path.join(dir, "long-"));
|
||||
const longSource = path.join(long, "long.mp4");
|
||||
await writeSyntheticClip(longSource, { seconds: 30, width: 1280, height: 720 });
|
||||
const plan = compile(workDir, {
|
||||
edl: EdlSchema.parse({
|
||||
edlVersion: 1,
|
||||
output: {
|
||||
profileId: profile.id,
|
||||
width: 1080,
|
||||
height: 1920,
|
||||
fps: 30,
|
||||
durationTargetMs: 30_000,
|
||||
durationHardMinMs: 1000,
|
||||
durationHardMaxMs: 120_000,
|
||||
},
|
||||
tracks: {
|
||||
video: [
|
||||
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 30_000, atMs: 0, reframe: "none" },
|
||||
],
|
||||
},
|
||||
metadata: {},
|
||||
}),
|
||||
sources: new Map([
|
||||
[
|
||||
"asset-a",
|
||||
{
|
||||
path: longSource,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
durationMs: 30_000,
|
||||
fps: 30,
|
||||
hasAudio: true,
|
||||
rotation: 0,
|
||||
},
|
||||
],
|
||||
]),
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const started = Date.now();
|
||||
const rendering = renderer().render(plan, {
|
||||
signal: controller.signal,
|
||||
onProgress: (p) => {
|
||||
if (p.progress > 0.01) controller.abort();
|
||||
},
|
||||
});
|
||||
await expect(rendering).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(Date.now() - started).toBeLessThan(25_000);
|
||||
await expect(stat(path.join(workDir, "output.mp4"))).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
@ -316,6 +316,13 @@ export class FfmpegMediaRenderer implements MediaRenderer {
|
||||
signal,
|
||||
report: (fraction) => report(step.id, fraction, weight(step)),
|
||||
});
|
||||
// Never report success for an output ffmpeg did not actually write.
|
||||
for (const produced of step.produces) {
|
||||
const file = await stat(produced).catch(() => null);
|
||||
if (!file?.isFile() || file.size === 0) {
|
||||
throw new Error(`step ${step.id} wrote no ${path.basename(produced)}`);
|
||||
}
|
||||
}
|
||||
done += weight(step);
|
||||
report(step.id, 0, 0);
|
||||
break;
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
import { execFile, execFileSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, realpath, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import type {
|
||||
JobPublisher,
|
||||
LinkShortener,
|
||||
@ -43,6 +45,70 @@ import type {
|
||||
MediaServiceDeps,
|
||||
} from "./media-context.js";
|
||||
|
||||
const run = promisify(execFile);
|
||||
|
||||
function onPath(binary: string): boolean {
|
||||
try {
|
||||
execFileSync(binary, ["-version"], { stdio: "ignore" });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Real-ffmpeg tests run when both tools are installed and skip otherwise. */
|
||||
export const hasMediaTools = onPath("ffmpeg") && onPath("ffprobe");
|
||||
|
||||
/**
|
||||
* Synthetic clip: testsrc2 pattern plus a sine tone, H.264/AAC in MP4, with the
|
||||
* kind of tags a phone writes (GPS, device, capture time) that every export
|
||||
* must drop. No real people or places.
|
||||
*/
|
||||
export async function writeSyntheticClip(
|
||||
file: string,
|
||||
options: { seconds: number; width: number; height: number; fps?: number; volumeDb?: number },
|
||||
): Promise<void> {
|
||||
const fps = options.fps ?? 30;
|
||||
await mkdir(path.dirname(file), { recursive: true });
|
||||
await run("ffmpeg", [
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
`testsrc2=size=${options.width}x${options.height}:rate=${fps}:duration=${options.seconds}`,
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
`sine=frequency=440:sample_rate=48000:duration=${options.seconds}`,
|
||||
"-af",
|
||||
`volume=${options.volumeDb ?? -20}dB`,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-shortest",
|
||||
"-metadata",
|
||||
"creation_time=2020-01-01T10:00:00Z",
|
||||
"-metadata",
|
||||
"location=+00.0000+000.0000/",
|
||||
"-metadata",
|
||||
"make=ExampleCam",
|
||||
"-metadata",
|
||||
"model=ExampleCam One",
|
||||
"-metadata:s:v",
|
||||
"handler_name=ExampleCam serial 0000",
|
||||
file,
|
||||
]);
|
||||
}
|
||||
|
||||
/*
|
||||
* Test harness for the render service and job: the repos the render slice
|
||||
* reads, either in memory or backed by PostgreSQL (VERIFY_DATABASE), plus a
|
||||
@ -134,7 +200,11 @@ export interface RenderHarness {
|
||||
addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise<void>;
|
||||
addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise<void>;
|
||||
addArtifact(file: string, name: string, mimeType: string): Promise<string>;
|
||||
createProject(input: { assetIds: string[]; kit: BrandKit | null; profileId: string }): Promise<string>;
|
||||
createProject(input: {
|
||||
assetIds: string[];
|
||||
kit: BrandKit | null;
|
||||
profileId: string;
|
||||
}): Promise<string>;
|
||||
createPlan(projectId: string, edl: unknown): Promise<MediaEditPlan>;
|
||||
planVersions(projectId: string, profileId: string): Promise<MediaEditPlan[]>;
|
||||
artifactCount(): Promise<number>;
|
||||
@ -271,7 +341,11 @@ interface Backend {
|
||||
createAsset(name: string): Promise<string>;
|
||||
addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise<void>;
|
||||
addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise<void>;
|
||||
createProject(input: { assetIds: string[]; kit: BrandKit | null; profileId: string }): Promise<string>;
|
||||
createProject(input: {
|
||||
assetIds: string[];
|
||||
kit: BrandKit | null;
|
||||
profileId: string;
|
||||
}): Promise<string>;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
@ -461,7 +535,9 @@ function memoryBackend(now: () => Date): Backend {
|
||||
},
|
||||
async listPlanVersions(space: string, projectId: string, profileId: string) {
|
||||
return plans
|
||||
.filter((p) => p.spaceId === space && p.projectId === projectId && p.profileId === profileId)
|
||||
.filter(
|
||||
(p) => p.spaceId === space && p.projectId === projectId && p.profileId === profileId,
|
||||
)
|
||||
.sort((a, b) => b.version - a.version)
|
||||
.map(mapMediaEditPlan);
|
||||
},
|
||||
|
||||
456
packages/adapters/src/media-render.test.ts
Normal file
456
packages/adapters/src/media-render.test.ts
Normal file
@ -0,0 +1,456 @@
|
||||
import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MediaRenderer, MotionGraphicsRenderer } from "@rakazo/adapter-kit";
|
||||
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||
import type { FfprobeReport, GateIssue, MediaRender, RenderPlan } from "@rakazo/contracts";
|
||||
import { findMetadataLeaks } from "@rakazo/media";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { MediaPrivacyService } from "./media-context.js";
|
||||
import { FfmpegMediaRenderer } from "./media-ffmpeg-renderer.js";
|
||||
import type { FakePrivacy, RenderHarness } from "./media-render.fixtures.js";
|
||||
import {
|
||||
createFakePrivacy,
|
||||
createRenderHarness,
|
||||
hasMediaTools,
|
||||
renderTestProfile,
|
||||
writeSyntheticClip,
|
||||
} from "./media-render.fixtures.js";
|
||||
import { createMediaRenderHandler, createMediaRenderService, pickEncoder } from "./media-render.js";
|
||||
|
||||
const profile = renderTestProfile();
|
||||
|
||||
const motion: MotionGraphicsRenderer = {
|
||||
id: "unused",
|
||||
templates: () => [],
|
||||
render: async () => {
|
||||
throw new Error("no motion in these plans");
|
||||
},
|
||||
};
|
||||
|
||||
interface FakeRenderer extends MediaRenderer {
|
||||
plans: RenderPlan[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes write_file steps and writes placeholder outputs; probes report a
|
||||
* clean, on-target file. `failEncoders` makes renders with those encoders fail.
|
||||
*/
|
||||
function fakeRenderer(
|
||||
options: {
|
||||
hardware?: string[];
|
||||
failEncoders?: string[];
|
||||
tags?: Record<string, string>;
|
||||
posterTags?: Record<string, string>;
|
||||
/** Resolves once a render started; the render then waits for its signal. */
|
||||
hold?: { started: () => void };
|
||||
} = {},
|
||||
): FakeRenderer {
|
||||
const plans: RenderPlan[] = [];
|
||||
let durationMs = 0;
|
||||
return {
|
||||
id: "fake",
|
||||
plans,
|
||||
hardwareEncoders: async () => options.hardware ?? [],
|
||||
async render(plan, { signal, onProgress } = {}) {
|
||||
plans.push(plan);
|
||||
const step = plan.steps.find((s) => s.kind === "ffmpeg" && s.id === "render");
|
||||
if (step?.kind !== "ffmpeg") throw new Error("no render step");
|
||||
durationMs = step.durationMs;
|
||||
const encoder = step.argv[step.argv.indexOf("-c:v") + 1]!;
|
||||
if (options.failEncoders?.includes(encoder)) throw new Error(`${encoder}: device lost`);
|
||||
if (options.hold) {
|
||||
options.hold.started();
|
||||
await new Promise<void>((_, reject) => {
|
||||
signal?.addEventListener("abort", () => {
|
||||
const error = new Error("render aborted");
|
||||
error.name = "AbortError";
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
for (const s of plan.steps) {
|
||||
if (s.kind === "write_file") {
|
||||
await mkdir(path.dirname(s.path), { recursive: true });
|
||||
await writeFile(s.path, s.content);
|
||||
}
|
||||
if (s.kind === "ffmpeg") {
|
||||
for (const file of s.produces) await writeFile(file, `rendered ${s.id}`);
|
||||
}
|
||||
}
|
||||
onProgress?.({ stepId: "render", progress: 1 });
|
||||
},
|
||||
async probe(file): Promise<FfprobeReport> {
|
||||
const poster = file.endsWith(".jpg");
|
||||
return {
|
||||
durationMs: poster ? 0 : durationMs,
|
||||
width: profile.video.width,
|
||||
height: profile.video.height,
|
||||
fps: poster ? 0 : profile.video.fps,
|
||||
videoCodec: poster ? "mjpeg" : "h264",
|
||||
audioCodec: poster ? null : "aac",
|
||||
sizeBytes: (await stat(file)).size,
|
||||
integratedLufs: poster ? null : -14.2,
|
||||
truePeakDb: poster ? null : -1.5,
|
||||
tags: poster ? (options.posterTags ?? {}) : (options.tags ?? {}),
|
||||
};
|
||||
},
|
||||
extractFrame: async () => {
|
||||
throw new Error("not used");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function edlFor(assetId: string) {
|
||||
return {
|
||||
edlVersion: 1,
|
||||
output: {
|
||||
profileId: profile.id,
|
||||
width: profile.video.width,
|
||||
height: profile.video.height,
|
||||
fps: 30,
|
||||
durationTargetMs: 3000,
|
||||
durationHardMinMs: 1000,
|
||||
durationHardMaxMs: 120_000,
|
||||
},
|
||||
tracks: {
|
||||
video: [
|
||||
{ id: "c1", assetId, inMs: 0, outMs: 1500, atMs: 0, reframe: "auto" },
|
||||
{ id: "c2", assetId, inMs: 2000, outMs: 3500, atMs: 1500, reframe: "auto" },
|
||||
],
|
||||
},
|
||||
captions: { mode: "both" },
|
||||
overlays: [{ id: "t1", kind: "text", atMs: 200, durationMs: 1000, text: "Example" }],
|
||||
metadata: { title: "Example teaser" },
|
||||
};
|
||||
}
|
||||
|
||||
interface Setup {
|
||||
harness: RenderHarness;
|
||||
privacy: FakePrivacy;
|
||||
renderer: MediaRenderer;
|
||||
run: (planQuality?: "draft" | "final") => Promise<MediaRender>;
|
||||
renderDir: (renderId: string) => string;
|
||||
}
|
||||
|
||||
let cleanups: Array<() => Promise<void>> = [];
|
||||
afterEach(async () => {
|
||||
for (const cleanup of cleanups) await cleanup();
|
||||
cleanups = [];
|
||||
});
|
||||
|
||||
async function setup(options: {
|
||||
renderer: MediaRenderer;
|
||||
privacy?: (sources: RenderHarness["sources"]) => MediaPrivacyService;
|
||||
failWith?: GateIssue;
|
||||
realSource?: boolean;
|
||||
}): Promise<Setup & { service: ReturnType<typeof createMediaRenderService> }> {
|
||||
const harness = await createRenderHarness({
|
||||
profiles: [profile],
|
||||
renderer: options.renderer,
|
||||
motion,
|
||||
});
|
||||
cleanups.push(() => harness.cleanup());
|
||||
const assetId = await harness.createAsset("source.mp4");
|
||||
const file = harness.assetPath(assetId, "source.mp4");
|
||||
if (options.realSource) {
|
||||
await writeSyntheticClip(file, { seconds: 4, width: 320, height: 180, volumeDb: -30 });
|
||||
} else {
|
||||
await mkdir(path.dirname(file), { recursive: true });
|
||||
await writeFile(file, "synthetic");
|
||||
}
|
||||
harness.setSource(assetId, {
|
||||
path: file,
|
||||
width: 320,
|
||||
height: 180,
|
||||
durationMs: 4000,
|
||||
fps: 30,
|
||||
hasAudio: true,
|
||||
rotation: 0,
|
||||
});
|
||||
await harness.addAnalysis(assetId, "transcript", {
|
||||
language: "en",
|
||||
words: [
|
||||
{ startMs: 100, endMs: 500, text: "Hello", confidence: 0.9 },
|
||||
{ startMs: 2100, endMs: 2600, text: "again.", confidence: 0.9 },
|
||||
],
|
||||
});
|
||||
await harness.addSegments(assetId, [
|
||||
{ startMs: 0, endMs: 2000 },
|
||||
{ startMs: 2000, endMs: 4000 },
|
||||
]);
|
||||
const projectId = await harness.createProject({
|
||||
assetIds: [assetId],
|
||||
kit: null,
|
||||
profileId: profile.id,
|
||||
});
|
||||
const plan = await harness.createPlan(projectId, edlFor(assetId));
|
||||
const privacy = createFakePrivacy(harness.sources, { failWith: options.failWith });
|
||||
const service = createMediaRenderService(harness.deps);
|
||||
const handler = createMediaRenderHandler(
|
||||
harness.deps,
|
||||
options.privacy ? options.privacy(harness.sources) : privacy,
|
||||
{ cancelPollMs: 50 },
|
||||
);
|
||||
const actor = { spaceId: harness.spaceId, userId: harness.userId };
|
||||
return {
|
||||
harness,
|
||||
privacy,
|
||||
renderer: options.renderer,
|
||||
service,
|
||||
renderDir: (renderId) => path.join(harness.mediaRoot, harness.spaceId, "renders", renderId),
|
||||
async run(quality = "draft") {
|
||||
const render = await service.start(actor, { planId: plan.id, quality });
|
||||
await handler({ renderId: render.id });
|
||||
return harness.deps.repos.getRender(harness.spaceId, render.id);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function exists(file: string): Promise<boolean> {
|
||||
return stat(file).then(
|
||||
() => true,
|
||||
() => false,
|
||||
);
|
||||
}
|
||||
|
||||
describe("media render service", () => {
|
||||
it("queues a render job and announces it", async () => {
|
||||
const { harness, service } = await setup({ renderer: fakeRenderer() });
|
||||
expect(harness.backend).toBe(
|
||||
process.env.VERIFY_DATABASE && process.env.DATABASE_URL ? "postgres" : "memory",
|
||||
);
|
||||
const projectId = await harness.createProject({
|
||||
assetIds: [],
|
||||
kit: null,
|
||||
profileId: profile.id,
|
||||
});
|
||||
const assetId = [...harness.sources.keys()][0]!;
|
||||
const plan = await harness.createPlan(projectId, edlFor(assetId));
|
||||
const actor = { spaceId: harness.spaceId, userId: harness.userId };
|
||||
const render = await service.start(actor, { planId: plan.id, quality: "final" });
|
||||
expect(render).toMatchObject({ status: "queued", quality: "final", planId: plan.id });
|
||||
expect(harness.enqueued).toContain(`media.render:${JSON.stringify({ renderId: render.id })}`);
|
||||
expect(harness.events).toContainEqual({
|
||||
type: "media.render",
|
||||
renderId: render.id,
|
||||
projectId,
|
||||
status: "queued",
|
||||
progress: 0,
|
||||
});
|
||||
await expect(
|
||||
service.start({ ...actor, spaceId: "space-other" }, { planId: plan.id, quality: "draft" }),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("renders, gates, stores outputs and serves only recorded files", async () => {
|
||||
const renderer = fakeRenderer();
|
||||
const { harness, service, run, renderDir } = await setup({ renderer });
|
||||
const render = await run();
|
||||
expect(render.status).toBe("done");
|
||||
expect(render.progress).toBe(1);
|
||||
expect(render.gateReport?.passed).toBe(true);
|
||||
expect(render.ffprobe?.integratedLufs).toBe(-14.2);
|
||||
expect(render.outputs.map((output) => output.fileName).sort()).toEqual([
|
||||
"captions.srt",
|
||||
"captions.vtt",
|
||||
"poster.jpg",
|
||||
"video.mp4",
|
||||
]);
|
||||
const video = render.outputs.find((output) => output.kind === "video")!;
|
||||
expect(video.artifactId).toBeNull();
|
||||
expect(render.outputs.filter((output) => output.artifactId !== null)).toHaveLength(3);
|
||||
expect(await harness.artifactCount()).toBe(3);
|
||||
// Intermediates are gone; only the published outputs stay.
|
||||
expect((await readdir(renderDir(render.id))).sort()).toEqual([
|
||||
"captions.srt",
|
||||
"captions.vtt",
|
||||
"poster.jpg",
|
||||
"video.mp4",
|
||||
]);
|
||||
const actor = { spaceId: harness.spaceId, userId: harness.userId };
|
||||
expect(await service.outputPath(actor, render.id, "video.mp4")).toBe(
|
||||
path.join(renderDir(render.id), "video.mp4"),
|
||||
);
|
||||
expect(await service.outputPath(actor, render.id, "../video.mp4")).toBeNull();
|
||||
expect(await service.outputPath(actor, render.id, "work")).toBeNull();
|
||||
await expect(
|
||||
service.outputPath({ ...actor, spaceId: "space-other" }, render.id, "video.mp4"),
|
||||
).rejects.toThrow();
|
||||
const statuses = harness.events.flatMap((event) =>
|
||||
event.type === "media.render" && event.renderId === render.id ? [event.status] : [],
|
||||
);
|
||||
expect(statuses[0]).toBe("queued");
|
||||
expect(statuses).toContain("gating");
|
||||
expect(statuses.at(-1)).toBe("done");
|
||||
});
|
||||
|
||||
it("refuses to render when redactions cannot be applied (fail closed)", async () => {
|
||||
const renderer = fakeRenderer();
|
||||
let calls = 0;
|
||||
const { harness, run, renderDir } = await setup({
|
||||
renderer,
|
||||
privacy: (sources) => ({
|
||||
...createFakePrivacy(sources),
|
||||
async prepareSources() {
|
||||
calls += 1;
|
||||
throw new Error("face track lost for 800 ms");
|
||||
},
|
||||
}),
|
||||
});
|
||||
const render = await run("final");
|
||||
expect(calls).toBe(1);
|
||||
expect(render.status).toBe("failed");
|
||||
expect(render.error).toBe("Anonymization could not be applied. Nothing was rendered.");
|
||||
expect(render.outputs).toEqual([]);
|
||||
expect(renderer.plans).toHaveLength(0);
|
||||
expect(await exists(renderDir(render.id))).toBe(false);
|
||||
expect(await harness.artifactCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("names the missing capability and refuses when a clip was not prepared", async () => {
|
||||
const unsupported = await setup({
|
||||
renderer: fakeRenderer(),
|
||||
privacy: (sources) => ({
|
||||
...createFakePrivacy(sources),
|
||||
async prepareSources() {
|
||||
throw new UnsupportedMediaOperationError("inpaintVideo", "no model");
|
||||
},
|
||||
}),
|
||||
});
|
||||
expect((await unsupported.run()).error).toBe(
|
||||
"Anonymization needs inpaintVideo, which is not available. Nothing was rendered.",
|
||||
);
|
||||
|
||||
const renderer = fakeRenderer();
|
||||
const partial = await setup({
|
||||
renderer,
|
||||
privacy: (sources) => ({
|
||||
...createFakePrivacy(sources),
|
||||
async prepareSources() {
|
||||
return { sources: new Map(), aiAlterations: [], fallbacks: [] };
|
||||
},
|
||||
}),
|
||||
});
|
||||
const render = await partial.run();
|
||||
expect(render.status).toBe("failed");
|
||||
expect(render.error).toBe("Anonymization did not prepare every clip. Nothing was rendered.");
|
||||
expect(renderer.plans).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("falls back to software when a hardware encoder fails and skips it afterwards", async () => {
|
||||
const renderer = fakeRenderer({
|
||||
hardware: ["hevc_vaapi", "h264_vaapi"],
|
||||
failEncoders: ["h264_vaapi"],
|
||||
});
|
||||
const { run } = await setup({ renderer });
|
||||
const encoders = () =>
|
||||
renderer.plans.map((plan) => {
|
||||
const step = plan.steps.find((s) => s.kind === "ffmpeg" && s.id === "render");
|
||||
return step?.kind === "ffmpeg" ? step.argv[step.argv.indexOf("-c:v") + 1] : null;
|
||||
});
|
||||
expect((await run("final")).status).toBe("done");
|
||||
expect(encoders()).toEqual(["h264_vaapi", "libx264"]);
|
||||
expect((await run("final")).status).toBe("done");
|
||||
expect(encoders()).toEqual(["h264_vaapi", "libx264", "libx264"]);
|
||||
});
|
||||
|
||||
it("uses software for drafts and a working hardware encoder for finals", async () => {
|
||||
const renderer = fakeRenderer({ hardware: ["hevc_nvenc", "h264_nvenc"] });
|
||||
expect(await pickEncoder(renderer, profile, "draft")).toBe("libx264");
|
||||
expect(await pickEncoder(renderer, profile, "final")).toBe("h264_nvenc");
|
||||
expect(await pickEncoder(fakeRenderer(), profile, "final")).toBe("libx264");
|
||||
const hevc = renderTestProfile({
|
||||
video: { ...profile.video, codec: "hevc" },
|
||||
});
|
||||
expect(await pickEncoder(renderer, hevc, "final")).toBe("hevc_nvenc");
|
||||
expect(await pickEncoder(renderer, hevc, "draft")).toBe("libx265");
|
||||
});
|
||||
|
||||
it("fails a render whose video or poster still carries source metadata", async () => {
|
||||
const leaky = await setup({
|
||||
renderer: fakeRenderer({ tags: { "format.location": "+00+000/" } }),
|
||||
});
|
||||
const render = await leaky.run();
|
||||
expect(render.status).toBe("failed");
|
||||
expect(render.error).toBe("The export still carried source metadata.");
|
||||
expect(await exists(leaky.renderDir(render.id))).toBe(false);
|
||||
|
||||
const poster = await setup({
|
||||
renderer: fakeRenderer({ posterTags: { "format.make": "ExampleCam" } }),
|
||||
});
|
||||
expect((await poster.run()).error).toBe("The poster still carried source metadata.");
|
||||
});
|
||||
|
||||
it("auto-fixes a blocked gate at most three times, then blocks", async () => {
|
||||
const renderer = fakeRenderer();
|
||||
const { harness, privacy, run, renderDir } = await setup({
|
||||
renderer,
|
||||
failWith: {
|
||||
check: "content",
|
||||
severity: "block",
|
||||
code: "classifier_violence",
|
||||
message: "violence score above the profile threshold",
|
||||
atMs: 0,
|
||||
endMs: 400,
|
||||
},
|
||||
});
|
||||
const render = await run();
|
||||
expect(render.status).toBe("blocked");
|
||||
expect(render.gateReport?.passed).toBe(false);
|
||||
expect(privacy.gateCalls.map((call) => call.autoFixRound)).toEqual([0, 1, 2, 3]);
|
||||
expect(renderer.plans).toHaveLength(4);
|
||||
const record = await harness.deps.repos.getRenderRecord(harness.spaceId, render.id);
|
||||
expect(record.autoFixRound).toBe(3);
|
||||
const versions = await harness.planVersions(render.projectId, profile.id);
|
||||
expect(versions).toHaveLength(4);
|
||||
expect(versions[0]!.createdBy).toBe("agent");
|
||||
expect(versions[0]!.rationale).toContain("Auto-fix 3/3");
|
||||
expect(render.outputs).toEqual([]);
|
||||
expect(await exists(renderDir(render.id))).toBe(false);
|
||||
expect(await harness.artifactCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("cancels a running render, stops it and removes its files", async () => {
|
||||
let started!: () => void;
|
||||
const running = new Promise<void>((resolve) => {
|
||||
started = resolve;
|
||||
});
|
||||
const renderer = fakeRenderer({ hold: { started: () => started() } });
|
||||
const { harness, service, run, renderDir } = await setup({ renderer });
|
||||
const result = run();
|
||||
await running;
|
||||
const active = harness.events.findLast((event) => event.type === "media.render");
|
||||
const renderId = active?.type === "media.render" ? active.renderId : "";
|
||||
const canceled = await service.cancel(
|
||||
{ spaceId: harness.spaceId, userId: harness.userId },
|
||||
renderId,
|
||||
);
|
||||
expect(canceled.status).toBe("canceled");
|
||||
const render = await result;
|
||||
expect(render.status).toBe("canceled");
|
||||
expect(await exists(renderDir(renderId))).toBe(false);
|
||||
await expect(
|
||||
service.cancel({ spaceId: harness.spaceId, userId: harness.userId }, renderId),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasMediaTools)("media render job with real ffmpeg", () => {
|
||||
it("renders a synthetic project end to end and records a clean probe", async () => {
|
||||
const renderer = new FfmpegMediaRenderer();
|
||||
const { run, renderDir } = await setup({ renderer, realSource: true });
|
||||
const render = await run("draft");
|
||||
expect(render.error).toBeNull();
|
||||
expect(render.status).toBe("done");
|
||||
const probe = render.ffprobe!;
|
||||
expect(probe.width).toBe(profile.video.width);
|
||||
expect(probe.height).toBe(profile.video.height);
|
||||
expect(Math.abs(probe.durationMs - 3000)).toBeLessThanOrEqual(1000 / 30 + 1);
|
||||
expect(Math.abs(probe.integratedLufs! - profile.audio.targetLufs)).toBeLessThanOrEqual(1);
|
||||
expect(findMetadataLeaks(probe.tags)).toEqual([]);
|
||||
const video = path.join(renderDir(render.id), "video.mp4");
|
||||
const reprobed = await renderer.probe(video);
|
||||
expect(reprobed.tags["format.title"]).toBe("Example teaser");
|
||||
expect(Object.values(reprobed.tags).join(" ")).not.toContain("ExampleCam");
|
||||
}, 120_000);
|
||||
});
|
||||
@ -23,6 +23,7 @@ import { compileRenderPlan, findMetadataLeaks, planAutoFix } from "@rakazo/media
|
||||
import { resolveInside } from "@rakazo/media/node";
|
||||
import type {
|
||||
MediaActor,
|
||||
MediaPreparedSources,
|
||||
MediaPrivacyService,
|
||||
MediaRenderService,
|
||||
MediaServiceDeps,
|
||||
@ -452,15 +453,37 @@ async function runRender(input: RunInput): Promise<void> {
|
||||
const roundDir = path.join(workDir, `round-${round}`);
|
||||
await mkdir(roundDir, { recursive: true });
|
||||
|
||||
const prepared = await privacy.prepareSources({
|
||||
actor,
|
||||
projectId: project.id,
|
||||
edl,
|
||||
quality,
|
||||
workDir: roundDir,
|
||||
signal,
|
||||
});
|
||||
// Fail closed: no redacted sources, no render.
|
||||
let prepared: MediaPreparedSources;
|
||||
try {
|
||||
prepared = await privacy.prepareSources({
|
||||
actor,
|
||||
projectId: project.id,
|
||||
edl,
|
||||
quality,
|
||||
workDir: roundDir,
|
||||
signal,
|
||||
});
|
||||
} catch (error) {
|
||||
checkCanceled();
|
||||
getLogger().warn("media render sources not prepared", {
|
||||
renderId,
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
throw new MediaRenderFailure(
|
||||
error instanceof UnsupportedMediaOperationError
|
||||
? `Anonymization needs ${error.operation}, which is not available. Nothing was rendered.`
|
||||
: "Anonymization could not be applied. Nothing was rendered.",
|
||||
);
|
||||
}
|
||||
checkCanceled();
|
||||
for (const clip of edl.tracks.video) {
|
||||
if (!prepared.sources.has(clip.assetId)) {
|
||||
throw new MediaRenderFailure(
|
||||
"Anonymization did not prepare every clip. Nothing was rendered.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const clip of edl.tracks.audio) {
|
||||
if (audioFiles.has(clip.artifactId)) continue;
|
||||
@ -475,37 +498,42 @@ async function runRender(input: RunInput): Promise<void> {
|
||||
|
||||
const motionClips = await renderMotionClips(deps, edl, kit, brand, roundDir, signal);
|
||||
const { transcripts, subjects } = await analysisInputs(deps, spaceId, edl);
|
||||
const videoEncoder = await pickEncoder(deps, profile, quality);
|
||||
|
||||
const renderPlan = compileRenderPlan({
|
||||
edl,
|
||||
profile,
|
||||
brandKit: kit,
|
||||
quality,
|
||||
workDir: roundDir,
|
||||
sources: prepared.sources,
|
||||
transcripts,
|
||||
subjects,
|
||||
motionClips,
|
||||
files: {
|
||||
lutPath: brand.lut,
|
||||
watermarkPath: brand.watermark,
|
||||
fonts: brand.fonts,
|
||||
audio: audioFiles,
|
||||
logoPath: brand.logos[0] ?? null,
|
||||
},
|
||||
videoEncoder,
|
||||
});
|
||||
|
||||
await executePlan(input, renderPlan, update);
|
||||
const compile = (videoEncoder: string) =>
|
||||
compileRenderPlan({
|
||||
edl,
|
||||
profile,
|
||||
brandKit: kit,
|
||||
quality,
|
||||
workDir: roundDir,
|
||||
sources: prepared.sources,
|
||||
transcripts,
|
||||
subjects,
|
||||
motionClips,
|
||||
files: {
|
||||
lutPath: brand.lut,
|
||||
watermarkPath: brand.watermark,
|
||||
fonts: brand.fonts,
|
||||
audio: audioFiles,
|
||||
logoPath: brand.logos[0] ?? null,
|
||||
},
|
||||
videoEncoder,
|
||||
});
|
||||
const renderPlan = await renderWithFallback(input, profile, quality, compile, update);
|
||||
|
||||
const video = renderPlan.outputs.find((output) => output.kind === "video");
|
||||
if (!video) throw new Error("render plan has no video output");
|
||||
const ffprobe = await deps.renderer.probe(video.path, { measureLoudness: true, signal });
|
||||
// A render whose metadata could not be removed is failed, not gated.
|
||||
if (findMetadataLeaks(ffprobe.tags).length > 0) {
|
||||
// Spec 5: a render whose metadata could not be removed is failed, not gated.
|
||||
throw new MediaRenderFailure("The export still carried source metadata.");
|
||||
}
|
||||
for (const output of renderPlan.outputs) {
|
||||
if (output.kind !== "poster") continue;
|
||||
const poster = await deps.renderer.probe(output.path, { signal });
|
||||
if (findMetadataLeaks(poster.tags).length > 0) {
|
||||
throw new MediaRenderFailure("The poster still carried source metadata.");
|
||||
}
|
||||
}
|
||||
const gating = await update({ status: "gating", progress: 0.9, ffprobe });
|
||||
await publishRender(deps, spaceId, gating);
|
||||
|
||||
@ -688,17 +716,64 @@ async function analysisInputs(deps: MediaServiceDeps, spaceId: string, edl: Edl)
|
||||
return { transcripts, subjects };
|
||||
}
|
||||
|
||||
/** Hardware encoders that failed a real render, per renderer, so later renders skip them. */
|
||||
const failedEncoders = new WeakMap<object, Set<string>>();
|
||||
|
||||
export function softwareEncoder(profile: PlatformProfile): string {
|
||||
return profile.video.codec === "h264" ? "libx264" : "libx265";
|
||||
}
|
||||
|
||||
/** Final renders prefer a working hardware encoder of the profile's codec; drafts use software. */
|
||||
async function pickEncoder(
|
||||
deps: MediaServiceDeps,
|
||||
export async function pickEncoder(
|
||||
renderer: MediaServiceDeps["renderer"],
|
||||
profile: PlatformProfile,
|
||||
quality: MediaRenderQuality,
|
||||
): Promise<string> {
|
||||
const family = profile.video.codec;
|
||||
const software = family === "h264" ? "libx264" : "libx265";
|
||||
const software = softwareEncoder(profile);
|
||||
if (quality === "draft") return software;
|
||||
const hardware = await deps.renderer.hardwareEncoders();
|
||||
return hardware.find((encoder) => encoder.startsWith(`${family}_`)) ?? software;
|
||||
const failed = failedEncoders.get(renderer);
|
||||
const hardware = await renderer.hardwareEncoders();
|
||||
return (
|
||||
hardware.find(
|
||||
(encoder) => encoder.startsWith(`${profile.video.codec}_`) && !failed?.has(encoder),
|
||||
) ?? software
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the plan with the preferred encoder. A hardware encoder that fails
|
||||
* mid-render (driver, device, or session limits) is retried once in software
|
||||
* and skipped by this process afterwards; software failures are final.
|
||||
*/
|
||||
async function renderWithFallback(
|
||||
input: RunInput,
|
||||
profile: PlatformProfile,
|
||||
quality: MediaRenderQuality,
|
||||
compile: (videoEncoder: string) => RenderPlan,
|
||||
update: (patch: MediaRenderPatch) => Promise<MediaRender>,
|
||||
): Promise<RenderPlan> {
|
||||
const { deps, signal } = input;
|
||||
const encoder = await pickEncoder(deps.renderer, profile, quality);
|
||||
const plan = compile(encoder);
|
||||
const software = softwareEncoder(profile);
|
||||
try {
|
||||
await executePlan(input, plan, update);
|
||||
return plan;
|
||||
} catch (error) {
|
||||
if (encoder === software || signal.aborted || error instanceof RenderCanceled) throw error;
|
||||
if (error instanceof MediaToolMissingError) throw error;
|
||||
getLogger().warn("media hardware encoder failed; rendering in software", {
|
||||
renderId: input.renderId,
|
||||
encoder,
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
const failed = failedEncoders.get(deps.renderer) ?? new Set<string>();
|
||||
failed.add(encoder);
|
||||
failedEncoders.set(deps.renderer, failed);
|
||||
}
|
||||
const fallback = compile(software);
|
||||
await executePlan(input, fallback, update);
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/** Segments the auto-fix may swap in: the project's material, never paid-only. */
|
||||
|
||||
208
packages/testkit/src/fixtures/media/synthetic-media.ts
Normal file
208
packages/testkit/src/fixtures/media/synthetic-media.ts
Normal file
@ -0,0 +1,208 @@
|
||||
import { execFile, execFileSync } from "node:child_process";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
/*
|
||||
* Synthetic media fixtures for the render integration test, generated with
|
||||
* ffmpeg lavfi sources at test time (nothing binary is checked in). No real
|
||||
* people, faces, places, or devices: the "face" is a flat generated disc, the
|
||||
* picture is testsrc2, the sound is a sine tone, and the device tags are
|
||||
* placeholders that every export must drop.
|
||||
*/
|
||||
|
||||
const run = promisify(execFile);
|
||||
|
||||
export function hasFfmpeg(): boolean {
|
||||
try {
|
||||
execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });
|
||||
execFileSync("ffprobe", ["-version"], { stdio: "ignore" });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function ffmpeg(args: string[]): Promise<void> {
|
||||
await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...args]);
|
||||
}
|
||||
|
||||
/** Tags a phone writes (GPS, make, model, capture time, serial in a handler name). */
|
||||
export const PHONE_TAGS: Record<string, string> = {
|
||||
creation_time: "2020-01-01T10:00:00Z",
|
||||
location: "+00.0000+000.0000/",
|
||||
"com.apple.quicktime.location.ISO6709": "+00.0000+000.0000/",
|
||||
make: "ExampleCam",
|
||||
model: "ExampleCam One",
|
||||
"com.apple.quicktime.software": "ExampleCam OS 1.0",
|
||||
};
|
||||
|
||||
export interface SyntheticClipOptions {
|
||||
seconds: number;
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
/** Sine level in dBFS; quiet on purpose so loudness normalization has work to do. */
|
||||
volumeDb: number;
|
||||
/**
|
||||
* Draw a skin-toned disc ("face") centred at this normalized point with this
|
||||
* radius relative to the height. The media emulator reports faces for files
|
||||
* named with `__faces-1__`; the disc sits where its first face box lies.
|
||||
*/
|
||||
face?: { cx: number; cy: number; radius: number };
|
||||
}
|
||||
|
||||
/** testsrc2 + sine, H.264/AAC MP4 with phone-style tags. */
|
||||
export async function writeSyntheticClip(file: string, options: SyntheticClipOptions) {
|
||||
await mkdir(path.dirname(file), { recursive: true });
|
||||
const { seconds, width, height, fps } = options;
|
||||
const inputs = [
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
`testsrc2=size=${width}x${height}:rate=${fps}:duration=${seconds}`,
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
`sine=frequency=330:sample_rate=48000:duration=${seconds}`,
|
||||
];
|
||||
let video = "[0:v]null[v]";
|
||||
if (options.face) {
|
||||
const r = Math.round(options.face.radius * height);
|
||||
const size = r * 2;
|
||||
const x = Math.round(options.face.cx * width - r);
|
||||
const y = Math.round(options.face.cy * height - r);
|
||||
inputs.push("-f", "lavfi", "-i", `color=c=black:size=${size}x${size}:rate=${fps}`);
|
||||
video = [
|
||||
`[2:v]format=rgba,geq=r=230:g=190:b=160:a='if(lte(hypot(X-${r},Y-${r}),${r}),255,0)'[disc]`,
|
||||
`[0:v][disc]overlay=x=${x}:y=${y}:shortest=1[v]`,
|
||||
].join(";");
|
||||
}
|
||||
const tags = Object.entries(PHONE_TAGS).flatMap(([key, value]) => [
|
||||
"-metadata",
|
||||
`${key}=${value}`,
|
||||
]);
|
||||
await ffmpeg([
|
||||
...inputs,
|
||||
"-filter_complex",
|
||||
`${video};[1:a]volume=${options.volumeDb}dB[a]`,
|
||||
"-map",
|
||||
"[v]",
|
||||
"-map",
|
||||
"[a]",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-t",
|
||||
String(seconds),
|
||||
"-movflags",
|
||||
"+use_metadata_tags",
|
||||
...tags,
|
||||
"-metadata:s:v",
|
||||
"handler_name=ExampleCam serial 0000",
|
||||
file,
|
||||
]);
|
||||
}
|
||||
|
||||
/** Sine music bed as WAV. */
|
||||
export async function writeMusicBed(file: string, seconds: number): Promise<void> {
|
||||
await mkdir(path.dirname(file), { recursive: true });
|
||||
await ffmpeg([
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
`sine=frequency=220:sample_rate=48000:duration=${seconds}`,
|
||||
"-af",
|
||||
"volume=-18dB",
|
||||
file,
|
||||
]);
|
||||
}
|
||||
|
||||
/** Flat square PNG used as the brand logo and watermark. */
|
||||
export async function writeLogo(file: string): Promise<void> {
|
||||
await mkdir(path.dirname(file), { recursive: true });
|
||||
await ffmpeg(["-f", "lavfi", "-i", "color=c=white:size=96x96", "-frames:v", "1", file]);
|
||||
}
|
||||
|
||||
/** Decode one frame as 8-bit grayscale at the given size. */
|
||||
export async function grayFrame(
|
||||
file: string,
|
||||
tMs: number,
|
||||
width: number,
|
||||
height: number,
|
||||
): Promise<Uint8Array> {
|
||||
const { stdout } = await run(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-ss",
|
||||
(tMs / 1000).toFixed(3),
|
||||
"-i",
|
||||
file,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-vf",
|
||||
`scale=${width}:${height}:flags=area`,
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"gray",
|
||||
"-",
|
||||
],
|
||||
{ encoding: "buffer", maxBuffer: 64 * 1024 * 1024 },
|
||||
);
|
||||
if (stdout.length !== width * height) throw new Error("unexpected frame size");
|
||||
return new Uint8Array(stdout);
|
||||
}
|
||||
|
||||
export interface PixelBox {
|
||||
x0: number;
|
||||
y0: number;
|
||||
x1: number;
|
||||
y1: number;
|
||||
}
|
||||
|
||||
/** Variance of the pixels inside a box. */
|
||||
export function variance(frame: Uint8Array, width: number, box: PixelBox): number {
|
||||
let sum = 0;
|
||||
let squares = 0;
|
||||
let count = 0;
|
||||
for (let y = box.y0; y < box.y1; y++) {
|
||||
for (let x = box.x0; x < box.x1; x++) {
|
||||
const value = frame[y * width + x]!;
|
||||
sum += value;
|
||||
squares += value * value;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
const mean = sum / count;
|
||||
return squares / count - mean * mean;
|
||||
}
|
||||
|
||||
/** Mean absolute difference of two frames outside a box. */
|
||||
export function meanDifferenceOutside(
|
||||
a: Uint8Array,
|
||||
b: Uint8Array,
|
||||
width: number,
|
||||
height: number,
|
||||
box: PixelBox,
|
||||
): number {
|
||||
let total = 0;
|
||||
let count = 0;
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
if (x >= box.x0 && x < box.x1 && y >= box.y0 && y < box.y1) continue;
|
||||
total += Math.abs(a[y * width + x]! - b[y * width + x]!);
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
return total / count;
|
||||
}
|
||||
359
packages/testkit/src/media-render.docker.test.ts
Normal file
359
packages/testkit/src/media-render.docker.test.ts
Normal file
@ -0,0 +1,359 @@
|
||||
import path from "node:path";
|
||||
import type { BoxKeyframe, Edl } from "@rakazo/contracts";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { MediaPrivacyService } from "../../adapters/src/media-context.js";
|
||||
import { EmulatorMediaInferenceProvider } from "../../adapters/src/media-emulator.js";
|
||||
import { FfmpegMediaRenderer } from "../../adapters/src/media-ffmpeg-renderer.js";
|
||||
import type { RenderHarness } from "../../adapters/src/media-render.fixtures.js";
|
||||
import {
|
||||
createFakePrivacy,
|
||||
createRenderHarness,
|
||||
renderTestKit,
|
||||
renderTestProfile,
|
||||
} from "../../adapters/src/media-render.fixtures.js";
|
||||
import {
|
||||
createMediaRenderHandler,
|
||||
createMediaRenderService,
|
||||
} from "../../adapters/src/media-render.js";
|
||||
import { findMetadataLeaks } from "../../media/src/gate.js";
|
||||
import type { PixelBox } from "./fixtures/media/synthetic-media.js";
|
||||
import {
|
||||
grayFrame,
|
||||
hasFfmpeg,
|
||||
meanDifferenceOutside,
|
||||
variance,
|
||||
writeLogo,
|
||||
writeMusicBed,
|
||||
writeSyntheticClip,
|
||||
} from "./fixtures/media/synthetic-media.js";
|
||||
|
||||
/*
|
||||
* Real render integration (spec 13, Docker job). Synthetic fixtures are made
|
||||
* with ffmpeg lavfi; inference runs on the deterministic emulator; the render
|
||||
* goes through the real service, job handler, compiler, and ffmpeg renderer,
|
||||
* and every result is checked with ffprobe. Runs wherever ffmpeg is installed
|
||||
* (the CI test job and the media Docker job) and skips cleanly without it.
|
||||
*
|
||||
* Redactions belong to the privacy slice. Its stand-in here applies EDL region
|
||||
* redactions through the emulator's `redact` (the same MediaInferenceProvider
|
||||
* call the privacy service uses) and refuses to return an unredacted source.
|
||||
*/
|
||||
|
||||
const FPS = 30;
|
||||
const FRAME_MS = 1000 / FPS;
|
||||
/** Grayscale variance a blurred region must stay under. */
|
||||
const VARIANCE_THRESHOLD = 15;
|
||||
/**
|
||||
* The generated "face": a flat disc inside every box the media emulator can
|
||||
* report as the first face of a `__faces-1__` file (boxes are seeded by the
|
||||
* file, but always contain this point and radius).
|
||||
*/
|
||||
const FACE = { cx: 0.19, cy: 0.28, radius: 0.05 };
|
||||
|
||||
const vertical = renderTestProfile();
|
||||
const landscape = renderTestProfile({
|
||||
id: "example-landscape-test",
|
||||
name: "Example landscape test",
|
||||
video: {
|
||||
aspectRatios: ["16:9"],
|
||||
width: 640,
|
||||
height: 360,
|
||||
fps: FPS,
|
||||
codec: "h264",
|
||||
maxBitrateKbps: 6000,
|
||||
maxFileSizeMb: 500,
|
||||
},
|
||||
safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 },
|
||||
});
|
||||
|
||||
const motion = {
|
||||
id: "none",
|
||||
templates: () => [],
|
||||
render: async (): Promise<never> => {
|
||||
throw new Error("no motion templates in this plan");
|
||||
},
|
||||
};
|
||||
|
||||
/** Privacy stand-in: region redactions through the inference provider, fail closed. */
|
||||
function emulatorPrivacy(
|
||||
harness: RenderHarness,
|
||||
inference: EmulatorMediaInferenceProvider,
|
||||
): MediaPrivacyService {
|
||||
const base = createFakePrivacy(harness.sources);
|
||||
return {
|
||||
...base,
|
||||
async prepareSources(input) {
|
||||
const sources = new Map();
|
||||
for (const clip of input.edl.tracks.video) {
|
||||
const source = harness.sources.get(clip.assetId);
|
||||
if (!source) throw new Error(`no source for ${clip.assetId}`);
|
||||
const regions = input.edl.redactions.regions.filter((r) => r.clipId === clip.id);
|
||||
if (regions.length === 0) {
|
||||
sources.set(clip.assetId, source);
|
||||
continue;
|
||||
}
|
||||
const outPath = path.join(input.workDir, `redacted-${clip.id}.mp4`);
|
||||
const result = await inference.redact({
|
||||
path: source.path,
|
||||
outPath,
|
||||
redactions: regions.map((region) => ({
|
||||
keyframes: region.keyframes,
|
||||
mode: region.treatment === "cover" ? "cover" : "blur",
|
||||
paddingPct: 10,
|
||||
featherPct: 0,
|
||||
})),
|
||||
signal: input.signal,
|
||||
});
|
||||
if (result.framesProcessed <= 0) throw new Error("redaction processed no frames");
|
||||
sources.set(clip.assetId, { ...source, path: result.outPath });
|
||||
}
|
||||
return { sources, aiAlterations: [], fallbacks: [] };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
interface Fixture {
|
||||
harness: RenderHarness;
|
||||
inference: EmulatorMediaInferenceProvider;
|
||||
renderer: FfmpegMediaRenderer;
|
||||
assetId: string;
|
||||
sourcePath: string;
|
||||
musicArtifactId: string;
|
||||
render: (edl: unknown, quality: "draft" | "final") => Promise<{ id: string; dir: string }>;
|
||||
}
|
||||
|
||||
describe.skipIf(!hasFfmpeg())("media render integration with real ffmpeg", () => {
|
||||
let fx: Fixture;
|
||||
|
||||
beforeAll(async () => {
|
||||
const renderer = new FfmpegMediaRenderer();
|
||||
const harness = await createRenderHarness({
|
||||
profiles: [vertical, landscape],
|
||||
renderer,
|
||||
motion,
|
||||
});
|
||||
const inference = new EmulatorMediaInferenceProvider({ root: harness.mediaRoot });
|
||||
harness.deps.inference = inference;
|
||||
const name = "lake__faces-1__speech__.mp4";
|
||||
const assetId = await harness.createAsset(name);
|
||||
const sourcePath = harness.assetPath(assetId, name);
|
||||
await writeSyntheticClip(sourcePath, {
|
||||
seconds: 6,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
fps: FPS,
|
||||
volumeDb: -32,
|
||||
face: FACE,
|
||||
});
|
||||
harness.setSource(assetId, {
|
||||
path: sourcePath,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
durationMs: 6000,
|
||||
fps: FPS,
|
||||
hasAudio: true,
|
||||
rotation: 0,
|
||||
});
|
||||
const timesMs = Array.from({ length: 13 }, (_, i) => i * 500);
|
||||
await harness.addAnalysis(
|
||||
assetId,
|
||||
"transcript",
|
||||
await inference.transcribe({ path: sourcePath, language: "en" }),
|
||||
);
|
||||
await harness.addAnalysis(
|
||||
assetId,
|
||||
"saliency",
|
||||
await inference.saliency({ path: sourcePath, timesMs }),
|
||||
);
|
||||
|
||||
const brandDir = path.join(harness.mediaRoot, "fixtures");
|
||||
await writeLogo(path.join(brandDir, "logo.png"));
|
||||
await writeMusicBed(path.join(brandDir, "bed.wav"), 6);
|
||||
const logoArtifactId = await harness.addArtifact(
|
||||
path.join(brandDir, "logo.png"),
|
||||
"logo.png",
|
||||
"image/png",
|
||||
);
|
||||
const musicArtifactId = await harness.addArtifact(
|
||||
path.join(brandDir, "bed.wav"),
|
||||
"bed.wav",
|
||||
"audio/wav",
|
||||
);
|
||||
const kit = renderTestKit({ logoArtifactId, musicArtifactId });
|
||||
|
||||
const privacy = emulatorPrivacy(harness, inference);
|
||||
const service = createMediaRenderService(harness.deps);
|
||||
const handler = createMediaRenderHandler(harness.deps, privacy);
|
||||
const actor = { spaceId: harness.spaceId, userId: harness.userId };
|
||||
fx = {
|
||||
harness,
|
||||
inference,
|
||||
renderer,
|
||||
assetId,
|
||||
sourcePath,
|
||||
musicArtifactId,
|
||||
async render(edl, quality) {
|
||||
const projectId = await harness.createProject({
|
||||
assetIds: [assetId],
|
||||
kit,
|
||||
profileId: (edl as Edl).output.profileId,
|
||||
});
|
||||
const plan = await harness.createPlan(projectId, edl);
|
||||
const started = await service.start(actor, { planId: plan.id, quality });
|
||||
await handler({ renderId: started.id });
|
||||
const render = await harness.deps.repos.getRender(harness.spaceId, started.id);
|
||||
expect(render.error).toBeNull();
|
||||
expect(render.status).toBe("done");
|
||||
return {
|
||||
id: render.id,
|
||||
dir: path.join(harness.mediaRoot, harness.spaceId, "renders", render.id),
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await fx?.harness.cleanup();
|
||||
});
|
||||
|
||||
it("renders 16:9 to 9:16 with auto-reframe, captions, brand overlay and loudness", async () => {
|
||||
const edl = {
|
||||
edlVersion: 1,
|
||||
output: {
|
||||
profileId: vertical.id,
|
||||
width: vertical.video.width,
|
||||
height: vertical.video.height,
|
||||
fps: FPS,
|
||||
durationTargetMs: 4000,
|
||||
durationHardMinMs: 1000,
|
||||
durationHardMaxMs: 120_000,
|
||||
},
|
||||
tracks: {
|
||||
video: [
|
||||
{ id: "c1", assetId: fx.assetId, inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" },
|
||||
{ id: "c2", assetId: fx.assetId, inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" },
|
||||
],
|
||||
audio: [
|
||||
{
|
||||
id: "bed",
|
||||
kind: "music",
|
||||
artifactId: fx.musicArtifactId,
|
||||
inMs: 0,
|
||||
outMs: 4000,
|
||||
atMs: 0,
|
||||
gainDb: -6,
|
||||
fadeOutMs: 500,
|
||||
duckUnderSpeech: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "cut", durationMs: 0 }],
|
||||
captions: { mode: "both", presetId: "clean" },
|
||||
overlays: [
|
||||
{ id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 },
|
||||
{ id: "lt", kind: "lower_third", atMs: 500, durationMs: 1500, text: "Example: 50% off" },
|
||||
],
|
||||
metadata: { title: "Example teaser" },
|
||||
};
|
||||
const render = await fx.render(edl, "final");
|
||||
const video = path.join(render.dir, "video.mp4");
|
||||
const probe = await fx.renderer.probe(video, { measureLoudness: true });
|
||||
|
||||
expect(Math.abs(probe.durationMs - 4000)).toBeLessThanOrEqual(FRAME_MS + 1);
|
||||
expect(probe.width / probe.height).toBeCloseTo(9 / 16, 3);
|
||||
expect(probe.width).toBe(vertical.video.width);
|
||||
expect(probe.fps).toBe(FPS);
|
||||
expect(probe.videoCodec).toBe("h264");
|
||||
expect(Math.abs(probe.integratedLufs! - vertical.audio.targetLufs)).toBeLessThanOrEqual(1);
|
||||
expect(probe.truePeakDb!).toBeLessThanOrEqual(vertical.audio.truePeakDb + 0.5);
|
||||
|
||||
expect(findMetadataLeaks(probe.tags)).toEqual([]);
|
||||
const tagText = JSON.stringify(probe.tags).toLowerCase();
|
||||
for (const leak of ["location", "iso6709", "make", "model", "creation_time", "examplecam"]) {
|
||||
expect(tagText).not.toContain(leak);
|
||||
}
|
||||
expect(probe.tags["format.title"]).toBe("Example teaser");
|
||||
|
||||
const poster = await fx.renderer.probe(path.join(render.dir, "poster.jpg"));
|
||||
expect(findMetadataLeaks(poster.tags)).toEqual([]);
|
||||
|
||||
const stored = await fx.harness.deps.repos.getRender(fx.harness.spaceId, render.id);
|
||||
expect(stored.gateReport?.passed).toBe(true);
|
||||
expect(stored.outputs.map((output) => output.fileName).sort()).toEqual([
|
||||
"captions.srt",
|
||||
"captions.vtt",
|
||||
"poster.jpg",
|
||||
"video.mp4",
|
||||
]);
|
||||
}, 240_000);
|
||||
|
||||
it("tracks the generated face disc with the emulator", async () => {
|
||||
const tracks = await fx.inference.trackFaces({ path: fx.sourcePath, sampleFps: 2 });
|
||||
expect(tracks.tracks).toHaveLength(1);
|
||||
const disc = { x0: FACE.cx - FACE.radius * (9 / 16), x1: FACE.cx + FACE.radius * (9 / 16) };
|
||||
for (const { box } of tracks.tracks[0]!.keyframes) {
|
||||
expect(box.x).toBeLessThanOrEqual(disc.x0);
|
||||
expect(box.x + box.w).toBeGreaterThanOrEqual(disc.x1);
|
||||
expect(box.y).toBeLessThanOrEqual(FACE.cy - FACE.radius);
|
||||
expect(box.y + box.h).toBeGreaterThanOrEqual(FACE.cy + FACE.radius);
|
||||
}
|
||||
});
|
||||
|
||||
it("blurs a redaction region below the variance threshold and leaves the rest unchanged", async () => {
|
||||
// A static region over testsrc2's fine checkerboard, held for the whole clip.
|
||||
const box = { x: 0.68, y: 0.64, w: 0.14, h: 0.2 };
|
||||
const keyframes: BoxKeyframe[] = [
|
||||
{ tMs: 0, box },
|
||||
{ tMs: 3000, box },
|
||||
];
|
||||
const edl = (regions: unknown[]) => ({
|
||||
edlVersion: 1,
|
||||
output: {
|
||||
profileId: landscape.id,
|
||||
width: landscape.video.width,
|
||||
height: landscape.video.height,
|
||||
fps: FPS,
|
||||
durationTargetMs: 3000,
|
||||
durationHardMinMs: 1000,
|
||||
durationHardMaxMs: 120_000,
|
||||
},
|
||||
tracks: {
|
||||
video: [{ id: "c1", assetId: fx.assetId, inMs: 0, outMs: 3000, atMs: 0, reframe: "none" }],
|
||||
},
|
||||
redactions: { regions },
|
||||
metadata: {},
|
||||
});
|
||||
const plain = await fx.render(edl([]), "draft");
|
||||
const redacted = await fx.render(
|
||||
edl([{ id: "region", clipId: "c1", keyframes, treatment: "blur" }]),
|
||||
"draft",
|
||||
);
|
||||
|
||||
const W = landscape.video.width;
|
||||
const H = landscape.video.height;
|
||||
const inner: PixelBox = {
|
||||
x0: Math.ceil((box.x + box.w * 0.15) * W),
|
||||
y0: Math.ceil((box.y + box.h * 0.15) * H),
|
||||
x1: Math.floor((box.x + box.w * 0.85) * W),
|
||||
y1: Math.floor((box.y + box.h * 0.85) * H),
|
||||
};
|
||||
// Padding (10 %) and blur bleed at the edge stay out of the comparison.
|
||||
const margin = 0.04;
|
||||
const outer: PixelBox = {
|
||||
x0: Math.floor((box.x - margin) * W),
|
||||
y0: Math.floor((box.y - margin) * H),
|
||||
x1: Math.ceil((box.x + box.w + margin) * W),
|
||||
y1: Math.ceil((box.y + box.h + margin) * H),
|
||||
};
|
||||
for (const tMs of [500, 1500, 2500]) {
|
||||
const before = await grayFrame(path.join(plain.dir, "video.mp4"), tMs, W, H);
|
||||
const after = await grayFrame(path.join(redacted.dir, "video.mp4"), tMs, W, H);
|
||||
const original = variance(before, W, inner);
|
||||
const blurred = variance(after, W, inner);
|
||||
const outside = meanDifferenceOutside(before, after, W, H, outer);
|
||||
expect(original).toBeGreaterThan(VARIANCE_THRESHOLD * 4);
|
||||
expect(blurred).toBeLessThan(VARIANCE_THRESHOLD);
|
||||
expect(outside).toBeLessThan(1.5);
|
||||
}
|
||||
}, 240_000);
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user