merge: W2 media-w2-render
This commit is contained in:
Claude
commit
0e82982452
584
packages/adapters/src/media-ffmpeg-renderer.test.ts
Normal file
584
packages/adapters/src/media-ffmpeg-renderer.test.ts
Normal file
@ -0,0 +1,584 @@
|
|||||||
|
import {
|
||||||
|
chmod,
|
||||||
|
mkdir,
|
||||||
|
mkdtemp,
|
||||||
|
readdir,
|
||||||
|
readFile,
|
||||||
|
realpath,
|
||||||
|
rm,
|
||||||
|
stat,
|
||||||
|
writeFile,
|
||||||
|
} from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { MediaRenderProgress } from "@rakazo/adapter-kit";
|
||||||
|
import type { RenderPlan } from "@rakazo/contracts";
|
||||||
|
import { EdlSchema } from "@rakazo/contracts";
|
||||||
|
import type { CompileInput } from "@rakazo/media";
|
||||||
|
import { compileRenderPlan, findMetadataLeaks } from "@rakazo/media";
|
||||||
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
FfmpegMediaRenderer,
|
||||||
|
findTwoPassLoudnorm,
|
||||||
|
parseEbur128Summary,
|
||||||
|
parseFrameRate,
|
||||||
|
parseLoudnormJson,
|
||||||
|
parseProbeReport,
|
||||||
|
} from "./media-ffmpeg-renderer.js";
|
||||||
|
import { MediaToolMissingError } from "./media-folder-source-probe.js";
|
||||||
|
import { hasMediaTools, renderTestProfile, writeSyntheticClip } from "./media-render.fixtures.js";
|
||||||
|
|
||||||
|
describe("ffmpeg output parsing", () => {
|
||||||
|
it("parses frame rates", () => {
|
||||||
|
expect(parseFrameRate("30000/1001")).toBe(29.97);
|
||||||
|
expect(parseFrameRate("25/1")).toBe(25);
|
||||||
|
expect(parseFrameRate("0/0")).toBe(0);
|
||||||
|
expect(parseFrameRate(undefined)).toBe(0);
|
||||||
|
expect(parseFrameRate("abc")).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("flattens format and stream tags and falls back to the file size", () => {
|
||||||
|
const report = parseProbeReport(
|
||||||
|
{
|
||||||
|
streams: [
|
||||||
|
{
|
||||||
|
codec_type: "video",
|
||||||
|
codec_name: "h264",
|
||||||
|
width: 640,
|
||||||
|
height: 360,
|
||||||
|
r_frame_rate: "30/1",
|
||||||
|
},
|
||||||
|
{ codec_type: "audio", codec_name: "aac", tags: { language: "und" } },
|
||||||
|
],
|
||||||
|
format: { duration: "2.5", tags: { title: "Example", location: "+00+000/" } },
|
||||||
|
},
|
||||||
|
1234,
|
||||||
|
);
|
||||||
|
expect(report).toEqual({
|
||||||
|
durationMs: 2500,
|
||||||
|
width: 640,
|
||||||
|
height: 360,
|
||||||
|
fps: 30,
|
||||||
|
videoCodec: "h264",
|
||||||
|
audioCodec: "aac",
|
||||||
|
sizeBytes: 1234,
|
||||||
|
integratedLufs: null,
|
||||||
|
truePeakDb: null,
|
||||||
|
tags: {
|
||||||
|
"format.title": "Example",
|
||||||
|
"format.location": "+00+000/",
|
||||||
|
"stream.1.language": "und",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads the ebur128 summary and treats the silence gate as no programme", () => {
|
||||||
|
const stderr = `[Parsed_ebur128_0 @ 0x1] Summary:
|
||||||
|
|
||||||
|
Integrated loudness:
|
||||||
|
I: -14.2 LUFS
|
||||||
|
Threshold: -24.3 LUFS
|
||||||
|
|
||||||
|
True peak:
|
||||||
|
Peak: -1.4 dBFS`;
|
||||||
|
expect(parseEbur128Summary(stderr)).toEqual({ integratedLufs: -14.2, truePeakDb: -1.4 });
|
||||||
|
expect(parseEbur128Summary("Summary:\n I: -70.0 LUFS\n True peak:\n Peak: -inf dBFS")).toEqual({
|
||||||
|
integratedLufs: null,
|
||||||
|
truePeakDb: null,
|
||||||
|
});
|
||||||
|
expect(parseEbur128Summary("no summary")).toEqual({ integratedLufs: null, truePeakDb: null });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("finds exactly one linear loudnorm without measured values", () => {
|
||||||
|
const graph = "[a]anull[b];[b]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]";
|
||||||
|
const site = findTwoPassLoudnorm(graph);
|
||||||
|
expect(site?.options).toBe("I=-14:TP=-1:LRA=11:linear=true");
|
||||||
|
expect(graph.slice(site!.start, site!.end)).toBe(site!.options);
|
||||||
|
expect(findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true:measured_I=-20[b]")).toBeNull();
|
||||||
|
expect(findTwoPassLoudnorm("[a]loudnorm=I=-14[b]")).toBeNull();
|
||||||
|
expect(
|
||||||
|
findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true[b];[c]loudnorm=I=-14:linear=true[d]"),
|
||||||
|
).toBeNull();
|
||||||
|
expect(findTwoPassLoudnorm("[a]anull[b]")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("parses the loudnorm measurement block", () => {
|
||||||
|
const stderr = `[Parsed_loudnorm_3 @ 0x55]
|
||||||
|
{
|
||||||
|
\t"input_i" : "-31.20",
|
||||||
|
\t"input_tp" : "-20.11",
|
||||||
|
\t"input_lra" : "0.00",
|
||||||
|
\t"input_thresh" : "-41.20",
|
||||||
|
\t"output_i" : "-14.02",
|
||||||
|
\t"output_tp" : "-1.00",
|
||||||
|
\t"output_lra" : "0.00",
|
||||||
|
\t"output_thresh" : "-24.02",
|
||||||
|
\t"normalization_type" : "dynamic",
|
||||||
|
\t"target_offset" : "0.02"
|
||||||
|
}`;
|
||||||
|
expect(parseLoudnormJson(stderr)).toEqual({
|
||||||
|
inputI: -31.2,
|
||||||
|
inputTp: -20.11,
|
||||||
|
inputLra: 0,
|
||||||
|
inputThresh: -41.2,
|
||||||
|
targetOffset: 0.02,
|
||||||
|
});
|
||||||
|
expect(parseLoudnormJson(stderr.replace('"-31.20"', '"-inf"'))).toBeNull();
|
||||||
|
expect(parseLoudnormJson("nothing")).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("hardware encoder detection", () => {
|
||||||
|
let dir: string;
|
||||||
|
beforeAll(async () => {
|
||||||
|
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-hw-")));
|
||||||
|
});
|
||||||
|
afterAll(async () => {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
/** A fake ffmpeg that lists encoders and succeeds only for the given test encodes. */
|
||||||
|
async function fakeFfmpeg(name: string, working: string[]): Promise<string> {
|
||||||
|
const file = path.join(dir, name);
|
||||||
|
const cases = working.map((encoder) => `*" ${encoder} "*) exit 0 ;;`).join("\n ");
|
||||||
|
await writeFile(
|
||||||
|
file,
|
||||||
|
`#!/bin/sh
|
||||||
|
case " $* " in
|
||||||
|
*" -encoders "*)
|
||||||
|
echo " V....D libx264 libx264"
|
||||||
|
echo " V....D h264_nvenc NVIDIA NVENC H.264 encoder"
|
||||||
|
echo " V....D hevc_nvenc NVIDIA NVENC hevc encoder"
|
||||||
|
echo " V....D h264_vaapi H.264/AVC (VAAPI)"
|
||||||
|
exit 0 ;;
|
||||||
|
${cases}
|
||||||
|
esac
|
||||||
|
echo "No device available" >&2
|
||||||
|
exit 1
|
||||||
|
`,
|
||||||
|
);
|
||||||
|
await chmod(file, 0o755);
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
it("keeps only listed encoders whose one-frame test encode works", async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer({
|
||||||
|
ffmpegPath: await fakeFfmpeg("ffmpeg-nvenc", ["h264_nvenc"]),
|
||||||
|
vaapiDevice: path.join(dir, "no-render-node"),
|
||||||
|
});
|
||||||
|
expect(await renderer.hardwareEncoders()).toEqual(["h264_nvenc"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to software when no hardware encoder works or ffmpeg is missing", async () => {
|
||||||
|
const broken = new FfmpegMediaRenderer({ ffmpegPath: await fakeFfmpeg("ffmpeg-none", []) });
|
||||||
|
expect(await broken.hardwareEncoders()).toEqual([]);
|
||||||
|
const missing = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") });
|
||||||
|
expect(await missing.hardwareEncoders()).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips VAAPI encoders without a render node", async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer({
|
||||||
|
ffmpegPath: await fakeFfmpeg("ffmpeg-vaapi", ["h264_vaapi"]),
|
||||||
|
vaapiDevice: path.join(dir, "no-render-node"),
|
||||||
|
});
|
||||||
|
expect(await renderer.hardwareEncoders()).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.skipIf(!hasMediaTools)("never reports an encoder this host cannot run", async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer();
|
||||||
|
for (const encoder of await renderer.hardwareEncoders()) {
|
||||||
|
expect(encoder).toMatch(/^(h264|hevc)_(vaapi|videotoolbox|nvenc)$/);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("plan safety", () => {
|
||||||
|
let dir: string;
|
||||||
|
beforeAll(async () => {
|
||||||
|
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-plan-")));
|
||||||
|
});
|
||||||
|
afterAll(async () => {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses writes outside the work directory", async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer({ root: dir });
|
||||||
|
const workDir = path.join(dir, "work");
|
||||||
|
const plan: RenderPlan = {
|
||||||
|
quality: "draft",
|
||||||
|
workDir,
|
||||||
|
steps: [
|
||||||
|
{ kind: "write_file", id: "escape", path: path.join(dir, "outside.txt"), content: "x" },
|
||||||
|
],
|
||||||
|
outputs: [],
|
||||||
|
};
|
||||||
|
await expect(renderer.render(plan, {})).rejects.toThrow("outside");
|
||||||
|
await expect(stat(path.join(dir, "outside.txt"))).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses a work directory outside the media root", async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer({ root: path.join(dir, "root") });
|
||||||
|
await mkdir(path.join(dir, "root"), { recursive: true });
|
||||||
|
const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] };
|
||||||
|
await expect(renderer.render(plan, {})).rejects.toThrow("outside");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("requires motion, inpaint and identity steps to be prepared first", async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer();
|
||||||
|
const plan: RenderPlan = {
|
||||||
|
quality: "final",
|
||||||
|
workDir: path.join(dir, "motion"),
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
kind: "motion",
|
||||||
|
id: "intro",
|
||||||
|
template: { templateId: "intro-minimal", params: {}, durationMs: 1000 },
|
||||||
|
width: 320,
|
||||||
|
height: 180,
|
||||||
|
fps: 30,
|
||||||
|
outPath: path.join(dir, "motion", "intro.mov"),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
outputs: [],
|
||||||
|
};
|
||||||
|
await expect(renderer.render(plan, {})).rejects.toThrow("was not prepared");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a missing ffmpeg instead of pretending to render", async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") });
|
||||||
|
const out = path.join(dir, "missing", "out.mp4");
|
||||||
|
const plan: RenderPlan = {
|
||||||
|
quality: "draft",
|
||||||
|
workDir: path.join(dir, "missing"),
|
||||||
|
steps: [{ kind: "ffmpeg", id: "render", argv: ["-version"], produces: [out], durationMs: 1 }],
|
||||||
|
outputs: [{ kind: "video", path: out }],
|
||||||
|
};
|
||||||
|
await expect(renderer.render(plan, {})).rejects.toBeInstanceOf(MediaToolMissingError);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects an already aborted render without spawning anything", async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") });
|
||||||
|
const controller = new AbortController();
|
||||||
|
controller.abort();
|
||||||
|
const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] };
|
||||||
|
await expect(renderer.render(plan, { signal: controller.signal })).rejects.toMatchObject({
|
||||||
|
name: "AbortError",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("two-pass loudness", () => {
|
||||||
|
let dir: string;
|
||||||
|
let ffmpegPath: string;
|
||||||
|
let log: string;
|
||||||
|
const GRAPH = "[0:a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]";
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-loudnorm-")));
|
||||||
|
log = path.join(dir, "calls.log");
|
||||||
|
ffmpegPath = path.join(dir, "ffmpeg");
|
||||||
|
// Logs each call; the measurement pass prints loudnorm's JSON, the render
|
||||||
|
// pass writes its output file (the last argument).
|
||||||
|
await writeFile(
|
||||||
|
ffmpegPath,
|
||||||
|
`#!/bin/sh
|
||||||
|
echo "$*" >> "${log}"
|
||||||
|
case " $* " in
|
||||||
|
*print_format=json*|*.measure.txt*)
|
||||||
|
printf '[Parsed_loudnorm_0 @ 0x1] \\n{\\n "input_i" : "-31.20",\\n "input_tp" : "-20.11",\\n "input_lra" : "0.00",\\n "input_thresh" : "-41.20",\\n "target_offset" : "0.02"\\n}\\n' >&2
|
||||||
|
exit 0 ;;
|
||||||
|
esac
|
||||||
|
for last; do :; done
|
||||||
|
[ "$last" = "-" ] || echo rendered > "$last"
|
||||||
|
`,
|
||||||
|
);
|
||||||
|
await chmod(ffmpegPath, 0o755);
|
||||||
|
});
|
||||||
|
afterAll(async () => {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
async function calls(): Promise<string[]> {
|
||||||
|
return (await readFile(log, "utf8")).trim().split("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
it("measures first, then renders with the measured values", async () => {
|
||||||
|
await rm(log, { force: true });
|
||||||
|
const workDir = path.join(dir, "inline");
|
||||||
|
const out = path.join(workDir, "out.m4a");
|
||||||
|
const plan: RenderPlan = {
|
||||||
|
quality: "final",
|
||||||
|
workDir,
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
kind: "ffmpeg",
|
||||||
|
id: "render",
|
||||||
|
argv: ["-i", "in.wav", "-filter_complex", GRAPH, "-map", "[aout]", "-t", "2", out],
|
||||||
|
produces: [out],
|
||||||
|
durationMs: 2000,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
outputs: [],
|
||||||
|
};
|
||||||
|
await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {});
|
||||||
|
const [measure, render] = await calls();
|
||||||
|
expect(measure).toContain("linear=true:print_format=json");
|
||||||
|
expect(measure).toContain("-map [aout] -t 2 -f null -");
|
||||||
|
expect(measure).not.toContain(out);
|
||||||
|
expect(render).toContain(
|
||||||
|
"linear=true:measured_I=-31.20:measured_TP=-20.11:measured_LRA=0.00:measured_thresh=-41.20:offset=0.02,aresample=48000",
|
||||||
|
);
|
||||||
|
expect(render?.endsWith(out)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rewrites a filtergraph script and removes the measurement script", async () => {
|
||||||
|
await rm(log, { force: true });
|
||||||
|
const workDir = path.join(dir, "script");
|
||||||
|
const script = path.join(workDir, "filtergraph.txt");
|
||||||
|
const out = path.join(workDir, "out.m4a");
|
||||||
|
const plan: RenderPlan = {
|
||||||
|
quality: "final",
|
||||||
|
workDir,
|
||||||
|
steps: [
|
||||||
|
{ kind: "write_file", id: "filtergraph", path: script, content: GRAPH },
|
||||||
|
{
|
||||||
|
kind: "ffmpeg",
|
||||||
|
id: "render",
|
||||||
|
argv: ["-i", "in.wav", "-filter_complex_script", script, "-map", "[aout]", out],
|
||||||
|
produces: [out],
|
||||||
|
durationMs: 2000,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
outputs: [],
|
||||||
|
};
|
||||||
|
await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {});
|
||||||
|
const [measure] = await calls();
|
||||||
|
expect(measure).toContain(path.join(workDir, "filtergraph.measure.txt"));
|
||||||
|
expect(await readFile(script, "utf8")).toContain("measured_I=-31.20");
|
||||||
|
expect((await readdir(workDir)).sort()).toEqual(["filtergraph.txt", "out.m4a"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails a step whose output was never written and removes what it wrote", async () => {
|
||||||
|
const workDir = path.join(dir, "silent");
|
||||||
|
const out = path.join(workDir, "out.m4a");
|
||||||
|
const script = path.join(workDir, "notes.txt");
|
||||||
|
const plan: RenderPlan = {
|
||||||
|
quality: "draft",
|
||||||
|
workDir,
|
||||||
|
steps: [
|
||||||
|
{ kind: "write_file", id: "notes", path: script, content: "x" },
|
||||||
|
{
|
||||||
|
kind: "ffmpeg",
|
||||||
|
id: "render",
|
||||||
|
// The fake writes its last argument; the declared output stays missing.
|
||||||
|
argv: ["-i", "in.wav", path.join(workDir, "elsewhere.m4a")],
|
||||||
|
produces: [out],
|
||||||
|
durationMs: 1000,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
outputs: [],
|
||||||
|
};
|
||||||
|
await expect(new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {})).rejects.toThrow(
|
||||||
|
"wrote no out.m4a",
|
||||||
|
);
|
||||||
|
await expect(stat(script)).rejects.toThrow();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!hasMediaTools)("real ffmpeg renders", () => {
|
||||||
|
let dir: string;
|
||||||
|
let source: string;
|
||||||
|
const renderer = () => new FfmpegMediaRenderer({ root: dir });
|
||||||
|
const profile = renderTestProfile();
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-ffmpeg-")));
|
||||||
|
source = path.join(dir, "assets", "source.mp4");
|
||||||
|
// Deliberately quiet (-36 dBFS sine) so loudness normalization has real work to do.
|
||||||
|
await writeSyntheticClip(source, { seconds: 6, width: 320, height: 180, volumeDb: -36 });
|
||||||
|
});
|
||||||
|
afterAll(async () => {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
function compile(workDir: string, overrides: Partial<CompileInput> = {}): RenderPlan {
|
||||||
|
const edl = EdlSchema.parse({
|
||||||
|
edlVersion: 1,
|
||||||
|
output: {
|
||||||
|
profileId: profile.id,
|
||||||
|
width: 180,
|
||||||
|
height: 320,
|
||||||
|
fps: 30,
|
||||||
|
durationTargetMs: 4000,
|
||||||
|
durationHardMinMs: 1000,
|
||||||
|
durationHardMaxMs: 120_000,
|
||||||
|
},
|
||||||
|
tracks: {
|
||||||
|
video: [
|
||||||
|
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" },
|
||||||
|
{ id: "c2", assetId: "asset-a", inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
captions: { mode: "both" },
|
||||||
|
overlays: [{ id: "t1", kind: "text", atMs: 500, durationMs: 1500, text: "Example: 50%" }],
|
||||||
|
metadata: { title: "Example teaser" },
|
||||||
|
});
|
||||||
|
return compileRenderPlan({
|
||||||
|
edl,
|
||||||
|
profile,
|
||||||
|
brandKit: null,
|
||||||
|
quality: "draft",
|
||||||
|
workDir,
|
||||||
|
sources: new Map([
|
||||||
|
[
|
||||||
|
"asset-a",
|
||||||
|
{
|
||||||
|
path: source,
|
||||||
|
width: 320,
|
||||||
|
height: 180,
|
||||||
|
durationMs: 6000,
|
||||||
|
fps: 30,
|
||||||
|
hasAudio: true,
|
||||||
|
rotation: 0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
transcripts: new Map([
|
||||||
|
[
|
||||||
|
"asset-a",
|
||||||
|
[
|
||||||
|
{ startMs: 100, endMs: 600, text: "Hello" },
|
||||||
|
{ startMs: 650, endMs: 1200, text: "there." },
|
||||||
|
{ startMs: 3100, endMs: 3800, text: "Bye." },
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
subjects: new Map([
|
||||||
|
[
|
||||||
|
"asset-a",
|
||||||
|
[
|
||||||
|
{ tMs: 0, cx: 0.3, cy: 0.5, strength: 1 },
|
||||||
|
{ tMs: 5000, cx: 0.7, cy: 0.5, strength: 1 },
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
motionClips: new Map(),
|
||||||
|
files: { lutPath: null, watermarkPath: null, fonts: new Map(), audio: new Map() },
|
||||||
|
videoEncoder: "libx264",
|
||||||
|
...overrides,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
it("renders 16:9 to 9:16 with captions, overlay, two-pass loudness and clean metadata", async () => {
|
||||||
|
const workDir = path.join(dir, "render-ok");
|
||||||
|
const plan = compile(workDir);
|
||||||
|
const progress: MediaRenderProgress[] = [];
|
||||||
|
await renderer().render(plan, { onProgress: (p) => progress.push(p) });
|
||||||
|
|
||||||
|
const video = plan.outputs.find((output) => output.kind === "video")!.path;
|
||||||
|
const report = await renderer().probe(video, { measureLoudness: true });
|
||||||
|
expect(report.width).toBe(180);
|
||||||
|
expect(report.height).toBe(320);
|
||||||
|
expect(report.fps).toBe(30);
|
||||||
|
expect(report.videoCodec).toBe("h264");
|
||||||
|
expect(report.audioCodec).toBe("aac");
|
||||||
|
expect(Math.abs(report.durationMs - 4000)).toBeLessThanOrEqual(1000 / 30 + 1);
|
||||||
|
expect(report.integratedLufs).not.toBeNull();
|
||||||
|
expect(Math.abs(report.integratedLufs! - profile.audio.targetLufs)).toBeLessThanOrEqual(1);
|
||||||
|
expect(report.truePeakDb!).toBeLessThanOrEqual(profile.audio.truePeakDb + 0.5);
|
||||||
|
expect(findMetadataLeaks(report.tags)).toEqual([]);
|
||||||
|
expect(report.tags["format.title"]).toBe("Example teaser");
|
||||||
|
const keys = Object.keys(report.tags).join(" ").toLowerCase();
|
||||||
|
for (const leak of ["location", "creation_time", "make", "model"]) {
|
||||||
|
expect(keys).not.toContain(leak);
|
||||||
|
}
|
||||||
|
expect(Object.values(report.tags).join(" ")).not.toContain("ExampleCam");
|
||||||
|
|
||||||
|
const poster = plan.outputs.find((output) => output.kind === "poster");
|
||||||
|
if (poster) expect(findMetadataLeaks((await renderer().probe(poster.path)).tags)).toEqual([]);
|
||||||
|
expect(progress.length).toBeGreaterThan(0);
|
||||||
|
expect(progress.at(-1)!.progress).toBeCloseTo(1, 5);
|
||||||
|
for (let i = 1; i < progress.length; i++) {
|
||||||
|
expect(progress[i]!.progress).toBeGreaterThanOrEqual(progress[i - 1]!.progress - 1e-9);
|
||||||
|
}
|
||||||
|
// The measurement pass leaves nothing behind.
|
||||||
|
expect((await readdir(workDir)).sort()).toEqual(
|
||||||
|
["captions.ass", "captions.srt", "captions.vtt", "output.mp4", "poster.jpg"].filter(
|
||||||
|
(name) => name !== "poster.jpg" || poster,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts an EXIF-free PNG frame", async () => {
|
||||||
|
const out = path.join(dir, "frames", "frame.png");
|
||||||
|
await mkdir(path.dirname(out), { recursive: true });
|
||||||
|
await renderer().extractFrame({ path: source, tMs: 1000, outPath: out, width: 161 });
|
||||||
|
const report = await renderer().probe(out);
|
||||||
|
expect(report.width).toBe(160);
|
||||||
|
expect(report.videoCodec).toBe("png");
|
||||||
|
expect(findMetadataLeaks(report.tags)).toEqual([]);
|
||||||
|
await expect(
|
||||||
|
renderer().extractFrame({ path: source, tMs: 1000, outPath: path.join(dir, "frame.jpg") }),
|
||||||
|
).rejects.toThrow("PNG");
|
||||||
|
await expect(
|
||||||
|
renderer().extractFrame({ path: source, tMs: 60_000, outPath: path.join(dir, "late.png") }),
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses to probe outside the media root", async () => {
|
||||||
|
await expect(renderer().probe("/etc/hostname")).rejects.toThrow("outside");
|
||||||
|
await expect(renderer().probe("relative.mp4")).rejects.toThrow("absolute");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("aborts a running render, kills ffmpeg and removes partial outputs", async () => {
|
||||||
|
const workDir = path.join(dir, "render-abort");
|
||||||
|
const long = await mkdtemp(path.join(dir, "long-"));
|
||||||
|
const longSource = path.join(long, "long.mp4");
|
||||||
|
await writeSyntheticClip(longSource, { seconds: 30, width: 1280, height: 720 });
|
||||||
|
const plan = compile(workDir, {
|
||||||
|
edl: EdlSchema.parse({
|
||||||
|
edlVersion: 1,
|
||||||
|
output: {
|
||||||
|
profileId: profile.id,
|
||||||
|
width: 1080,
|
||||||
|
height: 1920,
|
||||||
|
fps: 30,
|
||||||
|
durationTargetMs: 30_000,
|
||||||
|
durationHardMinMs: 1000,
|
||||||
|
durationHardMaxMs: 120_000,
|
||||||
|
},
|
||||||
|
tracks: {
|
||||||
|
video: [
|
||||||
|
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 30_000, atMs: 0, reframe: "none" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
metadata: {},
|
||||||
|
}),
|
||||||
|
sources: new Map([
|
||||||
|
[
|
||||||
|
"asset-a",
|
||||||
|
{
|
||||||
|
path: longSource,
|
||||||
|
width: 1280,
|
||||||
|
height: 720,
|
||||||
|
durationMs: 30_000,
|
||||||
|
fps: 30,
|
||||||
|
hasAudio: true,
|
||||||
|
rotation: 0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
const controller = new AbortController();
|
||||||
|
const started = Date.now();
|
||||||
|
const rendering = renderer().render(plan, {
|
||||||
|
signal: controller.signal,
|
||||||
|
onProgress: (p) => {
|
||||||
|
if (p.progress > 0.01) controller.abort();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await expect(rendering).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
expect(Date.now() - started).toBeLessThan(25_000);
|
||||||
|
await expect(stat(path.join(workDir, "output.mp4"))).rejects.toThrow();
|
||||||
|
});
|
||||||
|
});
|
||||||
606
packages/adapters/src/media-ffmpeg-renderer.ts
Normal file
606
packages/adapters/src/media-ffmpeg-renderer.ts
Normal file
@ -0,0 +1,606 @@
|
|||||||
|
import { spawn } from "node:child_process";
|
||||||
|
import { access, mkdir, rm, stat, writeFile } from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { MediaRenderer, MediaRenderProgress } from "@rakazo/adapter-kit";
|
||||||
|
import type { FfprobeReport, RenderPlan, RenderStep } from "@rakazo/contracts";
|
||||||
|
import { getLogger } from "@rakazo/logging";
|
||||||
|
import { assertInside } from "@rakazo/media/node";
|
||||||
|
import { MediaToolMissingError, runMediaTool } from "./media-folder-source-probe.js";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Local ffmpeg executor for compiled render plans. Every process is spawned
|
||||||
|
* with an argv array and no shell. Plans come from compileRenderPlan, which
|
||||||
|
* already escapes every value; this module only adds runtime options
|
||||||
|
* (progress, hardware device) and the loudness measurement pass.
|
||||||
|
*
|
||||||
|
* Loudness: when the render step carries a single `loudnorm` with
|
||||||
|
* linear=true and no measured values, a first pass measures the program
|
||||||
|
* (print_format=json, null muxer) and the filter is rewritten with the
|
||||||
|
* measured_* options, so the second pass normalizes linearly to the target.
|
||||||
|
* When the measurement cannot be parsed the step runs single-pass (dynamic).
|
||||||
|
*/
|
||||||
|
|
||||||
|
const HARDWARE_CANDIDATES = [
|
||||||
|
"h264_vaapi",
|
||||||
|
"hevc_vaapi",
|
||||||
|
"h264_videotoolbox",
|
||||||
|
"hevc_videotoolbox",
|
||||||
|
"h264_nvenc",
|
||||||
|
"hevc_nvenc",
|
||||||
|
] as const;
|
||||||
|
const DEFAULT_VAAPI_DEVICE = "/dev/dri/renderD128";
|
||||||
|
const DEFAULT_KILL_GRACE_MS = 2000;
|
||||||
|
const STDERR_TAIL_BYTES = 64 * 1024;
|
||||||
|
|
||||||
|
export interface FfmpegMediaRendererOptions {
|
||||||
|
ffmpegPath?: string;
|
||||||
|
ffprobePath?: string;
|
||||||
|
/** When set, plan work dirs and probed or extracted paths must lie inside it. */
|
||||||
|
root?: string;
|
||||||
|
vaapiDevice?: string;
|
||||||
|
/** Time between SIGTERM and SIGKILL when a render is aborted. */
|
||||||
|
killGraceMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortError(signal: AbortSignal | undefined): Error {
|
||||||
|
const reason = signal?.reason;
|
||||||
|
if (reason instanceof Error && reason.name === "AbortError") return reason;
|
||||||
|
const error = new Error("render aborted");
|
||||||
|
error.name = "AbortError";
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "30000/1001" → 29.97; "0/0" → 0. */
|
||||||
|
export function parseFrameRate(value: string | undefined): number {
|
||||||
|
if (!value) return 0;
|
||||||
|
const [num, den] = value.split("/").map(Number);
|
||||||
|
if (num === undefined || !Number.isFinite(num)) return 0;
|
||||||
|
const rate = den === undefined ? num : den > 0 ? num / den : 0;
|
||||||
|
return Number.isFinite(rate) ? Math.round(rate * 1000) / 1000 : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProbeJson {
|
||||||
|
streams?: Array<{
|
||||||
|
codec_type?: string;
|
||||||
|
codec_name?: string;
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
r_frame_rate?: string;
|
||||||
|
tags?: Record<string, unknown>;
|
||||||
|
}>;
|
||||||
|
format?: { duration?: string; size?: string; tags?: Record<string, unknown> };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ffprobe JSON → report without loudness. Tags are flattened as `format.x` / `stream.N.x`. */
|
||||||
|
export function parseProbeReport(json: ProbeJson, fileSize: number): FfprobeReport {
|
||||||
|
const streams = json.streams ?? [];
|
||||||
|
const video = streams.find((stream) => stream.codec_type === "video");
|
||||||
|
const audio = streams.find((stream) => stream.codec_type === "audio");
|
||||||
|
const tags: Record<string, string> = {};
|
||||||
|
for (const [key, value] of Object.entries(json.format?.tags ?? {})) {
|
||||||
|
tags[`format.${key}`] = String(value);
|
||||||
|
}
|
||||||
|
streams.forEach((stream, index) => {
|
||||||
|
for (const [key, value] of Object.entries(stream.tags ?? {})) {
|
||||||
|
tags[`stream.${index}.${key}`] = String(value);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const seconds = Number(json.format?.duration);
|
||||||
|
const size = Number(json.format?.size);
|
||||||
|
return {
|
||||||
|
durationMs: Number.isFinite(seconds) && seconds > 0 ? Math.round(seconds * 1000) : 0,
|
||||||
|
width: video?.width ?? 0,
|
||||||
|
height: video?.height ?? 0,
|
||||||
|
fps: parseFrameRate(video?.r_frame_rate),
|
||||||
|
videoCodec: video?.codec_name ?? "none",
|
||||||
|
audioCodec: audio?.codec_name ?? null,
|
||||||
|
sizeBytes: Number.isFinite(size) && size > 0 ? Math.round(size) : fileSize,
|
||||||
|
integratedLufs: null,
|
||||||
|
truePeakDb: null,
|
||||||
|
tags,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Integrated loudness and true peak from the ebur128 summary on stderr. */
|
||||||
|
export function parseEbur128Summary(stderr: string): {
|
||||||
|
integratedLufs: number | null;
|
||||||
|
truePeakDb: number | null;
|
||||||
|
} {
|
||||||
|
const at = stderr.lastIndexOf("Summary:");
|
||||||
|
if (at < 0) return { integratedLufs: null, truePeakDb: null };
|
||||||
|
const summary = stderr.slice(at);
|
||||||
|
const integrated = /I:\s+(-?\d+(?:\.\d+)?) LUFS/.exec(summary);
|
||||||
|
const peak = /True peak:\s+Peak:\s+(-?\d+(?:\.\d+)?|-inf) dBFS/.exec(summary);
|
||||||
|
const lufs = integrated ? Number(integrated[1]) : null;
|
||||||
|
const tp = peak && peak[1] !== "-inf" ? Number(peak[1]) : null;
|
||||||
|
return {
|
||||||
|
// ebur128 reports -70 LUFS (its gate) for silence; that is "no programme".
|
||||||
|
integratedLufs: lufs !== null && Number.isFinite(lufs) && lufs > -70 ? lufs : null,
|
||||||
|
truePeakDb: tp !== null && Number.isFinite(tp) ? tp : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Two-pass loudnorm
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const LOUDNORM = /loudnorm=((?:[^,;[\]\\]|\\.)*)/g;
|
||||||
|
|
||||||
|
interface LoudnormSite {
|
||||||
|
/** Index of the options string inside the graph. */
|
||||||
|
start: number;
|
||||||
|
end: number;
|
||||||
|
options: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The single linear loudnorm without measured values, or null. */
|
||||||
|
export function findTwoPassLoudnorm(graph: string): LoudnormSite | null {
|
||||||
|
const matches = [...graph.matchAll(LOUDNORM)];
|
||||||
|
if (matches.length !== 1) return null;
|
||||||
|
const match = matches[0]!;
|
||||||
|
const options = match[1] ?? "";
|
||||||
|
const parts = options.split(":");
|
||||||
|
if (!parts.includes("linear=true")) return null;
|
||||||
|
if (parts.some((part) => part.startsWith("measured_"))) return null;
|
||||||
|
const start = (match.index ?? 0) + "loudnorm=".length;
|
||||||
|
return { start, end: start + options.length, options };
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LoudnormMeasurement {
|
||||||
|
inputI: number;
|
||||||
|
inputTp: number;
|
||||||
|
inputLra: number;
|
||||||
|
inputThresh: number;
|
||||||
|
targetOffset: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The JSON block loudnorm prints after its summary line; null when absent or not finite. */
|
||||||
|
export function parseLoudnormJson(stderr: string): LoudnormMeasurement | null {
|
||||||
|
const at = stderr.lastIndexOf("Parsed_loudnorm");
|
||||||
|
if (at < 0) return null;
|
||||||
|
const open = stderr.indexOf("{", at);
|
||||||
|
const close = stderr.indexOf("}", open);
|
||||||
|
if (open < 0 || close < 0) return null;
|
||||||
|
let parsed: Record<string, unknown>;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(stderr.slice(open, close + 1)) as Record<string, unknown>;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const read = (key: string) => {
|
||||||
|
const value = Number(parsed[key]);
|
||||||
|
return Number.isFinite(value) ? value : null;
|
||||||
|
};
|
||||||
|
const inputI = read("input_i");
|
||||||
|
const inputTp = read("input_tp");
|
||||||
|
const inputLra = read("input_lra");
|
||||||
|
const inputThresh = read("input_thresh");
|
||||||
|
const targetOffset = read("target_offset");
|
||||||
|
if (
|
||||||
|
inputI === null ||
|
||||||
|
inputTp === null ||
|
||||||
|
inputLra === null ||
|
||||||
|
inputThresh === null ||
|
||||||
|
targetOffset === null
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return { inputI, inputTp, inputLra, inputThresh, targetOffset };
|
||||||
|
}
|
||||||
|
|
||||||
|
function withLoudnormOptions(graph: string, site: LoudnormSite, extra: string): string {
|
||||||
|
return `${graph.slice(0, site.start)}${site.options}:${extra}${graph.slice(site.end)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function measuredOptions(m: LoudnormMeasurement): string {
|
||||||
|
const f = (value: number) => value.toFixed(2);
|
||||||
|
return [
|
||||||
|
`measured_I=${f(m.inputI)}`,
|
||||||
|
`measured_TP=${f(m.inputTp)}`,
|
||||||
|
`measured_LRA=${f(m.inputLra)}`,
|
||||||
|
`measured_thresh=${f(m.inputThresh)}`,
|
||||||
|
`offset=${f(m.targetOffset)}`,
|
||||||
|
].join(":");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Renderer
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
interface RunOptions {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
/** Called with the output time reached, in ms. */
|
||||||
|
onTime?: (ms: number) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class FfmpegMediaRenderer implements MediaRenderer {
|
||||||
|
readonly id = "ffmpeg";
|
||||||
|
private readonly ffmpegPath: string;
|
||||||
|
private readonly ffprobePath: string;
|
||||||
|
private readonly vaapiDevice: string;
|
||||||
|
private readonly killGraceMs: number;
|
||||||
|
private encoders: Promise<string[]> | null = null;
|
||||||
|
|
||||||
|
constructor(private readonly options: FfmpegMediaRendererOptions = {}) {
|
||||||
|
this.ffmpegPath = options.ffmpegPath ?? "ffmpeg";
|
||||||
|
this.ffprobePath = options.ffprobePath ?? "ffprobe";
|
||||||
|
this.vaapiDevice = options.vaapiDevice ?? DEFAULT_VAAPI_DEVICE;
|
||||||
|
this.killGraceMs = options.killGraceMs ?? DEFAULT_KILL_GRACE_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
hardwareEncoders(): Promise<string[]> {
|
||||||
|
this.encoders ??= this.detectEncoders().catch((error) => {
|
||||||
|
getLogger().warn("media hardware encoder detection failed", {
|
||||||
|
reason: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
return this.encoders;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async detectEncoders(): Promise<string[]> {
|
||||||
|
const { stdout } = await runMediaTool(this.ffmpegPath, ["-hide_banner", "-encoders"]);
|
||||||
|
const listed = new Set(
|
||||||
|
stdout
|
||||||
|
.toString("utf8")
|
||||||
|
.split("\n")
|
||||||
|
.map((line) => line.trim().split(/\s+/)[1])
|
||||||
|
.filter((name): name is string => Boolean(name)),
|
||||||
|
);
|
||||||
|
const hasVaapiDevice = await access(this.vaapiDevice).then(
|
||||||
|
() => true,
|
||||||
|
() => false,
|
||||||
|
);
|
||||||
|
const usable: string[] = [];
|
||||||
|
for (const encoder of HARDWARE_CANDIDATES) {
|
||||||
|
if (!listed.has(encoder)) continue;
|
||||||
|
const vaapi = encoder.endsWith("_vaapi");
|
||||||
|
if (vaapi && !hasVaapiDevice) continue;
|
||||||
|
// One-frame test encode: listed encoders often lack the device or driver.
|
||||||
|
const args = ["-hide_banner", "-v", "error", "-nostdin"];
|
||||||
|
if (vaapi) args.push("-vaapi_device", this.vaapiDevice);
|
||||||
|
args.push("-f", "lavfi", "-i", "color=c=black:s=256x256:r=30:d=0.1", "-frames:v", "1");
|
||||||
|
if (vaapi) args.push("-vf", "format=nv12,hwupload");
|
||||||
|
args.push("-c:v", encoder, "-f", "null", "-");
|
||||||
|
try {
|
||||||
|
await runMediaTool(this.ffmpegPath, args);
|
||||||
|
usable.push(encoder);
|
||||||
|
} catch {
|
||||||
|
// Not usable on this host.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return usable;
|
||||||
|
}
|
||||||
|
|
||||||
|
async render(
|
||||||
|
plan: RenderPlan,
|
||||||
|
options: { signal?: AbortSignal; onProgress?: (progress: MediaRenderProgress) => void } = {},
|
||||||
|
): Promise<void> {
|
||||||
|
const { signal, onProgress } = options;
|
||||||
|
if (signal?.aborted) throw abortError(signal);
|
||||||
|
const workDir = await this.workDir(plan.workDir);
|
||||||
|
const weight = (step: RenderStep) =>
|
||||||
|
step.kind === "ffmpeg" ? Math.max(step.durationMs, 1) : 0;
|
||||||
|
const total = plan.steps.reduce((sum, step) => sum + weight(step), 0) || 1;
|
||||||
|
let done = 0;
|
||||||
|
const report = (stepId: string, fraction: number, stepWeight: number) => {
|
||||||
|
const value = (done + Math.min(1, Math.max(0, fraction)) * stepWeight) / total;
|
||||||
|
onProgress?.({ stepId, progress: Math.min(1, Math.max(0, value)) });
|
||||||
|
};
|
||||||
|
const written: string[] = [];
|
||||||
|
try {
|
||||||
|
for (const step of plan.steps) {
|
||||||
|
if (signal?.aborted) throw abortError(signal);
|
||||||
|
switch (step.kind) {
|
||||||
|
case "write_file": {
|
||||||
|
const file = await this.insideWorkDir(workDir, step.path);
|
||||||
|
await mkdir(path.dirname(file), { recursive: true });
|
||||||
|
await writeFile(file, step.content, "utf8");
|
||||||
|
written.push(file);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "motion":
|
||||||
|
case "inpaint":
|
||||||
|
case "synthetic_identity": {
|
||||||
|
const ready = await access(step.outPath).then(
|
||||||
|
() => true,
|
||||||
|
() => false,
|
||||||
|
);
|
||||||
|
if (!ready) throw new Error(`${step.kind} step ${step.id} was not prepared`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "ffmpeg": {
|
||||||
|
for (const produced of step.produces) await this.insideWorkDir(workDir, produced);
|
||||||
|
written.push(...step.produces);
|
||||||
|
await this.runStep(step, plan, workDir, {
|
||||||
|
signal,
|
||||||
|
report: (fraction) => report(step.id, fraction, weight(step)),
|
||||||
|
});
|
||||||
|
// Never report success for an output ffmpeg did not actually write.
|
||||||
|
for (const produced of step.produces) {
|
||||||
|
const file = await stat(produced).catch(() => null);
|
||||||
|
if (!file?.isFile() || file.size === 0) {
|
||||||
|
throw new Error(`step ${step.id} wrote no ${path.basename(produced)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
done += weight(step);
|
||||||
|
report(step.id, 0, 0);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
await Promise.all(written.map((file) => rm(file, { force: true }).catch(() => undefined)));
|
||||||
|
if (signal?.aborted) throw abortError(signal);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runStep(
|
||||||
|
step: Extract<RenderStep, { kind: "ffmpeg" }>,
|
||||||
|
plan: RenderPlan,
|
||||||
|
workDir: string,
|
||||||
|
options: { signal?: AbortSignal; report: (fraction: number) => void },
|
||||||
|
): Promise<void> {
|
||||||
|
let argv = [...step.argv];
|
||||||
|
const vaapi = argv.some((arg, i) => argv[i - 1] === "-c:v" && arg.endsWith("_vaapi"));
|
||||||
|
const prefix = vaapi ? ["-vaapi_device", this.vaapiDevice] : [];
|
||||||
|
const durationMs = Math.max(step.durationMs, 1);
|
||||||
|
const twoPass = await this.loudnormPlan(argv, plan, workDir);
|
||||||
|
if (twoPass) {
|
||||||
|
let measurement: LoudnormMeasurement | null = null;
|
||||||
|
try {
|
||||||
|
const { stderr } = await this.run([...prefix, ...twoPass.measureArgv], {
|
||||||
|
signal: options.signal,
|
||||||
|
onTime: (ms) => options.report((ms / durationMs) * 0.5),
|
||||||
|
});
|
||||||
|
measurement = parseLoudnormJson(stderr);
|
||||||
|
} finally {
|
||||||
|
if (twoPass.measureScript) await rm(twoPass.measureScript, { force: true });
|
||||||
|
}
|
||||||
|
if (measurement) {
|
||||||
|
argv = await twoPass.apply(measuredOptions(measurement));
|
||||||
|
} else {
|
||||||
|
getLogger().warn("loudnorm measurement unreadable; rendering single pass", {
|
||||||
|
stepId: step.id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const offset = twoPass ? 0.5 : 0;
|
||||||
|
const scale = twoPass ? 0.5 : 1;
|
||||||
|
await this.run([...prefix, ...argv], {
|
||||||
|
signal: options.signal,
|
||||||
|
onTime: (ms) => options.report(offset + (ms / durationMs) * scale),
|
||||||
|
});
|
||||||
|
options.report(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Measurement argv and a function that writes the measured filter into the
|
||||||
|
* render argv (or its filtergraph script), or null when the step needs no
|
||||||
|
* measurement pass.
|
||||||
|
*/
|
||||||
|
private async loudnormPlan(
|
||||||
|
argv: string[],
|
||||||
|
plan: RenderPlan,
|
||||||
|
workDir: string,
|
||||||
|
): Promise<{
|
||||||
|
measureArgv: string[];
|
||||||
|
measureScript: string | null;
|
||||||
|
apply: (extra: string) => Promise<string[]>;
|
||||||
|
} | null> {
|
||||||
|
const inline = argv.indexOf("-filter_complex");
|
||||||
|
const scriptAt = argv.indexOf("-filter_complex_script");
|
||||||
|
const flagAt = inline >= 0 ? inline : scriptAt;
|
||||||
|
if (flagAt < 0 || flagAt + 1 >= argv.length) return null;
|
||||||
|
let graph: string;
|
||||||
|
let scriptPath: string | null = null;
|
||||||
|
if (inline >= 0) {
|
||||||
|
graph = argv[inline + 1]!;
|
||||||
|
} else {
|
||||||
|
scriptPath = await this.insideWorkDir(workDir, argv[scriptAt + 1]!);
|
||||||
|
let content: string | null = null;
|
||||||
|
for (const step of plan.steps) {
|
||||||
|
if (step.kind !== "write_file") continue;
|
||||||
|
if ((await this.insideWorkDir(workDir, step.path)) === scriptPath) content = step.content;
|
||||||
|
}
|
||||||
|
if (content === null) return null;
|
||||||
|
graph = content;
|
||||||
|
}
|
||||||
|
const site = findTwoPassLoudnorm(graph);
|
||||||
|
if (!site) return null;
|
||||||
|
|
||||||
|
const tail = argv.slice(flagAt + 2);
|
||||||
|
const maps: string[] = [];
|
||||||
|
let duration: string | null = null;
|
||||||
|
for (let i = 0; i < tail.length - 1; i++) {
|
||||||
|
if (tail[i] === "-map") maps.push("-map", tail[i + 1]!);
|
||||||
|
if (tail[i] === "-t") duration = tail[i + 1]!;
|
||||||
|
}
|
||||||
|
const measureGraph = withLoudnormOptions(graph, site, "print_format=json");
|
||||||
|
let measureScript: string | null = null;
|
||||||
|
let graphArg = measureGraph;
|
||||||
|
if (scriptPath) {
|
||||||
|
measureScript = path.join(path.dirname(scriptPath), "filtergraph.measure.txt");
|
||||||
|
await writeFile(measureScript, measureGraph, "utf8");
|
||||||
|
graphArg = measureScript;
|
||||||
|
}
|
||||||
|
const measureArgv = [
|
||||||
|
...argv.slice(0, flagAt + 1),
|
||||||
|
graphArg,
|
||||||
|
...maps,
|
||||||
|
...(duration ? ["-t", duration] : []),
|
||||||
|
"-f",
|
||||||
|
"null",
|
||||||
|
"-",
|
||||||
|
];
|
||||||
|
return {
|
||||||
|
measureArgv,
|
||||||
|
measureScript,
|
||||||
|
apply: async (extra) => {
|
||||||
|
const next = withLoudnormOptions(graph, site, extra);
|
||||||
|
const out = [...argv];
|
||||||
|
if (scriptPath) await writeFile(scriptPath, next, "utf8");
|
||||||
|
else out[flagAt + 1] = next;
|
||||||
|
return out;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Spawn ffmpeg with progress on stdout. Rejects with the stderr tail on failure. */
|
||||||
|
private run(args: string[], options: RunOptions): Promise<{ stderr: string }> {
|
||||||
|
const { signal } = options;
|
||||||
|
if (signal?.aborted) return Promise.reject(abortError(signal));
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const child = spawn(this.ffmpegPath, ["-progress", "pipe:1", "-nostats", ...args], {
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
shell: false,
|
||||||
|
windowsHide: true,
|
||||||
|
});
|
||||||
|
let stderr = "";
|
||||||
|
let pending = "";
|
||||||
|
let killTimer: NodeJS.Timeout | null = null;
|
||||||
|
const onAbort = () => {
|
||||||
|
child.kill("SIGTERM");
|
||||||
|
killTimer = setTimeout(() => child.kill("SIGKILL"), this.killGraceMs);
|
||||||
|
killTimer.unref();
|
||||||
|
};
|
||||||
|
signal?.addEventListener("abort", onAbort, { once: true });
|
||||||
|
child.stdout.setEncoding("utf8");
|
||||||
|
child.stdout.on("data", (chunk: string) => {
|
||||||
|
pending += chunk;
|
||||||
|
const lines = pending.split("\n");
|
||||||
|
pending = lines.pop() ?? "";
|
||||||
|
for (const line of lines) {
|
||||||
|
const match = /^out_time_(us|ms)=(\d+)/.exec(line.trim());
|
||||||
|
// Both keys are microseconds in ffmpeg 5+ (out_time_ms is misnamed).
|
||||||
|
if (match) options.onTime?.(Number(match[2]) / 1000);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
child.stderr.setEncoding("utf8");
|
||||||
|
child.stderr.on("data", (chunk: string) => {
|
||||||
|
stderr += chunk;
|
||||||
|
if (stderr.length > STDERR_TAIL_BYTES * 2) stderr = stderr.slice(-STDERR_TAIL_BYTES);
|
||||||
|
});
|
||||||
|
child.on("error", (error: NodeJS.ErrnoException) => {
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
reject(error.code === "ENOENT" ? new MediaToolMissingError(this.ffmpegPath) : error);
|
||||||
|
});
|
||||||
|
child.on("close", (code) => {
|
||||||
|
signal?.removeEventListener("abort", onAbort);
|
||||||
|
if (killTimer) clearTimeout(killTimer);
|
||||||
|
if (signal?.aborted) {
|
||||||
|
reject(abortError(signal));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (code !== 0) {
|
||||||
|
const tail = stderr.trim().split("\n").slice(-4).join(" ");
|
||||||
|
reject(new Error(`ffmpeg failed (${code}): ${tail}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
resolve({ stderr });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async probe(
|
||||||
|
file: string,
|
||||||
|
options: { measureLoudness?: boolean; signal?: AbortSignal } = {},
|
||||||
|
): Promise<FfprobeReport> {
|
||||||
|
const target = await this.checked(file);
|
||||||
|
const { stdout } = await runMediaTool(
|
||||||
|
this.ffprobePath,
|
||||||
|
["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "--", target],
|
||||||
|
{ signal: options.signal },
|
||||||
|
);
|
||||||
|
const report = parseProbeReport(
|
||||||
|
JSON.parse(stdout.toString("utf8")) as ProbeJson,
|
||||||
|
(await stat(target)).size,
|
||||||
|
);
|
||||||
|
if (!options.measureLoudness || report.audioCodec === null) return report;
|
||||||
|
const { stderr } = await runMediaTool(
|
||||||
|
this.ffmpegPath,
|
||||||
|
[
|
||||||
|
"-hide_banner",
|
||||||
|
"-nostats",
|
||||||
|
"-nostdin",
|
||||||
|
"-i",
|
||||||
|
target,
|
||||||
|
"-map",
|
||||||
|
"0:a:0",
|
||||||
|
"-af",
|
||||||
|
"ebur128=peak=true",
|
||||||
|
"-f",
|
||||||
|
"null",
|
||||||
|
"-",
|
||||||
|
],
|
||||||
|
{ signal: options.signal },
|
||||||
|
);
|
||||||
|
return { ...report, ...parseEbur128Summary(stderr) };
|
||||||
|
}
|
||||||
|
|
||||||
|
async extractFrame(input: {
|
||||||
|
path: string;
|
||||||
|
tMs: number;
|
||||||
|
outPath: string;
|
||||||
|
width?: number;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<void> {
|
||||||
|
const source = await this.checked(input.path);
|
||||||
|
if (path.extname(input.outPath).toLowerCase() !== ".png") {
|
||||||
|
throw new Error("frames are extracted as PNG");
|
||||||
|
}
|
||||||
|
const outPath = await this.checked(input.outPath);
|
||||||
|
if (!Number.isFinite(input.tMs) || input.tMs < 0) throw new Error("invalid frame time");
|
||||||
|
const args = [
|
||||||
|
"-hide_banner",
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-y",
|
||||||
|
"-ss",
|
||||||
|
(input.tMs / 1000).toFixed(3),
|
||||||
|
"-i",
|
||||||
|
source,
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
];
|
||||||
|
if (input.width !== undefined) {
|
||||||
|
const width = Math.max(2, Math.floor(input.width / 2) * 2);
|
||||||
|
args.push("-vf", `scale=${width}:-2`);
|
||||||
|
}
|
||||||
|
args.push(
|
||||||
|
"-map_metadata",
|
||||||
|
"-1",
|
||||||
|
"-fflags",
|
||||||
|
"+bitexact",
|
||||||
|
"-flags:v",
|
||||||
|
"+bitexact",
|
||||||
|
"-f",
|
||||||
|
"image2",
|
||||||
|
"-c:v",
|
||||||
|
"png",
|
||||||
|
outPath,
|
||||||
|
);
|
||||||
|
await runMediaTool(this.ffmpegPath, args, { signal: input.signal });
|
||||||
|
const written = await stat(outPath).catch(() => null);
|
||||||
|
if (!written || written.size === 0) throw new Error("no frame at the requested time");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async checked(file: string): Promise<string> {
|
||||||
|
if (!path.isAbsolute(file)) throw new Error("media paths must be absolute");
|
||||||
|
return this.options.root ? assertInside(this.options.root, file) : path.resolve(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async workDir(dir: string): Promise<string> {
|
||||||
|
const resolved = await this.checked(dir);
|
||||||
|
await mkdir(resolved, { recursive: true });
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Steps may only write inside the plan's work directory. */
|
||||||
|
private async insideWorkDir(workDir: string, file: string): Promise<string> {
|
||||||
|
if (!path.isAbsolute(file)) throw new Error("plan paths must be absolute");
|
||||||
|
return assertInside(workDir, file);
|
||||||
|
}
|
||||||
|
}
|
||||||
752
packages/adapters/src/media-render.fixtures.ts
Normal file
752
packages/adapters/src/media-render.fixtures.ts
Normal file
@ -0,0 +1,752 @@
|
|||||||
|
import { execFile, execFileSync } from "node:child_process";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { mkdir, mkdtemp, readFile, realpath, rm } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { promisify } from "node:util";
|
||||||
|
import type {
|
||||||
|
JobPublisher,
|
||||||
|
LinkShortener,
|
||||||
|
MediaInferenceProvider,
|
||||||
|
MediaPublisher,
|
||||||
|
MediaRenderer,
|
||||||
|
MotionGraphicsRenderer,
|
||||||
|
} from "@rakazo/adapter-kit";
|
||||||
|
import type {
|
||||||
|
BrandKit,
|
||||||
|
Edl,
|
||||||
|
FfprobeReport,
|
||||||
|
GateIssue,
|
||||||
|
GateReport,
|
||||||
|
MediaAnalysisKind,
|
||||||
|
MediaEditPlan,
|
||||||
|
MediaRender,
|
||||||
|
PlatformProfile,
|
||||||
|
} from "@rakazo/contracts";
|
||||||
|
import { BrandKitSchema, EdlSchema, PlatformProfileSchema } from "@rakazo/contracts";
|
||||||
|
import type { MediaRenderPatch, MediaRepos, PrismaClient } from "@rakazo/db";
|
||||||
|
import {
|
||||||
|
ACTIVE_RENDER_STATUSES,
|
||||||
|
bootstrapUserSpace,
|
||||||
|
createDb,
|
||||||
|
createMediaRepos,
|
||||||
|
IsolationError,
|
||||||
|
MediaStateConflictError,
|
||||||
|
mapMediaEditPlan,
|
||||||
|
mapMediaRender,
|
||||||
|
} from "@rakazo/db";
|
||||||
|
import type { CompileSource } from "@rakazo/media";
|
||||||
|
import { evaluateGate } from "@rakazo/media";
|
||||||
|
import { LocalArtifactStore } from "./artifacts.js";
|
||||||
|
import type {
|
||||||
|
MediaPreparedSources,
|
||||||
|
MediaPrivacyService,
|
||||||
|
MediaRealtimeEvent,
|
||||||
|
MediaServiceDeps,
|
||||||
|
} from "./media-context.js";
|
||||||
|
|
||||||
|
const run = promisify(execFile);
|
||||||
|
|
||||||
|
function onPath(binary: string): boolean {
|
||||||
|
try {
|
||||||
|
execFileSync(binary, ["-version"], { stdio: "ignore" });
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Real-ffmpeg tests run when both tools are installed and skip otherwise. */
|
||||||
|
export const hasMediaTools = onPath("ffmpeg") && onPath("ffprobe");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Synthetic clip: testsrc2 pattern plus a sine tone, H.264/AAC in MP4, with the
|
||||||
|
* kind of tags a phone writes (GPS, device, capture time) that every export
|
||||||
|
* must drop. No real people or places.
|
||||||
|
*/
|
||||||
|
export async function writeSyntheticClip(
|
||||||
|
file: string,
|
||||||
|
options: { seconds: number; width: number; height: number; fps?: number; volumeDb?: number },
|
||||||
|
): Promise<void> {
|
||||||
|
const fps = options.fps ?? 30;
|
||||||
|
await mkdir(path.dirname(file), { recursive: true });
|
||||||
|
await run("ffmpeg", [
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-y",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
`testsrc2=size=${options.width}x${options.height}:rate=${fps}:duration=${options.seconds}`,
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
`sine=frequency=440:sample_rate=48000:duration=${options.seconds}`,
|
||||||
|
"-af",
|
||||||
|
`volume=${options.volumeDb ?? -20}dB`,
|
||||||
|
"-c:v",
|
||||||
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"ultrafast",
|
||||||
|
"-pix_fmt",
|
||||||
|
"yuv420p",
|
||||||
|
"-c:a",
|
||||||
|
"aac",
|
||||||
|
"-shortest",
|
||||||
|
"-metadata",
|
||||||
|
"creation_time=2020-01-01T10:00:00Z",
|
||||||
|
"-metadata",
|
||||||
|
"location=+00.0000+000.0000/",
|
||||||
|
"-metadata",
|
||||||
|
"make=ExampleCam",
|
||||||
|
"-metadata",
|
||||||
|
"model=ExampleCam One",
|
||||||
|
"-metadata:s:v",
|
||||||
|
"handler_name=ExampleCam serial 0000",
|
||||||
|
file,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Test harness for the render service and job: the repos the render slice
|
||||||
|
* reads, either in memory or backed by PostgreSQL (VERIFY_DATABASE), plus a
|
||||||
|
* pass-through privacy service. Everything is synthetic.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export function renderTestProfile(overrides: Record<string, unknown> = {}): PlatformProfile {
|
||||||
|
return PlatformProfileSchema.parse({
|
||||||
|
id: "example-vertical-test",
|
||||||
|
name: "Example vertical test",
|
||||||
|
kind: "own_site",
|
||||||
|
reviewedAt: "2026-09-01",
|
||||||
|
video: {
|
||||||
|
aspectRatios: ["9:16"],
|
||||||
|
width: 1080,
|
||||||
|
height: 1920,
|
||||||
|
fps: 30,
|
||||||
|
codec: "h264",
|
||||||
|
maxBitrateKbps: 8000,
|
||||||
|
maxFileSizeMb: 500,
|
||||||
|
},
|
||||||
|
duration: { hardMinMs: 1000, hardMaxMs: 120_000, hookWindowMs: 2000 },
|
||||||
|
safeZones: { topPct: 8, bottomPct: 15, leftPct: 4, rightPct: 10 },
|
||||||
|
captions: { recommended: true, maxLines: 2 },
|
||||||
|
audio: { targetLufs: -14, truePeakDb: -1 },
|
||||||
|
text: { captionMaxChars: 500, hashtagLimit: 5, linkPolicy: "allowed" },
|
||||||
|
content: { thresholds: { violence: 0.5 } },
|
||||||
|
disclosure: { aiAlteredLabelRequired: true },
|
||||||
|
consent: { requirePerformerConsent: false, requireAgeVerification: false },
|
||||||
|
export: {},
|
||||||
|
...overrides,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function renderTestKit(input: {
|
||||||
|
logoArtifactId: string;
|
||||||
|
musicArtifactId: string;
|
||||||
|
}): BrandKit {
|
||||||
|
return BrandKitSchema.parse({
|
||||||
|
name: "Studio Example",
|
||||||
|
colors: {
|
||||||
|
primary: "#224488",
|
||||||
|
secondary: "#446688",
|
||||||
|
accent: "#FFAA00",
|
||||||
|
text: "#FFFFFF",
|
||||||
|
background: "#101010",
|
||||||
|
},
|
||||||
|
logoArtifactIds: [input.logoArtifactId],
|
||||||
|
watermark: {
|
||||||
|
artifactId: input.logoArtifactId,
|
||||||
|
position: "top_right",
|
||||||
|
opacity: 0.8,
|
||||||
|
scale: 0.2,
|
||||||
|
},
|
||||||
|
captionPresets: [
|
||||||
|
{
|
||||||
|
id: "clean",
|
||||||
|
name: "Clean",
|
||||||
|
template: "clean",
|
||||||
|
fontFamily: "DejaVu Sans",
|
||||||
|
sizePct: 5,
|
||||||
|
textColor: "#FFFFFF",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
musicBeds: [
|
||||||
|
{ artifactId: input.musicArtifactId, title: "Bed", license: "own", commercialUse: true },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HarnessAsset {
|
||||||
|
assetId: string;
|
||||||
|
source: CompileSource;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RenderHarness {
|
||||||
|
backend: "memory" | "postgres";
|
||||||
|
mediaRoot: string;
|
||||||
|
spaceId: string;
|
||||||
|
userId: string;
|
||||||
|
deps: MediaServiceDeps;
|
||||||
|
events: MediaRealtimeEvent[];
|
||||||
|
enqueued: string[];
|
||||||
|
/** Register an asset; the caller writes the file at `assetPath(assetId, fileName)`. */
|
||||||
|
createAsset(name: string): Promise<string>;
|
||||||
|
assetPath(assetId: string, fileName: string): string;
|
||||||
|
setSource(assetId: string, source: CompileSource): void;
|
||||||
|
sources: Map<string, CompileSource>;
|
||||||
|
addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise<void>;
|
||||||
|
addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise<void>;
|
||||||
|
addArtifact(file: string, name: string, mimeType: string): Promise<string>;
|
||||||
|
createProject(input: {
|
||||||
|
assetIds: string[];
|
||||||
|
kit: BrandKit | null;
|
||||||
|
profileId: string;
|
||||||
|
}): Promise<string>;
|
||||||
|
createPlan(projectId: string, edl: unknown): Promise<MediaEditPlan>;
|
||||||
|
planVersions(projectId: string, profileId: string): Promise<MediaEditPlan[]>;
|
||||||
|
artifactCount(): Promise<number>;
|
||||||
|
cleanup(): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HarnessOptions {
|
||||||
|
profiles: PlatformProfile[];
|
||||||
|
renderer: MediaRenderer;
|
||||||
|
motion: MotionGraphicsRenderer;
|
||||||
|
inference?: MediaInferenceProvider | null;
|
||||||
|
now?: () => Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
const unusedPublisher: MediaPublisher = {
|
||||||
|
id: "unused",
|
||||||
|
publish: async () => {
|
||||||
|
throw new Error("not used by render tests");
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
/** PostgreSQL when VERIFY_DATABASE and DATABASE_URL are set, otherwise in memory. */
|
||||||
|
export async function createRenderHarness(options: HarnessOptions): Promise<RenderHarness> {
|
||||||
|
const dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-render-")));
|
||||||
|
const mediaRoot = path.join(dir, "media");
|
||||||
|
await mkdir(mediaRoot, { recursive: true });
|
||||||
|
const artifacts = new LocalArtifactStore(path.join(dir, "store"));
|
||||||
|
const events: MediaRealtimeEvent[] = [];
|
||||||
|
const enqueued: string[] = [];
|
||||||
|
const jobs: JobPublisher = {
|
||||||
|
enqueue: async (job) => {
|
||||||
|
enqueued.push(`${job.name}:${JSON.stringify(job.payload)}`);
|
||||||
|
},
|
||||||
|
cancel: async () => undefined,
|
||||||
|
close: async () => undefined,
|
||||||
|
};
|
||||||
|
const databaseUrl = process.env.DATABASE_URL;
|
||||||
|
const backend =
|
||||||
|
process.env.VERIFY_DATABASE && databaseUrl
|
||||||
|
? await postgresBackend(databaseUrl)
|
||||||
|
: memoryBackend(options.now ?? (() => new Date()));
|
||||||
|
const sources = new Map<string, CompileSource>();
|
||||||
|
const deps: MediaServiceDeps = {
|
||||||
|
prisma: backend.prisma,
|
||||||
|
repos: backend.repos,
|
||||||
|
jobs,
|
||||||
|
inference: options.inference ?? null,
|
||||||
|
renderer: options.renderer,
|
||||||
|
motion: options.motion,
|
||||||
|
publisher: unusedPublisher,
|
||||||
|
shortener: null as LinkShortener | null,
|
||||||
|
sources: {},
|
||||||
|
mediaRoot,
|
||||||
|
profiles: async () => ({
|
||||||
|
profiles: new Map(
|
||||||
|
options.profiles.map((profile) => [profile.id, { profile, origin: "builtin" as const }]),
|
||||||
|
),
|
||||||
|
errors: [],
|
||||||
|
}),
|
||||||
|
artifacts,
|
||||||
|
model: { generate: async () => null },
|
||||||
|
publish: async (_spaceId, event) => {
|
||||||
|
events.push(event);
|
||||||
|
},
|
||||||
|
now: options.now ?? (() => new Date()),
|
||||||
|
};
|
||||||
|
const context = (userId: string) => ({
|
||||||
|
operationId: "test",
|
||||||
|
traceId: "test",
|
||||||
|
spaceId: backend.spaceId,
|
||||||
|
userId,
|
||||||
|
signal: new AbortController().signal,
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
backend: backend.kind,
|
||||||
|
mediaRoot,
|
||||||
|
spaceId: backend.spaceId,
|
||||||
|
userId: backend.userId,
|
||||||
|
deps,
|
||||||
|
events,
|
||||||
|
enqueued,
|
||||||
|
sources,
|
||||||
|
createAsset: (name) => backend.createAsset(name),
|
||||||
|
assetPath: (assetId, fileName) =>
|
||||||
|
path.join(mediaRoot, backend.spaceId, "assets", assetId, fileName),
|
||||||
|
setSource: (assetId, source) => sources.set(assetId, source),
|
||||||
|
addAnalysis: (assetId, kind, data) => backend.addAnalysis(assetId, kind, data),
|
||||||
|
addSegments: (assetId, ranges) => backend.addSegments(assetId, ranges),
|
||||||
|
async addArtifact(file, name, mimeType) {
|
||||||
|
const bytes = new Uint8Array(await readFile(file));
|
||||||
|
const stored = await artifacts.put({ name, mimeType, bytes }, context(backend.userId));
|
||||||
|
const row = await backend.prisma.artifact.create({
|
||||||
|
data: {
|
||||||
|
spaceId: backend.spaceId,
|
||||||
|
userId: backend.userId,
|
||||||
|
name,
|
||||||
|
mimeType,
|
||||||
|
size: bytes.byteLength,
|
||||||
|
hash: "test",
|
||||||
|
storageKey: stored.id,
|
||||||
|
},
|
||||||
|
select: { id: true },
|
||||||
|
});
|
||||||
|
return row.id;
|
||||||
|
},
|
||||||
|
createProject: (input) => backend.createProject(input),
|
||||||
|
createPlan: (projectId, edl) => {
|
||||||
|
const parsed = EdlSchema.parse(edl);
|
||||||
|
return backend.repos.createPlanVersion(backend.spaceId, {
|
||||||
|
projectId,
|
||||||
|
profileId: parsed.output.profileId,
|
||||||
|
edl: parsed,
|
||||||
|
createdBy: "user",
|
||||||
|
rationale: null,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
planVersions: (projectId, profileId) =>
|
||||||
|
backend.repos.listPlanVersions(backend.spaceId, projectId, profileId),
|
||||||
|
artifactCount: () => backend.prisma.artifact.count({ where: { spaceId: backend.spaceId } }),
|
||||||
|
async cleanup() {
|
||||||
|
await backend.cleanup();
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Backend {
|
||||||
|
kind: "memory" | "postgres";
|
||||||
|
prisma: PrismaClient;
|
||||||
|
repos: MediaRepos;
|
||||||
|
spaceId: string;
|
||||||
|
userId: string;
|
||||||
|
createAsset(name: string): Promise<string>;
|
||||||
|
addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise<void>;
|
||||||
|
addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise<void>;
|
||||||
|
createProject(input: {
|
||||||
|
assetIds: string[];
|
||||||
|
kit: BrandKit | null;
|
||||||
|
profileId: string;
|
||||||
|
}): Promise<string>;
|
||||||
|
cleanup(): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function postgresBackend(databaseUrl: string): Promise<Backend> {
|
||||||
|
const db = createDb(databaseUrl, { poolMax: 4 });
|
||||||
|
const prisma = db.prisma;
|
||||||
|
const repos = createMediaRepos(prisma);
|
||||||
|
const userId = `mr${randomUUID().replaceAll("-", "")}`;
|
||||||
|
await prisma.user.create({ data: { id: userId, name: "Test", email: `${userId}@rakazo.test` } });
|
||||||
|
const { spaceId } = await bootstrapUserSpace(
|
||||||
|
prisma,
|
||||||
|
{ id: userId },
|
||||||
|
{ signupsEnabled: "true", signupAllowlist: "" },
|
||||||
|
{ claimDeploymentOwner: false },
|
||||||
|
);
|
||||||
|
const source = await repos.createSource(spaceId, {
|
||||||
|
kind: "folder",
|
||||||
|
label: "Synthetic",
|
||||||
|
ownerUserId: userId,
|
||||||
|
config: {},
|
||||||
|
});
|
||||||
|
let analysisVersion = 0;
|
||||||
|
return {
|
||||||
|
kind: "postgres",
|
||||||
|
prisma,
|
||||||
|
repos,
|
||||||
|
spaceId,
|
||||||
|
userId,
|
||||||
|
async createAsset(name) {
|
||||||
|
const result = await repos.upsertSourceItems(spaceId, source.id, [
|
||||||
|
{
|
||||||
|
externalId: name,
|
||||||
|
kind: "video",
|
||||||
|
capturedAt: new Date(Date.UTC(2026, 0, 1)),
|
||||||
|
durationMs: 8000,
|
||||||
|
width: 640,
|
||||||
|
height: 360,
|
||||||
|
fps: 30,
|
||||||
|
gpsLat: null,
|
||||||
|
gpsLon: null,
|
||||||
|
deviceModel: null,
|
||||||
|
albumNames: [],
|
||||||
|
isFavorite: false,
|
||||||
|
fileName: name,
|
||||||
|
sizeBytes: 1,
|
||||||
|
revision: "r1",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
return result.changedAssetIds[0]!;
|
||||||
|
},
|
||||||
|
async addAnalysis(assetId, kind, data) {
|
||||||
|
analysisVersion += 1;
|
||||||
|
await repos.upsertAnalysis(spaceId, assetId, {
|
||||||
|
kind,
|
||||||
|
modelId: "emulator",
|
||||||
|
modelVersion: `v${analysisVersion}`,
|
||||||
|
data: data as never,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
async addSegments(assetId, ranges) {
|
||||||
|
await repos.replaceSegments(
|
||||||
|
spaceId,
|
||||||
|
assetId,
|
||||||
|
1,
|
||||||
|
ranges.map((range) => ({ kind: "shot", ...range })),
|
||||||
|
);
|
||||||
|
},
|
||||||
|
async createProject(input) {
|
||||||
|
const kit = input.kit ? await repos.upsertBrandKit(spaceId, { kit: input.kit }) : null;
|
||||||
|
const project = await repos.createProject(spaceId, userId, {
|
||||||
|
title: "Synthetic project",
|
||||||
|
goal: "teaser",
|
||||||
|
profileIds: [input.profileId],
|
||||||
|
assetIds: input.assetIds,
|
||||||
|
brandKitId: kit?.id ?? null,
|
||||||
|
});
|
||||||
|
return project.id;
|
||||||
|
},
|
||||||
|
async cleanup() {
|
||||||
|
const memberships = await prisma.member.findMany({
|
||||||
|
where: { userId },
|
||||||
|
select: { organizationId: true },
|
||||||
|
});
|
||||||
|
await prisma.organization.deleteMany({
|
||||||
|
where: { id: { in: memberships.map((member) => member.organizationId) } },
|
||||||
|
});
|
||||||
|
await prisma.user.deleteMany({ where: { id: userId } });
|
||||||
|
await prisma.$disconnect();
|
||||||
|
await db.pool.end();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface RenderRow {
|
||||||
|
id: string;
|
||||||
|
spaceId: string;
|
||||||
|
projectId: string;
|
||||||
|
planId: string;
|
||||||
|
profileId: string;
|
||||||
|
quality: string;
|
||||||
|
status: string;
|
||||||
|
progress: number;
|
||||||
|
outputs: unknown[];
|
||||||
|
gateReport: unknown;
|
||||||
|
ffprobe: unknown;
|
||||||
|
autoFixRound: number;
|
||||||
|
error: string | null;
|
||||||
|
createdBy: string;
|
||||||
|
createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** In-memory stand-in for the repos the render slice uses, with the same state rules. */
|
||||||
|
function memoryBackend(now: () => Date): Backend {
|
||||||
|
const spaceId = "space-memory";
|
||||||
|
const userId = "user-memory";
|
||||||
|
let counter = 0;
|
||||||
|
const id = (prefix: string) => `${prefix}${(++counter).toString().padStart(6, "0")}`;
|
||||||
|
const assets = new Set<string>();
|
||||||
|
const analyses = new Map<string, unknown>();
|
||||||
|
const segments: Array<{
|
||||||
|
id: string;
|
||||||
|
assetId: string;
|
||||||
|
startMs: number;
|
||||||
|
endMs: number;
|
||||||
|
paidOnly: boolean;
|
||||||
|
}> = [];
|
||||||
|
const kits = new Map<string, BrandKit>();
|
||||||
|
const projects = new Map<
|
||||||
|
string,
|
||||||
|
{ id: string; assetIds: string[]; collectionIds: string[]; brandKitId: string | null }
|
||||||
|
>();
|
||||||
|
const plans: Array<Parameters<typeof mapMediaEditPlan>[0]> = [];
|
||||||
|
const renders = new Map<string, RenderRow>();
|
||||||
|
const artifacts: Array<Record<string, unknown> & { id: string; spaceId: string }> = [];
|
||||||
|
|
||||||
|
const requireRender = (space: string, renderId: string) => {
|
||||||
|
const row = renders.get(renderId);
|
||||||
|
if (!row || row.spaceId !== space) throw new IsolationError();
|
||||||
|
return row;
|
||||||
|
};
|
||||||
|
const requirePlan = (space: string, planId: string) => {
|
||||||
|
const row = plans.find((plan) => plan.id === planId && plan.spaceId === space);
|
||||||
|
if (!row) throw new IsolationError();
|
||||||
|
return row;
|
||||||
|
};
|
||||||
|
const toRender = (row: RenderRow): MediaRender =>
|
||||||
|
mapMediaRender(structuredClone(row) as unknown as Parameters<typeof mapMediaRender>[0]);
|
||||||
|
|
||||||
|
const repos = {
|
||||||
|
async mediaSpaceOf(kind: string, renderId: string) {
|
||||||
|
return kind === "render" ? (renders.get(renderId)?.spaceId ?? null) : null;
|
||||||
|
},
|
||||||
|
async getPlan(space: string, planId: string) {
|
||||||
|
return mapMediaEditPlan(requirePlan(space, planId));
|
||||||
|
},
|
||||||
|
async createPlanVersion(
|
||||||
|
space: string,
|
||||||
|
input: {
|
||||||
|
projectId: string;
|
||||||
|
profileId: string;
|
||||||
|
edl: unknown;
|
||||||
|
createdBy: "agent" | "user";
|
||||||
|
rationale: string | null;
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
const edl = EdlSchema.parse(input.edl);
|
||||||
|
if (edl.output.profileId !== input.profileId) throw new Error("profile mismatch");
|
||||||
|
if (space !== spaceId || !projects.has(input.projectId)) throw new IsolationError();
|
||||||
|
const version =
|
||||||
|
plans
|
||||||
|
.filter((p) => p.projectId === input.projectId && p.profileId === input.profileId)
|
||||||
|
.reduce((max, p) => Math.max(max, p.version), 0) + 1;
|
||||||
|
const row = {
|
||||||
|
id: id("plan"),
|
||||||
|
spaceId: space,
|
||||||
|
projectId: input.projectId,
|
||||||
|
profileId: input.profileId,
|
||||||
|
version,
|
||||||
|
edl: structuredClone(edl),
|
||||||
|
createdBy: input.createdBy,
|
||||||
|
rationale: input.rationale,
|
||||||
|
createdAt: now(),
|
||||||
|
} as unknown as Parameters<typeof mapMediaEditPlan>[0];
|
||||||
|
plans.push(row);
|
||||||
|
return mapMediaEditPlan(row);
|
||||||
|
},
|
||||||
|
async listPlanVersions(space: string, projectId: string, profileId: string) {
|
||||||
|
return plans
|
||||||
|
.filter(
|
||||||
|
(p) => p.spaceId === space && p.projectId === projectId && p.profileId === profileId,
|
||||||
|
)
|
||||||
|
.sort((a, b) => b.version - a.version)
|
||||||
|
.map(mapMediaEditPlan);
|
||||||
|
},
|
||||||
|
async getProjectRecord(space: string, projectId: string) {
|
||||||
|
const project = projects.get(projectId);
|
||||||
|
if (!project || space !== spaceId) throw new IsolationError();
|
||||||
|
return { ...project, createdBy: userId };
|
||||||
|
},
|
||||||
|
async getBrandKit(space: string, kitId: string) {
|
||||||
|
const kit = kits.get(kitId);
|
||||||
|
if (!kit || space !== spaceId) throw new IsolationError();
|
||||||
|
return { id: kitId, kit, createdAt: now().toISOString(), updatedAt: now().toISOString() };
|
||||||
|
},
|
||||||
|
async collectionAssetIds() {
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
async listSegments(space: string, assetIds: string[]) {
|
||||||
|
if (space !== spaceId) return [];
|
||||||
|
return segments.filter((segment) => assetIds.includes(segment.assetId));
|
||||||
|
},
|
||||||
|
async latestAnalysis(space: string, assetId: string, kind: string) {
|
||||||
|
if (space !== spaceId || !assets.has(assetId)) throw new IsolationError();
|
||||||
|
const data = analyses.get(`${assetId}:${kind}`);
|
||||||
|
return data === undefined ? null : { data };
|
||||||
|
},
|
||||||
|
async createRender(
|
||||||
|
space: string,
|
||||||
|
input: { planId: string; quality: string; createdBy: string },
|
||||||
|
) {
|
||||||
|
const plan = requirePlan(space, input.planId);
|
||||||
|
const row: RenderRow = {
|
||||||
|
id: id("render"),
|
||||||
|
spaceId: space,
|
||||||
|
projectId: plan.projectId,
|
||||||
|
planId: plan.id,
|
||||||
|
profileId: plan.profileId,
|
||||||
|
quality: input.quality,
|
||||||
|
status: "queued",
|
||||||
|
progress: 0,
|
||||||
|
outputs: [],
|
||||||
|
gateReport: null,
|
||||||
|
ffprobe: null,
|
||||||
|
autoFixRound: 0,
|
||||||
|
error: null,
|
||||||
|
createdBy: input.createdBy,
|
||||||
|
createdAt: now(),
|
||||||
|
updatedAt: now(),
|
||||||
|
};
|
||||||
|
renders.set(row.id, row);
|
||||||
|
return toRender(row);
|
||||||
|
},
|
||||||
|
async getRender(space: string, renderId: string) {
|
||||||
|
return toRender(requireRender(space, renderId));
|
||||||
|
},
|
||||||
|
async getRenderRecord(space: string, renderId: string) {
|
||||||
|
return structuredClone(requireRender(space, renderId));
|
||||||
|
},
|
||||||
|
async claimRender(space: string, renderId: string) {
|
||||||
|
const row = requireRender(space, renderId);
|
||||||
|
if (row.status !== "queued") return null;
|
||||||
|
Object.assign(row, { status: "rendering", progress: 0, error: null, updatedAt: now() });
|
||||||
|
return toRender(row);
|
||||||
|
},
|
||||||
|
async updateRender(
|
||||||
|
space: string,
|
||||||
|
renderId: string,
|
||||||
|
patch: MediaRenderPatch,
|
||||||
|
from = ACTIVE_RENDER_STATUSES,
|
||||||
|
) {
|
||||||
|
const row = requireRender(space, renderId);
|
||||||
|
if (!from.includes(row.status as never)) return null;
|
||||||
|
for (const [key, value] of Object.entries(patch)) {
|
||||||
|
if (value === undefined) continue;
|
||||||
|
(row as unknown as Record<string, unknown>)[key] =
|
||||||
|
key === "progress" ? Math.min(1, Math.max(0, value as number)) : structuredClone(value);
|
||||||
|
}
|
||||||
|
row.updatedAt = now();
|
||||||
|
return toRender(row);
|
||||||
|
},
|
||||||
|
async cancelRender(space: string, renderId: string) {
|
||||||
|
const row = requireRender(space, renderId);
|
||||||
|
if (!ACTIVE_RENDER_STATUSES.includes(row.status as never)) {
|
||||||
|
throw new MediaStateConflictError(`Render is already ${row.status}`);
|
||||||
|
}
|
||||||
|
row.status = "canceled";
|
||||||
|
row.updatedAt = now();
|
||||||
|
return toRender(row);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const prisma = {
|
||||||
|
artifact: {
|
||||||
|
async findFirst(args: { where: { id: string; spaceId: string } }) {
|
||||||
|
return (
|
||||||
|
artifacts.find((a) => a.id === args.where.id && a.spaceId === args.where.spaceId) ?? null
|
||||||
|
);
|
||||||
|
},
|
||||||
|
async create(args: { data: Record<string, unknown> & { spaceId: string } }) {
|
||||||
|
const row = { ...args.data, id: id("artifact") };
|
||||||
|
artifacts.push(row);
|
||||||
|
return row;
|
||||||
|
},
|
||||||
|
async deleteMany(args: { where: { id: string; spaceId: string } }) {
|
||||||
|
const index = artifacts.findIndex(
|
||||||
|
(a) => a.id === args.where.id && a.spaceId === args.where.spaceId,
|
||||||
|
);
|
||||||
|
if (index >= 0) artifacts.splice(index, 1);
|
||||||
|
return { count: index >= 0 ? 1 : 0 };
|
||||||
|
},
|
||||||
|
async count(args: { where: { spaceId: string } }) {
|
||||||
|
return artifacts.filter((a) => a.spaceId === args.where.spaceId).length;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
kind: "memory",
|
||||||
|
prisma: prisma as unknown as PrismaClient,
|
||||||
|
repos: repos as unknown as MediaRepos,
|
||||||
|
spaceId,
|
||||||
|
userId,
|
||||||
|
async createAsset() {
|
||||||
|
const assetId = id("asset");
|
||||||
|
assets.add(assetId);
|
||||||
|
return assetId;
|
||||||
|
},
|
||||||
|
async addAnalysis(assetId, kind, data) {
|
||||||
|
analyses.set(`${assetId}:${kind}`, data);
|
||||||
|
},
|
||||||
|
async addSegments(assetId, ranges) {
|
||||||
|
for (const range of ranges) {
|
||||||
|
segments.push({ id: id("segment"), assetId, ...range, paidOnly: false });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
async createProject(input) {
|
||||||
|
let brandKitId: string | null = null;
|
||||||
|
if (input.kit) {
|
||||||
|
brandKitId = id("kit");
|
||||||
|
kits.set(brandKitId, input.kit);
|
||||||
|
}
|
||||||
|
const projectId = id("project");
|
||||||
|
projects.set(projectId, {
|
||||||
|
id: projectId,
|
||||||
|
assetIds: input.assetIds,
|
||||||
|
collectionIds: [],
|
||||||
|
brandKitId,
|
||||||
|
});
|
||||||
|
return projectId;
|
||||||
|
},
|
||||||
|
async cleanup() {},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Privacy stand-in
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export interface FakePrivacy extends MediaPrivacyService {
|
||||||
|
gateCalls: Array<{ autoFixRound: number; edl: Edl; ffprobe: FfprobeReport }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pass-through sources (no redactions in these plans) and the deterministic
|
||||||
|
* gate on the probed output. `failWith` adds a blocking content issue to every
|
||||||
|
* report, like a classifier hit at the start of the output.
|
||||||
|
*/
|
||||||
|
export function createFakePrivacy(
|
||||||
|
sources: Map<string, CompileSource>,
|
||||||
|
options: { failWith?: GateIssue; now?: () => Date } = {},
|
||||||
|
): FakePrivacy {
|
||||||
|
const gateCalls: FakePrivacy["gateCalls"] = [];
|
||||||
|
const unsupported = async (): Promise<never> => {
|
||||||
|
throw new Error("not used by render tests");
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
gateCalls,
|
||||||
|
async prepareSources(input): Promise<MediaPreparedSources> {
|
||||||
|
const prepared = new Map<string, CompileSource>();
|
||||||
|
for (const clip of input.edl.tracks.video) {
|
||||||
|
const source = sources.get(clip.assetId);
|
||||||
|
if (!source) throw new Error(`no source for ${clip.assetId}`);
|
||||||
|
prepared.set(clip.assetId, source);
|
||||||
|
}
|
||||||
|
return { sources: prepared, aiAlterations: [], fallbacks: [] };
|
||||||
|
},
|
||||||
|
async gate(input): Promise<GateReport> {
|
||||||
|
gateCalls.push({ autoFixRound: input.autoFixRound, edl: input.edl, ffprobe: input.ffprobe });
|
||||||
|
const report = evaluateGate({
|
||||||
|
profile: input.profile,
|
||||||
|
edl: input.edl,
|
||||||
|
ffprobe: input.ffprobe,
|
||||||
|
content: [],
|
||||||
|
faces: [],
|
||||||
|
trackingGaps: [],
|
||||||
|
marks: [],
|
||||||
|
people: [],
|
||||||
|
copy: input.copy,
|
||||||
|
aiAlterations: [],
|
||||||
|
bundleHasDisclosure: input.bundleHasDisclosure,
|
||||||
|
watermarkBox: null,
|
||||||
|
sampledFrames: 0,
|
||||||
|
autoFixRound: input.autoFixRound,
|
||||||
|
now: (options.now ?? (() => new Date()))(),
|
||||||
|
});
|
||||||
|
if (!options.failWith) return report;
|
||||||
|
return { ...report, passed: false, issues: [...report.issues, options.failWith] };
|
||||||
|
},
|
||||||
|
createMark: unsupported,
|
||||||
|
updateMark: unsupported,
|
||||||
|
removePerson: unsupported,
|
||||||
|
};
|
||||||
|
}
|
||||||
456
packages/adapters/src/media-render.test.ts
Normal file
456
packages/adapters/src/media-render.test.ts
Normal file
@ -0,0 +1,456 @@
|
|||||||
|
import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { MediaRenderer, MotionGraphicsRenderer } from "@rakazo/adapter-kit";
|
||||||
|
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||||
|
import type { FfprobeReport, GateIssue, MediaRender, RenderPlan } from "@rakazo/contracts";
|
||||||
|
import { findMetadataLeaks } from "@rakazo/media";
|
||||||
|
import { afterEach, describe, expect, it } from "vitest";
|
||||||
|
import type { MediaPrivacyService } from "./media-context.js";
|
||||||
|
import { FfmpegMediaRenderer } from "./media-ffmpeg-renderer.js";
|
||||||
|
import type { FakePrivacy, RenderHarness } from "./media-render.fixtures.js";
|
||||||
|
import {
|
||||||
|
createFakePrivacy,
|
||||||
|
createRenderHarness,
|
||||||
|
hasMediaTools,
|
||||||
|
renderTestProfile,
|
||||||
|
writeSyntheticClip,
|
||||||
|
} from "./media-render.fixtures.js";
|
||||||
|
import { createMediaRenderHandler, createMediaRenderService, pickEncoder } from "./media-render.js";
|
||||||
|
|
||||||
|
const profile = renderTestProfile();
|
||||||
|
|
||||||
|
const motion: MotionGraphicsRenderer = {
|
||||||
|
id: "unused",
|
||||||
|
templates: () => [],
|
||||||
|
render: async () => {
|
||||||
|
throw new Error("no motion in these plans");
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
interface FakeRenderer extends MediaRenderer {
|
||||||
|
plans: RenderPlan[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Executes write_file steps and writes placeholder outputs; probes report a
|
||||||
|
* clean, on-target file. `failEncoders` makes renders with those encoders fail.
|
||||||
|
*/
|
||||||
|
function fakeRenderer(
|
||||||
|
options: {
|
||||||
|
hardware?: string[];
|
||||||
|
failEncoders?: string[];
|
||||||
|
tags?: Record<string, string>;
|
||||||
|
posterTags?: Record<string, string>;
|
||||||
|
/** Resolves once a render started; the render then waits for its signal. */
|
||||||
|
hold?: { started: () => void };
|
||||||
|
} = {},
|
||||||
|
): FakeRenderer {
|
||||||
|
const plans: RenderPlan[] = [];
|
||||||
|
let durationMs = 0;
|
||||||
|
return {
|
||||||
|
id: "fake",
|
||||||
|
plans,
|
||||||
|
hardwareEncoders: async () => options.hardware ?? [],
|
||||||
|
async render(plan, { signal, onProgress } = {}) {
|
||||||
|
plans.push(plan);
|
||||||
|
const step = plan.steps.find((s) => s.kind === "ffmpeg" && s.id === "render");
|
||||||
|
if (step?.kind !== "ffmpeg") throw new Error("no render step");
|
||||||
|
durationMs = step.durationMs;
|
||||||
|
const encoder = step.argv[step.argv.indexOf("-c:v") + 1]!;
|
||||||
|
if (options.failEncoders?.includes(encoder)) throw new Error(`${encoder}: device lost`);
|
||||||
|
if (options.hold) {
|
||||||
|
options.hold.started();
|
||||||
|
await new Promise<void>((_, reject) => {
|
||||||
|
signal?.addEventListener("abort", () => {
|
||||||
|
const error = new Error("render aborted");
|
||||||
|
error.name = "AbortError";
|
||||||
|
reject(error);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for (const s of plan.steps) {
|
||||||
|
if (s.kind === "write_file") {
|
||||||
|
await mkdir(path.dirname(s.path), { recursive: true });
|
||||||
|
await writeFile(s.path, s.content);
|
||||||
|
}
|
||||||
|
if (s.kind === "ffmpeg") {
|
||||||
|
for (const file of s.produces) await writeFile(file, `rendered ${s.id}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
onProgress?.({ stepId: "render", progress: 1 });
|
||||||
|
},
|
||||||
|
async probe(file): Promise<FfprobeReport> {
|
||||||
|
const poster = file.endsWith(".jpg");
|
||||||
|
return {
|
||||||
|
durationMs: poster ? 0 : durationMs,
|
||||||
|
width: profile.video.width,
|
||||||
|
height: profile.video.height,
|
||||||
|
fps: poster ? 0 : profile.video.fps,
|
||||||
|
videoCodec: poster ? "mjpeg" : "h264",
|
||||||
|
audioCodec: poster ? null : "aac",
|
||||||
|
sizeBytes: (await stat(file)).size,
|
||||||
|
integratedLufs: poster ? null : -14.2,
|
||||||
|
truePeakDb: poster ? null : -1.5,
|
||||||
|
tags: poster ? (options.posterTags ?? {}) : (options.tags ?? {}),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
extractFrame: async () => {
|
||||||
|
throw new Error("not used");
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function edlFor(assetId: string) {
|
||||||
|
return {
|
||||||
|
edlVersion: 1,
|
||||||
|
output: {
|
||||||
|
profileId: profile.id,
|
||||||
|
width: profile.video.width,
|
||||||
|
height: profile.video.height,
|
||||||
|
fps: 30,
|
||||||
|
durationTargetMs: 3000,
|
||||||
|
durationHardMinMs: 1000,
|
||||||
|
durationHardMaxMs: 120_000,
|
||||||
|
},
|
||||||
|
tracks: {
|
||||||
|
video: [
|
||||||
|
{ id: "c1", assetId, inMs: 0, outMs: 1500, atMs: 0, reframe: "auto" },
|
||||||
|
{ id: "c2", assetId, inMs: 2000, outMs: 3500, atMs: 1500, reframe: "auto" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
captions: { mode: "both" },
|
||||||
|
overlays: [{ id: "t1", kind: "text", atMs: 200, durationMs: 1000, text: "Example" }],
|
||||||
|
metadata: { title: "Example teaser" },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Setup {
|
||||||
|
harness: RenderHarness;
|
||||||
|
privacy: FakePrivacy;
|
||||||
|
renderer: MediaRenderer;
|
||||||
|
run: (planQuality?: "draft" | "final") => Promise<MediaRender>;
|
||||||
|
renderDir: (renderId: string) => string;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cleanups: Array<() => Promise<void>> = [];
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const cleanup of cleanups) await cleanup();
|
||||||
|
cleanups = [];
|
||||||
|
});
|
||||||
|
|
||||||
|
async function setup(options: {
|
||||||
|
renderer: MediaRenderer;
|
||||||
|
privacy?: (sources: RenderHarness["sources"]) => MediaPrivacyService;
|
||||||
|
failWith?: GateIssue;
|
||||||
|
realSource?: boolean;
|
||||||
|
}): Promise<Setup & { service: ReturnType<typeof createMediaRenderService> }> {
|
||||||
|
const harness = await createRenderHarness({
|
||||||
|
profiles: [profile],
|
||||||
|
renderer: options.renderer,
|
||||||
|
motion,
|
||||||
|
});
|
||||||
|
cleanups.push(() => harness.cleanup());
|
||||||
|
const assetId = await harness.createAsset("source.mp4");
|
||||||
|
const file = harness.assetPath(assetId, "source.mp4");
|
||||||
|
if (options.realSource) {
|
||||||
|
await writeSyntheticClip(file, { seconds: 4, width: 320, height: 180, volumeDb: -30 });
|
||||||
|
} else {
|
||||||
|
await mkdir(path.dirname(file), { recursive: true });
|
||||||
|
await writeFile(file, "synthetic");
|
||||||
|
}
|
||||||
|
harness.setSource(assetId, {
|
||||||
|
path: file,
|
||||||
|
width: 320,
|
||||||
|
height: 180,
|
||||||
|
durationMs: 4000,
|
||||||
|
fps: 30,
|
||||||
|
hasAudio: true,
|
||||||
|
rotation: 0,
|
||||||
|
});
|
||||||
|
await harness.addAnalysis(assetId, "transcript", {
|
||||||
|
language: "en",
|
||||||
|
words: [
|
||||||
|
{ startMs: 100, endMs: 500, text: "Hello", confidence: 0.9 },
|
||||||
|
{ startMs: 2100, endMs: 2600, text: "again.", confidence: 0.9 },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
await harness.addSegments(assetId, [
|
||||||
|
{ startMs: 0, endMs: 2000 },
|
||||||
|
{ startMs: 2000, endMs: 4000 },
|
||||||
|
]);
|
||||||
|
const projectId = await harness.createProject({
|
||||||
|
assetIds: [assetId],
|
||||||
|
kit: null,
|
||||||
|
profileId: profile.id,
|
||||||
|
});
|
||||||
|
const plan = await harness.createPlan(projectId, edlFor(assetId));
|
||||||
|
const privacy = createFakePrivacy(harness.sources, { failWith: options.failWith });
|
||||||
|
const service = createMediaRenderService(harness.deps);
|
||||||
|
const handler = createMediaRenderHandler(
|
||||||
|
harness.deps,
|
||||||
|
options.privacy ? options.privacy(harness.sources) : privacy,
|
||||||
|
{ cancelPollMs: 50 },
|
||||||
|
);
|
||||||
|
const actor = { spaceId: harness.spaceId, userId: harness.userId };
|
||||||
|
return {
|
||||||
|
harness,
|
||||||
|
privacy,
|
||||||
|
renderer: options.renderer,
|
||||||
|
service,
|
||||||
|
renderDir: (renderId) => path.join(harness.mediaRoot, harness.spaceId, "renders", renderId),
|
||||||
|
async run(quality = "draft") {
|
||||||
|
const render = await service.start(actor, { planId: plan.id, quality });
|
||||||
|
await handler({ renderId: render.id });
|
||||||
|
return harness.deps.repos.getRender(harness.spaceId, render.id);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function exists(file: string): Promise<boolean> {
|
||||||
|
return stat(file).then(
|
||||||
|
() => true,
|
||||||
|
() => false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("media render service", () => {
|
||||||
|
it("queues a render job and announces it", async () => {
|
||||||
|
const { harness, service } = await setup({ renderer: fakeRenderer() });
|
||||||
|
expect(harness.backend).toBe(
|
||||||
|
process.env.VERIFY_DATABASE && process.env.DATABASE_URL ? "postgres" : "memory",
|
||||||
|
);
|
||||||
|
const projectId = await harness.createProject({
|
||||||
|
assetIds: [],
|
||||||
|
kit: null,
|
||||||
|
profileId: profile.id,
|
||||||
|
});
|
||||||
|
const assetId = [...harness.sources.keys()][0]!;
|
||||||
|
const plan = await harness.createPlan(projectId, edlFor(assetId));
|
||||||
|
const actor = { spaceId: harness.spaceId, userId: harness.userId };
|
||||||
|
const render = await service.start(actor, { planId: plan.id, quality: "final" });
|
||||||
|
expect(render).toMatchObject({ status: "queued", quality: "final", planId: plan.id });
|
||||||
|
expect(harness.enqueued).toContain(`media.render:${JSON.stringify({ renderId: render.id })}`);
|
||||||
|
expect(harness.events).toContainEqual({
|
||||||
|
type: "media.render",
|
||||||
|
renderId: render.id,
|
||||||
|
projectId,
|
||||||
|
status: "queued",
|
||||||
|
progress: 0,
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
service.start({ ...actor, spaceId: "space-other" }, { planId: plan.id, quality: "draft" }),
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders, gates, stores outputs and serves only recorded files", async () => {
|
||||||
|
const renderer = fakeRenderer();
|
||||||
|
const { harness, service, run, renderDir } = await setup({ renderer });
|
||||||
|
const render = await run();
|
||||||
|
expect(render.status).toBe("done");
|
||||||
|
expect(render.progress).toBe(1);
|
||||||
|
expect(render.gateReport?.passed).toBe(true);
|
||||||
|
expect(render.ffprobe?.integratedLufs).toBe(-14.2);
|
||||||
|
expect(render.outputs.map((output) => output.fileName).sort()).toEqual([
|
||||||
|
"captions.srt",
|
||||||
|
"captions.vtt",
|
||||||
|
"poster.jpg",
|
||||||
|
"video.mp4",
|
||||||
|
]);
|
||||||
|
const video = render.outputs.find((output) => output.kind === "video")!;
|
||||||
|
expect(video.artifactId).toBeNull();
|
||||||
|
expect(render.outputs.filter((output) => output.artifactId !== null)).toHaveLength(3);
|
||||||
|
expect(await harness.artifactCount()).toBe(3);
|
||||||
|
// Intermediates are gone; only the published outputs stay.
|
||||||
|
expect((await readdir(renderDir(render.id))).sort()).toEqual([
|
||||||
|
"captions.srt",
|
||||||
|
"captions.vtt",
|
||||||
|
"poster.jpg",
|
||||||
|
"video.mp4",
|
||||||
|
]);
|
||||||
|
const actor = { spaceId: harness.spaceId, userId: harness.userId };
|
||||||
|
expect(await service.outputPath(actor, render.id, "video.mp4")).toBe(
|
||||||
|
path.join(renderDir(render.id), "video.mp4"),
|
||||||
|
);
|
||||||
|
expect(await service.outputPath(actor, render.id, "../video.mp4")).toBeNull();
|
||||||
|
expect(await service.outputPath(actor, render.id, "work")).toBeNull();
|
||||||
|
await expect(
|
||||||
|
service.outputPath({ ...actor, spaceId: "space-other" }, render.id, "video.mp4"),
|
||||||
|
).rejects.toThrow();
|
||||||
|
const statuses = harness.events.flatMap((event) =>
|
||||||
|
event.type === "media.render" && event.renderId === render.id ? [event.status] : [],
|
||||||
|
);
|
||||||
|
expect(statuses[0]).toBe("queued");
|
||||||
|
expect(statuses).toContain("gating");
|
||||||
|
expect(statuses.at(-1)).toBe("done");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses to render when redactions cannot be applied (fail closed)", async () => {
|
||||||
|
const renderer = fakeRenderer();
|
||||||
|
let calls = 0;
|
||||||
|
const { harness, run, renderDir } = await setup({
|
||||||
|
renderer,
|
||||||
|
privacy: (sources) => ({
|
||||||
|
...createFakePrivacy(sources),
|
||||||
|
async prepareSources() {
|
||||||
|
calls += 1;
|
||||||
|
throw new Error("face track lost for 800 ms");
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const render = await run("final");
|
||||||
|
expect(calls).toBe(1);
|
||||||
|
expect(render.status).toBe("failed");
|
||||||
|
expect(render.error).toBe("Anonymization could not be applied. Nothing was rendered.");
|
||||||
|
expect(render.outputs).toEqual([]);
|
||||||
|
expect(renderer.plans).toHaveLength(0);
|
||||||
|
expect(await exists(renderDir(render.id))).toBe(false);
|
||||||
|
expect(await harness.artifactCount()).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("names the missing capability and refuses when a clip was not prepared", async () => {
|
||||||
|
const unsupported = await setup({
|
||||||
|
renderer: fakeRenderer(),
|
||||||
|
privacy: (sources) => ({
|
||||||
|
...createFakePrivacy(sources),
|
||||||
|
async prepareSources() {
|
||||||
|
throw new UnsupportedMediaOperationError("inpaintVideo", "no model");
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
expect((await unsupported.run()).error).toBe(
|
||||||
|
"Anonymization needs inpaintVideo, which is not available. Nothing was rendered.",
|
||||||
|
);
|
||||||
|
|
||||||
|
const renderer = fakeRenderer();
|
||||||
|
const partial = await setup({
|
||||||
|
renderer,
|
||||||
|
privacy: (sources) => ({
|
||||||
|
...createFakePrivacy(sources),
|
||||||
|
async prepareSources() {
|
||||||
|
return { sources: new Map(), aiAlterations: [], fallbacks: [] };
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const render = await partial.run();
|
||||||
|
expect(render.status).toBe("failed");
|
||||||
|
expect(render.error).toBe("Anonymization did not prepare every clip. Nothing was rendered.");
|
||||||
|
expect(renderer.plans).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to software when a hardware encoder fails and skips it afterwards", async () => {
|
||||||
|
const renderer = fakeRenderer({
|
||||||
|
hardware: ["hevc_vaapi", "h264_vaapi"],
|
||||||
|
failEncoders: ["h264_vaapi"],
|
||||||
|
});
|
||||||
|
const { run } = await setup({ renderer });
|
||||||
|
const encoders = () =>
|
||||||
|
renderer.plans.map((plan) => {
|
||||||
|
const step = plan.steps.find((s) => s.kind === "ffmpeg" && s.id === "render");
|
||||||
|
return step?.kind === "ffmpeg" ? step.argv[step.argv.indexOf("-c:v") + 1] : null;
|
||||||
|
});
|
||||||
|
expect((await run("final")).status).toBe("done");
|
||||||
|
expect(encoders()).toEqual(["h264_vaapi", "libx264"]);
|
||||||
|
expect((await run("final")).status).toBe("done");
|
||||||
|
expect(encoders()).toEqual(["h264_vaapi", "libx264", "libx264"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses software for drafts and a working hardware encoder for finals", async () => {
|
||||||
|
const renderer = fakeRenderer({ hardware: ["hevc_nvenc", "h264_nvenc"] });
|
||||||
|
expect(await pickEncoder(renderer, profile, "draft")).toBe("libx264");
|
||||||
|
expect(await pickEncoder(renderer, profile, "final")).toBe("h264_nvenc");
|
||||||
|
expect(await pickEncoder(fakeRenderer(), profile, "final")).toBe("libx264");
|
||||||
|
const hevc = renderTestProfile({
|
||||||
|
video: { ...profile.video, codec: "hevc" },
|
||||||
|
});
|
||||||
|
expect(await pickEncoder(renderer, hevc, "final")).toBe("hevc_nvenc");
|
||||||
|
expect(await pickEncoder(renderer, hevc, "draft")).toBe("libx265");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails a render whose video or poster still carries source metadata", async () => {
|
||||||
|
const leaky = await setup({
|
||||||
|
renderer: fakeRenderer({ tags: { "format.location": "+00+000/" } }),
|
||||||
|
});
|
||||||
|
const render = await leaky.run();
|
||||||
|
expect(render.status).toBe("failed");
|
||||||
|
expect(render.error).toBe("The export still carried source metadata.");
|
||||||
|
expect(await exists(leaky.renderDir(render.id))).toBe(false);
|
||||||
|
|
||||||
|
const poster = await setup({
|
||||||
|
renderer: fakeRenderer({ posterTags: { "format.make": "ExampleCam" } }),
|
||||||
|
});
|
||||||
|
expect((await poster.run()).error).toBe("The poster still carried source metadata.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("auto-fixes a blocked gate at most three times, then blocks", async () => {
|
||||||
|
const renderer = fakeRenderer();
|
||||||
|
const { harness, privacy, run, renderDir } = await setup({
|
||||||
|
renderer,
|
||||||
|
failWith: {
|
||||||
|
check: "content",
|
||||||
|
severity: "block",
|
||||||
|
code: "classifier_violence",
|
||||||
|
message: "violence score above the profile threshold",
|
||||||
|
atMs: 0,
|
||||||
|
endMs: 400,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const render = await run();
|
||||||
|
expect(render.status).toBe("blocked");
|
||||||
|
expect(render.gateReport?.passed).toBe(false);
|
||||||
|
expect(privacy.gateCalls.map((call) => call.autoFixRound)).toEqual([0, 1, 2, 3]);
|
||||||
|
expect(renderer.plans).toHaveLength(4);
|
||||||
|
const record = await harness.deps.repos.getRenderRecord(harness.spaceId, render.id);
|
||||||
|
expect(record.autoFixRound).toBe(3);
|
||||||
|
const versions = await harness.planVersions(render.projectId, profile.id);
|
||||||
|
expect(versions).toHaveLength(4);
|
||||||
|
expect(versions[0]!.createdBy).toBe("agent");
|
||||||
|
expect(versions[0]!.rationale).toContain("Auto-fix 3/3");
|
||||||
|
expect(render.outputs).toEqual([]);
|
||||||
|
expect(await exists(renderDir(render.id))).toBe(false);
|
||||||
|
expect(await harness.artifactCount()).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cancels a running render, stops it and removes its files", async () => {
|
||||||
|
let started!: () => void;
|
||||||
|
const running = new Promise<void>((resolve) => {
|
||||||
|
started = resolve;
|
||||||
|
});
|
||||||
|
const renderer = fakeRenderer({ hold: { started: () => started() } });
|
||||||
|
const { harness, service, run, renderDir } = await setup({ renderer });
|
||||||
|
const result = run();
|
||||||
|
await running;
|
||||||
|
const active = harness.events.findLast((event) => event.type === "media.render");
|
||||||
|
const renderId = active?.type === "media.render" ? active.renderId : "";
|
||||||
|
const canceled = await service.cancel(
|
||||||
|
{ spaceId: harness.spaceId, userId: harness.userId },
|
||||||
|
renderId,
|
||||||
|
);
|
||||||
|
expect(canceled.status).toBe("canceled");
|
||||||
|
const render = await result;
|
||||||
|
expect(render.status).toBe("canceled");
|
||||||
|
expect(await exists(renderDir(renderId))).toBe(false);
|
||||||
|
await expect(
|
||||||
|
service.cancel({ spaceId: harness.spaceId, userId: harness.userId }, renderId),
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!hasMediaTools)("media render job with real ffmpeg", () => {
|
||||||
|
it("renders a synthetic project end to end and records a clean probe", async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer();
|
||||||
|
const { run, renderDir } = await setup({ renderer, realSource: true });
|
||||||
|
const render = await run("draft");
|
||||||
|
expect(render.error).toBeNull();
|
||||||
|
expect(render.status).toBe("done");
|
||||||
|
const probe = render.ffprobe!;
|
||||||
|
expect(probe.width).toBe(profile.video.width);
|
||||||
|
expect(probe.height).toBe(profile.video.height);
|
||||||
|
expect(Math.abs(probe.durationMs - 3000)).toBeLessThanOrEqual(1000 / 30 + 1);
|
||||||
|
expect(Math.abs(probe.integratedLufs! - profile.audio.targetLufs)).toBeLessThanOrEqual(1);
|
||||||
|
expect(findMetadataLeaks(probe.tags)).toEqual([]);
|
||||||
|
const video = path.join(renderDir(render.id), "video.mp4");
|
||||||
|
const reprobed = await renderer.probe(video);
|
||||||
|
expect(reprobed.tags["format.title"]).toBe("Example teaser");
|
||||||
|
expect(Object.values(reprobed.tags).join(" ")).not.toContain("ExampleCam");
|
||||||
|
}, 120_000);
|
||||||
|
});
|
||||||
910
packages/adapters/src/media-render.ts
Normal file
910
packages/adapters/src/media-render.ts
Normal file
@ -0,0 +1,910 @@
|
|||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { AdapterContext, MediaJobHandlers, MediaRenderProgress } from "@rakazo/adapter-kit";
|
||||||
|
import { mediaRenderJob, UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||||
|
import type {
|
||||||
|
AiAlteration,
|
||||||
|
BrandKit,
|
||||||
|
Edl,
|
||||||
|
GateReport,
|
||||||
|
MediaRender,
|
||||||
|
MediaRenderOutput,
|
||||||
|
MediaRenderQuality,
|
||||||
|
PlatformProfile,
|
||||||
|
RenderPlan,
|
||||||
|
} from "@rakazo/contracts";
|
||||||
|
import { MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
|
||||||
|
import type { MediaRenderPatch } from "@rakazo/db";
|
||||||
|
import { InvalidMediaInputError } from "@rakazo/db";
|
||||||
|
import { getLogger } from "@rakazo/logging";
|
||||||
|
import type { CaptionWord } from "@rakazo/media";
|
||||||
|
import { compileRenderPlan, findMetadataLeaks, planAutoFix } from "@rakazo/media";
|
||||||
|
import { resolveInside } from "@rakazo/media/node";
|
||||||
|
import type {
|
||||||
|
MediaActor,
|
||||||
|
MediaPreparedSources,
|
||||||
|
MediaPrivacyService,
|
||||||
|
MediaRenderService,
|
||||||
|
MediaServiceDeps,
|
||||||
|
} from "./media-context.js";
|
||||||
|
import { MediaToolMissingError } from "./media-folder-source-probe.js";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Render service (API side) and render job (worker side).
|
||||||
|
*
|
||||||
|
* A render runs: claim → prepare redacted sources (privacy slice) → pre-render
|
||||||
|
* motion clips → compile → ffmpeg → probe → gate. A blocked gate gets up to
|
||||||
|
* MAX_AUTO_FIX_ROUNDS compliance-only repairs (planAutoFix); each repair is
|
||||||
|
* stored as a new plan version by the agent and rendered again under the same
|
||||||
|
* render id. Only a passed gate moves outputs out of the work directory and
|
||||||
|
* marks the render done; every other outcome removes the render's files.
|
||||||
|
*
|
||||||
|
* Layout: `<spaceId>/renders/<renderId>/{video.mp4,poster.jpg,captions.srt,
|
||||||
|
* captions.vtt}` with intermediates in `work/` beside them.
|
||||||
|
*
|
||||||
|
* Cancel: the API marks the render canceled and aborts it when this process
|
||||||
|
* runs it. A worker in another process notices through a failed status update
|
||||||
|
* (updateRender returns null once canceled) or the periodic status check.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Renders running in this process, so a cancel can kill ffmpeg immediately. */
|
||||||
|
export type RunningMediaRenders = Map<string, AbortController>;
|
||||||
|
|
||||||
|
const runningRenders: RunningMediaRenders = new Map();
|
||||||
|
|
||||||
|
const PROGRESS_INTERVAL_MS = 1000;
|
||||||
|
const CANCEL_POLL_MS = 2000;
|
||||||
|
/** Share of the progress bar spent rendering; probing and gating take the rest. */
|
||||||
|
const RENDER_SHARE = 0.85;
|
||||||
|
|
||||||
|
const OUTPUT_FILES = {
|
||||||
|
video: "video.mp4",
|
||||||
|
poster: "poster.jpg",
|
||||||
|
subtitles_srt: "captions.srt",
|
||||||
|
subtitles_vtt: "captions.vtt",
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
const OUTPUT_MIME: Record<keyof typeof OUTPUT_FILES, string> = {
|
||||||
|
video: "video/mp4",
|
||||||
|
poster: "image/jpeg",
|
||||||
|
subtitles_srt: "application/x-subrip",
|
||||||
|
subtitles_vtt: "text/vtt",
|
||||||
|
};
|
||||||
|
|
||||||
|
const EXTENSION_BY_MIME: Record<string, string> = {
|
||||||
|
"image/png": ".png",
|
||||||
|
"image/jpeg": ".jpg",
|
||||||
|
"image/svg+xml": ".svg",
|
||||||
|
"image/webp": ".webp",
|
||||||
|
"audio/mpeg": ".mp3",
|
||||||
|
"audio/wav": ".wav",
|
||||||
|
"audio/x-wav": ".wav",
|
||||||
|
"audio/aac": ".aac",
|
||||||
|
"audio/mp4": ".m4a",
|
||||||
|
"audio/ogg": ".ogg",
|
||||||
|
"audio/flac": ".flac",
|
||||||
|
"font/ttf": ".ttf",
|
||||||
|
"font/otf": ".otf",
|
||||||
|
"font/collection": ".ttc",
|
||||||
|
};
|
||||||
|
|
||||||
|
/** A failure whose message is safe to show the user (no paths, no tool output). */
|
||||||
|
export class MediaRenderFailure extends Error {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "MediaRenderFailure";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class RenderCanceled extends Error {
|
||||||
|
constructor() {
|
||||||
|
super("render canceled");
|
||||||
|
this.name = "RenderCanceled";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function userMessage(error: unknown): string {
|
||||||
|
if (error instanceof MediaRenderFailure) return error.message;
|
||||||
|
if (error instanceof UnsupportedMediaOperationError) {
|
||||||
|
return `${error.operation} is not available on this server.`;
|
||||||
|
}
|
||||||
|
if (error instanceof MediaToolMissingError) return "ffmpeg is not installed on the media worker.";
|
||||||
|
return "Rendering failed. Check the plan and try again.";
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderEvent(render: MediaRender) {
|
||||||
|
return {
|
||||||
|
type: "media.render" as const,
|
||||||
|
renderId: render.id,
|
||||||
|
projectId: render.projectId,
|
||||||
|
status: render.status,
|
||||||
|
progress: render.progress,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function publishRender(deps: MediaServiceDeps, spaceId: string, render: MediaRender) {
|
||||||
|
try {
|
||||||
|
await deps.publish(spaceId, renderEvent(render));
|
||||||
|
} catch (error) {
|
||||||
|
getLogger().warn("media render event not published", {
|
||||||
|
renderId: render.id,
|
||||||
|
reason: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Service
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export function createMediaRenderService(
|
||||||
|
deps: MediaServiceDeps,
|
||||||
|
options: { running?: RunningMediaRenders } = {},
|
||||||
|
): MediaRenderService {
|
||||||
|
const running = options.running ?? runningRenders;
|
||||||
|
return {
|
||||||
|
async start(actor, input) {
|
||||||
|
const plan = await deps.repos.getPlan(actor.spaceId, input.planId);
|
||||||
|
const { profiles } = await deps.profiles();
|
||||||
|
if (!profiles.has(plan.profileId)) {
|
||||||
|
throw new InvalidMediaInputError("The plan's platform profile is not available");
|
||||||
|
}
|
||||||
|
const render = await deps.repos.createRender(actor.spaceId, {
|
||||||
|
planId: plan.id,
|
||||||
|
quality: input.quality,
|
||||||
|
createdBy: actor.userId,
|
||||||
|
});
|
||||||
|
await deps.jobs.enqueue(mediaRenderJob(render.id));
|
||||||
|
await publishRender(deps, actor.spaceId, render);
|
||||||
|
return render;
|
||||||
|
},
|
||||||
|
|
||||||
|
async cancel(actor, renderId) {
|
||||||
|
const render = await deps.repos.cancelRender(actor.spaceId, renderId);
|
||||||
|
running.get(renderId)?.abort();
|
||||||
|
await deps.jobs.cancel(mediaRenderJob(renderId).replaceKey!).catch((error: unknown) => {
|
||||||
|
getLogger().warn("media render job not dequeued", {
|
||||||
|
renderId,
|
||||||
|
reason: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await publishRender(deps, actor.spaceId, render);
|
||||||
|
return render;
|
||||||
|
},
|
||||||
|
|
||||||
|
async outputPath(actor, renderId, fileName) {
|
||||||
|
const record = await deps.repos.getRenderRecord(actor.spaceId, renderId);
|
||||||
|
if (record.status !== "done") return null;
|
||||||
|
if (path.basename(fileName) !== fileName || fileName.startsWith(".")) return null;
|
||||||
|
const outputs = Array.isArray(record.outputs) ? record.outputs : [];
|
||||||
|
const recorded = outputs.some(
|
||||||
|
(output) =>
|
||||||
|
output !== null &&
|
||||||
|
typeof output === "object" &&
|
||||||
|
!Array.isArray(output) &&
|
||||||
|
output.fileName === fileName,
|
||||||
|
);
|
||||||
|
if (!recorded) return null;
|
||||||
|
// Derive the location instead of trusting a stored path.
|
||||||
|
const file = await resolveInside(
|
||||||
|
deps.mediaRoot,
|
||||||
|
`${actor.spaceId}/renders/${renderId}/${fileName}`,
|
||||||
|
);
|
||||||
|
const found = await stat(file).catch(() => null);
|
||||||
|
return found?.isFile() ? file : null;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Job handler
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
interface BrandFiles {
|
||||||
|
/** Font artifact id → file. */
|
||||||
|
fonts: Map<string, string>;
|
||||||
|
fontList: Array<{ family: string; path: string }>;
|
||||||
|
logos: string[];
|
||||||
|
watermark: string | null;
|
||||||
|
lut: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeExtension(name: string, mimeType: string): string {
|
||||||
|
const fromName = path.extname(name).toLowerCase();
|
||||||
|
if (/^\.[a-z0-9]{1,5}$/.test(fromName)) return fromName;
|
||||||
|
return EXTENSION_BY_MIME[mimeType.toLowerCase()] ?? ".bin";
|
||||||
|
}
|
||||||
|
|
||||||
|
const even = (value: number) => Math.max(2, Math.floor(value / 2) * 2);
|
||||||
|
|
||||||
|
function finiteNumber(value: unknown): value is number {
|
||||||
|
return typeof value === "number" && Number.isFinite(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Transcript words from a stored TranscribeResult. */
|
||||||
|
export function transcriptWords(data: unknown): CaptionWord[] {
|
||||||
|
const words = (data as { words?: unknown } | null)?.words;
|
||||||
|
if (!Array.isArray(words)) return [];
|
||||||
|
return words.flatMap((word) => {
|
||||||
|
const w = word as { startMs?: unknown; endMs?: unknown; text?: unknown };
|
||||||
|
if (!finiteNumber(w.startMs) || !finiteNumber(w.endMs) || typeof w.text !== "string") {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return [{ startMs: w.startMs, endMs: w.endMs, text: w.text }];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
type Subject = { tMs: number; cx: number; cy: number; strength: number };
|
||||||
|
|
||||||
|
/** Reframe subjects from a stored saliency result, or face centres from detections. */
|
||||||
|
export function subjectPoints(saliency: unknown, faces: unknown): Subject[] {
|
||||||
|
const points = (saliency as { points?: unknown } | null)?.points;
|
||||||
|
if (Array.isArray(points)) {
|
||||||
|
const parsed = points.flatMap((point) => {
|
||||||
|
const p = point as Partial<Subject>;
|
||||||
|
return finiteNumber(p.tMs) &&
|
||||||
|
finiteNumber(p.cx) &&
|
||||||
|
finiteNumber(p.cy) &&
|
||||||
|
finiteNumber(p.strength)
|
||||||
|
? [{ tMs: p.tMs, cx: p.cx, cy: p.cy, strength: p.strength }]
|
||||||
|
: [];
|
||||||
|
});
|
||||||
|
if (parsed.length > 0) return parsed;
|
||||||
|
}
|
||||||
|
const frames = (faces as { frames?: unknown } | null)?.frames;
|
||||||
|
if (!Array.isArray(frames)) return [];
|
||||||
|
return frames.flatMap((frame) => {
|
||||||
|
const f = frame as { tMs?: unknown; faces?: unknown };
|
||||||
|
if (!finiteNumber(f.tMs) || !Array.isArray(f.faces)) return [];
|
||||||
|
const tMs = f.tMs;
|
||||||
|
return f.faces.flatMap((face) => {
|
||||||
|
const box = (face as { box?: { x?: unknown; y?: unknown; w?: unknown; h?: unknown } }).box;
|
||||||
|
const confidence = (face as { confidence?: unknown }).confidence;
|
||||||
|
if (!box || !finiteNumber(box.x) || !finiteNumber(box.y)) return [];
|
||||||
|
if (!finiteNumber(box.w) || !finiteNumber(box.h)) return [];
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
tMs,
|
||||||
|
cx: box.x + box.w / 2,
|
||||||
|
cy: box.y + box.h / 2,
|
||||||
|
strength: finiteNumber(confidence) ? confidence : 1,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MediaRenderHandlerOptions {
|
||||||
|
running?: RunningMediaRenders;
|
||||||
|
/** How often a running render checks whether another process canceled it. */
|
||||||
|
cancelPollMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Worker handler for `media.render`. Never throws for a render failure: the
|
||||||
|
* render record carries the outcome, so a job retry has nothing to redo.
|
||||||
|
*/
|
||||||
|
export function createMediaRenderHandler(
|
||||||
|
deps: MediaServiceDeps,
|
||||||
|
privacy: MediaPrivacyService,
|
||||||
|
options: MediaRenderHandlerOptions = {},
|
||||||
|
): MediaJobHandlers["media.render"] {
|
||||||
|
const running = options.running ?? runningRenders;
|
||||||
|
const pollMs = options.cancelPollMs ?? CANCEL_POLL_MS;
|
||||||
|
const { repos } = deps;
|
||||||
|
|
||||||
|
return async ({ renderId }) => {
|
||||||
|
const spaceId = await repos.mediaSpaceOf("render", renderId);
|
||||||
|
if (!spaceId) return;
|
||||||
|
const claimed = await repos.claimRender(spaceId, renderId);
|
||||||
|
if (!claimed) return;
|
||||||
|
await publishRender(deps, spaceId, claimed);
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
running.set(renderId, controller);
|
||||||
|
const poll = setInterval(() => {
|
||||||
|
repos
|
||||||
|
.getRender(spaceId, renderId)
|
||||||
|
.then((render) => {
|
||||||
|
if (render.status === "canceled") controller.abort();
|
||||||
|
})
|
||||||
|
.catch(() => undefined);
|
||||||
|
}, pollMs);
|
||||||
|
poll.unref?.();
|
||||||
|
const createdArtifacts: Array<{ id: string; storageKey: string }> = [];
|
||||||
|
let renderDir: string | null = null;
|
||||||
|
let userId = "";
|
||||||
|
|
||||||
|
try {
|
||||||
|
renderDir = await resolveInside(deps.mediaRoot, `${spaceId}/renders/${renderId}`);
|
||||||
|
const record = await repos.getRenderRecord(spaceId, renderId);
|
||||||
|
userId = record.createdBy;
|
||||||
|
await runRender({
|
||||||
|
deps,
|
||||||
|
privacy,
|
||||||
|
spaceId,
|
||||||
|
renderId,
|
||||||
|
renderDir,
|
||||||
|
record,
|
||||||
|
signal: controller.signal,
|
||||||
|
abort: () => controller.abort(),
|
||||||
|
createdArtifacts,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (renderDir) await rm(renderDir, { recursive: true, force: true }).catch(() => undefined);
|
||||||
|
await removeArtifacts(deps, spaceId, userId, renderId, createdArtifacts);
|
||||||
|
const current = await repos.getRender(spaceId, renderId).catch(() => null);
|
||||||
|
if (current?.status === "canceled") {
|
||||||
|
await publishRender(deps, spaceId, current);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
getLogger().error("media render failed", {
|
||||||
|
renderId,
|
||||||
|
reason: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
const failed = await repos
|
||||||
|
.updateRender(spaceId, renderId, { status: "failed", error: userMessage(error) })
|
||||||
|
.catch(() => null);
|
||||||
|
if (failed) await publishRender(deps, spaceId, failed);
|
||||||
|
} finally {
|
||||||
|
clearInterval(poll);
|
||||||
|
running.delete(renderId);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function artifactContext(spaceId: string, userId: string, renderId: string): AdapterContext {
|
||||||
|
return {
|
||||||
|
operationId: `media-render:${renderId}`,
|
||||||
|
traceId: `media-render:${renderId}`,
|
||||||
|
spaceId,
|
||||||
|
userId,
|
||||||
|
signal: new AbortController().signal,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function removeArtifacts(
|
||||||
|
deps: MediaServiceDeps,
|
||||||
|
spaceId: string,
|
||||||
|
userId: string,
|
||||||
|
renderId: string,
|
||||||
|
created: Array<{ id: string; storageKey: string }>,
|
||||||
|
) {
|
||||||
|
for (const artifact of created) {
|
||||||
|
await deps.prisma.artifact
|
||||||
|
.deleteMany({ where: { id: artifact.id, spaceId } })
|
||||||
|
.catch(() => undefined);
|
||||||
|
await deps.artifacts
|
||||||
|
.remove(artifact.storageKey, artifactContext(spaceId, userId, renderId))
|
||||||
|
.catch(() => undefined);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type RenderRecord = Awaited<ReturnType<MediaServiceDeps["repos"]["getRenderRecord"]>>;
|
||||||
|
|
||||||
|
interface RunInput {
|
||||||
|
deps: MediaServiceDeps;
|
||||||
|
privacy: MediaPrivacyService;
|
||||||
|
spaceId: string;
|
||||||
|
renderId: string;
|
||||||
|
renderDir: string;
|
||||||
|
record: RenderRecord;
|
||||||
|
signal: AbortSignal;
|
||||||
|
abort: () => void;
|
||||||
|
createdArtifacts: Array<{ id: string; storageKey: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runRender(input: RunInput): Promise<void> {
|
||||||
|
const { deps, privacy, spaceId, renderId, renderDir, record, signal } = input;
|
||||||
|
const { repos } = deps;
|
||||||
|
const actor: MediaActor = { spaceId, userId: record.createdBy };
|
||||||
|
const context = artifactContext(spaceId, actor.userId, renderId);
|
||||||
|
|
||||||
|
/** Apply a patch while the render is active; a null result means it was canceled. */
|
||||||
|
const update = async (patch: MediaRenderPatch) => {
|
||||||
|
const updated = await repos.updateRender(spaceId, renderId, patch);
|
||||||
|
if (!updated) {
|
||||||
|
input.abort();
|
||||||
|
throw new RenderCanceled();
|
||||||
|
}
|
||||||
|
return updated;
|
||||||
|
};
|
||||||
|
const checkCanceled = () => {
|
||||||
|
if (signal.aborted) throw new RenderCanceled();
|
||||||
|
};
|
||||||
|
|
||||||
|
let plan = await repos.getPlan(spaceId, record.planId);
|
||||||
|
const project = await repos.getProjectRecord(spaceId, record.projectId);
|
||||||
|
const entry = (await deps.profiles()).profiles.get(record.profileId);
|
||||||
|
if (!entry) throw new MediaRenderFailure("The platform profile is no longer available.");
|
||||||
|
const profile = entry.profile;
|
||||||
|
const kit = project.brandKitId
|
||||||
|
? (await repos.getBrandKit(spaceId, project.brandKitId)).kit
|
||||||
|
: null;
|
||||||
|
const quality = record.quality as MediaRenderQuality;
|
||||||
|
|
||||||
|
// A retried job starts from a clean work directory.
|
||||||
|
const workDir = path.join(renderDir, "work");
|
||||||
|
await rm(renderDir, { recursive: true, force: true });
|
||||||
|
await mkdir(workDir, { recursive: true });
|
||||||
|
|
||||||
|
const fetchArtifact = async (artifactId: string, dir: string, baseName: string) => {
|
||||||
|
const row = await deps.prisma.artifact.findFirst({
|
||||||
|
where: { id: artifactId, spaceId },
|
||||||
|
select: { storageKey: true, name: true, mimeType: true },
|
||||||
|
});
|
||||||
|
if (!row) return null;
|
||||||
|
const bytes = await deps.artifacts.get(row.storageKey, context);
|
||||||
|
await mkdir(dir, { recursive: true });
|
||||||
|
const file = path.join(dir, `${baseName}${safeExtension(row.name, row.mimeType)}`);
|
||||||
|
await writeFile(file, bytes);
|
||||||
|
return file;
|
||||||
|
};
|
||||||
|
const brand = await loadBrandFiles(kit, path.join(workDir, "brand"), fetchArtifact);
|
||||||
|
const audioFiles = new Map<string, string>();
|
||||||
|
|
||||||
|
const alternatives = await alternativeSegments(deps, spaceId, project, plan.edl);
|
||||||
|
let round = record.autoFixRound;
|
||||||
|
let edl = plan.edl;
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
checkCanceled();
|
||||||
|
const roundDir = path.join(workDir, `round-${round}`);
|
||||||
|
await mkdir(roundDir, { recursive: true });
|
||||||
|
|
||||||
|
// Fail closed: no redacted sources, no render.
|
||||||
|
let prepared: MediaPreparedSources;
|
||||||
|
try {
|
||||||
|
prepared = await privacy.prepareSources({
|
||||||
|
actor,
|
||||||
|
projectId: project.id,
|
||||||
|
edl,
|
||||||
|
quality,
|
||||||
|
workDir: roundDir,
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
checkCanceled();
|
||||||
|
getLogger().warn("media render sources not prepared", {
|
||||||
|
renderId,
|
||||||
|
reason: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
throw new MediaRenderFailure(
|
||||||
|
error instanceof UnsupportedMediaOperationError
|
||||||
|
? `Anonymization needs ${error.operation}, which is not available. Nothing was rendered.`
|
||||||
|
: "Anonymization could not be applied. Nothing was rendered.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
checkCanceled();
|
||||||
|
for (const clip of edl.tracks.video) {
|
||||||
|
if (!prepared.sources.has(clip.assetId)) {
|
||||||
|
throw new MediaRenderFailure(
|
||||||
|
"Anonymization did not prepare every clip. Nothing was rendered.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const clip of edl.tracks.audio) {
|
||||||
|
if (audioFiles.has(clip.artifactId)) continue;
|
||||||
|
const file = await fetchArtifact(
|
||||||
|
clip.artifactId,
|
||||||
|
path.join(workDir, "audio"),
|
||||||
|
`audio-${audioFiles.size + 1}`,
|
||||||
|
);
|
||||||
|
if (!file) throw new MediaRenderFailure("An audio file of the plan is no longer available.");
|
||||||
|
audioFiles.set(clip.artifactId, file);
|
||||||
|
}
|
||||||
|
|
||||||
|
const motionClips = await renderMotionClips(deps, edl, kit, brand, roundDir, signal);
|
||||||
|
const { transcripts, subjects } = await analysisInputs(deps, spaceId, edl);
|
||||||
|
const compile = (videoEncoder: string) =>
|
||||||
|
compileRenderPlan({
|
||||||
|
edl,
|
||||||
|
profile,
|
||||||
|
brandKit: kit,
|
||||||
|
quality,
|
||||||
|
workDir: roundDir,
|
||||||
|
sources: prepared.sources,
|
||||||
|
transcripts,
|
||||||
|
subjects,
|
||||||
|
motionClips,
|
||||||
|
files: {
|
||||||
|
lutPath: brand.lut,
|
||||||
|
watermarkPath: brand.watermark,
|
||||||
|
fonts: brand.fonts,
|
||||||
|
audio: audioFiles,
|
||||||
|
logoPath: brand.logos[0] ?? null,
|
||||||
|
},
|
||||||
|
videoEncoder,
|
||||||
|
});
|
||||||
|
const renderPlan = await renderWithFallback(input, profile, quality, compile, update);
|
||||||
|
|
||||||
|
const video = renderPlan.outputs.find((output) => output.kind === "video");
|
||||||
|
if (!video) throw new Error("render plan has no video output");
|
||||||
|
const ffprobe = await deps.renderer.probe(video.path, { measureLoudness: true, signal });
|
||||||
|
// A render whose metadata could not be removed is failed, not gated.
|
||||||
|
if (findMetadataLeaks(ffprobe.tags).length > 0) {
|
||||||
|
throw new MediaRenderFailure("The export still carried source metadata.");
|
||||||
|
}
|
||||||
|
for (const output of renderPlan.outputs) {
|
||||||
|
if (output.kind !== "poster") continue;
|
||||||
|
const poster = await deps.renderer.probe(output.path, { signal });
|
||||||
|
if (findMetadataLeaks(poster.tags).length > 0) {
|
||||||
|
throw new MediaRenderFailure("The poster still carried source metadata.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const gating = await update({ status: "gating", progress: 0.9, ffprobe });
|
||||||
|
await publishRender(deps, spaceId, gating);
|
||||||
|
|
||||||
|
const aiAlterations = [
|
||||||
|
...new Set<AiAlteration>([...edl.metadata.aiAlterations, ...prepared.aiAlterations]),
|
||||||
|
];
|
||||||
|
const rendered: Edl = {
|
||||||
|
...edl,
|
||||||
|
metadata: {
|
||||||
|
...edl.metadata,
|
||||||
|
aiAlterations,
|
||||||
|
aiAltered: edl.metadata.aiAltered || aiAlterations.length > 0,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const report = await privacy.gate({
|
||||||
|
actor,
|
||||||
|
projectId: project.id,
|
||||||
|
edl: rendered,
|
||||||
|
profile,
|
||||||
|
outputPath: video.path,
|
||||||
|
ffprobe,
|
||||||
|
copy: {
|
||||||
|
title: edl.metadata.title ?? null,
|
||||||
|
caption: edl.metadata.description ?? null,
|
||||||
|
tags: edl.metadata.tags,
|
||||||
|
},
|
||||||
|
// The publish bundle always carries the AI disclosure file and checklist
|
||||||
|
// entry when the profile requires a label and the output is altered.
|
||||||
|
bundleHasDisclosure: profile.disclosure.aiAlteredLabelRequired && aiAlterations.length > 0,
|
||||||
|
autoFixRound: round,
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
checkCanceled();
|
||||||
|
|
||||||
|
if (report.passed) {
|
||||||
|
await finish(input, renderPlan, report, plan.id, update, context);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fix =
|
||||||
|
round < MAX_AUTO_FIX_ROUNDS
|
||||||
|
? planAutoFix(edl, { ...report, autoFixRound: round }, alternatives)
|
||||||
|
: null;
|
||||||
|
if (!fix) {
|
||||||
|
await rm(renderDir, { recursive: true, force: true });
|
||||||
|
const blocked = await update({ status: "blocked", progress: 1, gateReport: report });
|
||||||
|
await publishRender(deps, spaceId, blocked);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const codes = [
|
||||||
|
...new Set(report.issues.filter((i) => i.severity === "block").map((i) => i.code)),
|
||||||
|
];
|
||||||
|
plan = await repos.createPlanVersion(spaceId, {
|
||||||
|
projectId: project.id,
|
||||||
|
profileId: plan.profileId,
|
||||||
|
edl: fix.edl,
|
||||||
|
createdBy: "agent",
|
||||||
|
rationale: `Auto-fix ${round + 1}/${MAX_AUTO_FIX_ROUNDS} after the gate blocked (${codes.join(
|
||||||
|
", ",
|
||||||
|
)}): ${fix.changes.join("; ")}`,
|
||||||
|
});
|
||||||
|
round += 1;
|
||||||
|
edl = plan.edl;
|
||||||
|
await rm(roundDir, { recursive: true, force: true });
|
||||||
|
const retry = await update({
|
||||||
|
status: "rendering",
|
||||||
|
progress: 0,
|
||||||
|
autoFixRound: round,
|
||||||
|
gateReport: report,
|
||||||
|
ffprobe: null,
|
||||||
|
});
|
||||||
|
await publishRender(deps, spaceId, retry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadBrandFiles(
|
||||||
|
kit: BrandKit | null,
|
||||||
|
dir: string,
|
||||||
|
fetchArtifact: (artifactId: string, dir: string, baseName: string) => Promise<string | null>,
|
||||||
|
): Promise<BrandFiles> {
|
||||||
|
const files: BrandFiles = {
|
||||||
|
fonts: new Map(),
|
||||||
|
fontList: [],
|
||||||
|
logos: [],
|
||||||
|
watermark: null,
|
||||||
|
lut: null,
|
||||||
|
};
|
||||||
|
if (!kit) return files;
|
||||||
|
// Missing files are skipped here; the compiler fails only if the plan needs one.
|
||||||
|
const missing = (what: string) => getLogger().warn("brand kit file unavailable", { what });
|
||||||
|
for (const [index, font] of kit.fonts.entries()) {
|
||||||
|
const file = await fetchArtifact(font.artifactId, dir, `font-${index + 1}`);
|
||||||
|
if (!file) {
|
||||||
|
missing("font");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
files.fonts.set(font.artifactId, file);
|
||||||
|
files.fontList.push({ family: font.family, path: file });
|
||||||
|
}
|
||||||
|
for (const [index, logo] of kit.logoArtifactIds.entries()) {
|
||||||
|
const file = await fetchArtifact(logo, dir, `logo-${index + 1}`);
|
||||||
|
if (file) files.logos.push(file);
|
||||||
|
else missing("logo");
|
||||||
|
}
|
||||||
|
const watermarkId = kit.watermark?.artifactId;
|
||||||
|
if (watermarkId) {
|
||||||
|
files.watermark = await fetchArtifact(watermarkId, dir, "watermark");
|
||||||
|
if (!files.watermark) missing("watermark");
|
||||||
|
} else if (kit.watermark) {
|
||||||
|
files.watermark = files.logos[0] ?? null;
|
||||||
|
}
|
||||||
|
if (kit.lutArtifactId) {
|
||||||
|
files.lut = await fetchArtifact(kit.lutArtifactId, dir, "look");
|
||||||
|
if (!files.lut) missing("lut");
|
||||||
|
}
|
||||||
|
return files;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function renderMotionClips(
|
||||||
|
deps: MediaServiceDeps,
|
||||||
|
edl: Edl,
|
||||||
|
kit: BrandKit | null,
|
||||||
|
brand: BrandFiles,
|
||||||
|
dir: string,
|
||||||
|
signal: AbortSignal,
|
||||||
|
): Promise<Map<string, string>> {
|
||||||
|
const W = edl.output.width;
|
||||||
|
const H = edl.output.height;
|
||||||
|
const jobs: Array<{ key: string; template: NonNullable<Edl["intro"]>; w: number; h: number }> =
|
||||||
|
[];
|
||||||
|
if (edl.intro && edl.intro.durationMs > 0)
|
||||||
|
jobs.push({ key: "intro", template: edl.intro, w: W, h: H });
|
||||||
|
if (edl.outro && edl.outro.durationMs > 0)
|
||||||
|
jobs.push({ key: "outro", template: edl.outro, w: W, h: H });
|
||||||
|
for (const overlay of edl.overlays) {
|
||||||
|
if (!overlay.motion) continue;
|
||||||
|
const box = overlay.box;
|
||||||
|
jobs.push({
|
||||||
|
key: overlay.id,
|
||||||
|
template: overlay.motion,
|
||||||
|
w: box ? even(box.w * W) : W,
|
||||||
|
h: box ? even(box.h * H) : H,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const clips = new Map<string, string>();
|
||||||
|
if (jobs.length === 0) return clips;
|
||||||
|
if (!kit) throw new MediaRenderFailure("Motion templates need a brand kit on the project.");
|
||||||
|
for (const job of jobs) {
|
||||||
|
// Keys are SafeId values ([a-z0-9_-]), so they are safe file names.
|
||||||
|
const result = await deps.motion.render({
|
||||||
|
template: job.template,
|
||||||
|
brandKit: kit,
|
||||||
|
assets: { fonts: brand.fontList, logoPaths: brand.logos },
|
||||||
|
width: job.w,
|
||||||
|
height: job.h,
|
||||||
|
fps: edl.output.fps,
|
||||||
|
outPath: path.join(dir, `motion-${job.key}.mov`),
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
clips.set(job.key, result.path);
|
||||||
|
}
|
||||||
|
return clips;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function analysisInputs(deps: MediaServiceDeps, spaceId: string, edl: Edl) {
|
||||||
|
const transcripts = new Map<string, CaptionWord[]>();
|
||||||
|
const subjects = new Map<string, Subject[]>();
|
||||||
|
const assetIds = [...new Set(edl.tracks.video.map((clip) => clip.assetId))];
|
||||||
|
for (const assetId of assetIds) {
|
||||||
|
const [transcript, saliency, faces] = await Promise.all([
|
||||||
|
deps.repos.latestAnalysis(spaceId, assetId, "transcript"),
|
||||||
|
deps.repos.latestAnalysis(spaceId, assetId, "saliency"),
|
||||||
|
deps.repos.latestAnalysis(spaceId, assetId, "faces"),
|
||||||
|
]);
|
||||||
|
const words = transcriptWords(transcript?.data ?? null);
|
||||||
|
if (words.length > 0) transcripts.set(assetId, words);
|
||||||
|
const points = subjectPoints(saliency?.data ?? null, faces?.data ?? null);
|
||||||
|
if (points.length > 0) subjects.set(assetId, points);
|
||||||
|
}
|
||||||
|
return { transcripts, subjects };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Hardware encoders that failed a real render, per renderer, so later renders skip them. */
|
||||||
|
const failedEncoders = new WeakMap<object, Set<string>>();
|
||||||
|
|
||||||
|
export function softwareEncoder(profile: PlatformProfile): string {
|
||||||
|
return profile.video.codec === "h264" ? "libx264" : "libx265";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Final renders prefer a working hardware encoder of the profile's codec; drafts use software. */
|
||||||
|
export async function pickEncoder(
|
||||||
|
renderer: MediaServiceDeps["renderer"],
|
||||||
|
profile: PlatformProfile,
|
||||||
|
quality: MediaRenderQuality,
|
||||||
|
): Promise<string> {
|
||||||
|
const software = softwareEncoder(profile);
|
||||||
|
if (quality === "draft") return software;
|
||||||
|
const failed = failedEncoders.get(renderer);
|
||||||
|
const hardware = await renderer.hardwareEncoders();
|
||||||
|
return (
|
||||||
|
hardware.find(
|
||||||
|
(encoder) => encoder.startsWith(`${profile.video.codec}_`) && !failed?.has(encoder),
|
||||||
|
) ?? software
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run the plan with the preferred encoder. A hardware encoder that fails
|
||||||
|
* mid-render (driver, device, or session limits) is retried once in software
|
||||||
|
* and skipped by this process afterwards; software failures are final.
|
||||||
|
*/
|
||||||
|
async function renderWithFallback(
|
||||||
|
input: RunInput,
|
||||||
|
profile: PlatformProfile,
|
||||||
|
quality: MediaRenderQuality,
|
||||||
|
compile: (videoEncoder: string) => RenderPlan,
|
||||||
|
update: (patch: MediaRenderPatch) => Promise<MediaRender>,
|
||||||
|
): Promise<RenderPlan> {
|
||||||
|
const { deps, signal } = input;
|
||||||
|
const encoder = await pickEncoder(deps.renderer, profile, quality);
|
||||||
|
const plan = compile(encoder);
|
||||||
|
const software = softwareEncoder(profile);
|
||||||
|
try {
|
||||||
|
await executePlan(input, plan, update);
|
||||||
|
return plan;
|
||||||
|
} catch (error) {
|
||||||
|
if (encoder === software || signal.aborted || error instanceof RenderCanceled) throw error;
|
||||||
|
if (error instanceof MediaToolMissingError) throw error;
|
||||||
|
getLogger().warn("media hardware encoder failed; rendering in software", {
|
||||||
|
renderId: input.renderId,
|
||||||
|
encoder,
|
||||||
|
reason: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
const failed = failedEncoders.get(deps.renderer) ?? new Set<string>();
|
||||||
|
failed.add(encoder);
|
||||||
|
failedEncoders.set(deps.renderer, failed);
|
||||||
|
}
|
||||||
|
const fallback = compile(software);
|
||||||
|
await executePlan(input, fallback, update);
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Segments the auto-fix may swap in: the project's material, never paid-only. */
|
||||||
|
async function alternativeSegments(
|
||||||
|
deps: MediaServiceDeps,
|
||||||
|
spaceId: string,
|
||||||
|
project: { assetIds: string[]; collectionIds: string[] },
|
||||||
|
edl: Edl,
|
||||||
|
) {
|
||||||
|
const fromCollections = await deps.repos.collectionAssetIds(spaceId, project.collectionIds);
|
||||||
|
const assetIds = [
|
||||||
|
...new Set([
|
||||||
|
...project.assetIds,
|
||||||
|
...fromCollections,
|
||||||
|
...edl.tracks.video.map((clip) => clip.assetId),
|
||||||
|
]),
|
||||||
|
];
|
||||||
|
const segments = await deps.repos.listSegments(spaceId, assetIds);
|
||||||
|
return segments
|
||||||
|
.filter((segment) => !segment.paidOnly)
|
||||||
|
.map((segment) => ({
|
||||||
|
segmentId: segment.id,
|
||||||
|
assetId: segment.assetId,
|
||||||
|
startMs: segment.startMs,
|
||||||
|
endMs: segment.endMs,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function executePlan(
|
||||||
|
input: RunInput,
|
||||||
|
plan: RenderPlan,
|
||||||
|
update: (patch: MediaRenderPatch) => Promise<MediaRender>,
|
||||||
|
) {
|
||||||
|
const { deps, spaceId, signal } = input;
|
||||||
|
let lastWrite = Number.NEGATIVE_INFINITY;
|
||||||
|
let writing: Promise<void> | null = null;
|
||||||
|
let failure: unknown = null;
|
||||||
|
const onProgress = (progress: MediaRenderProgress) => {
|
||||||
|
const now = deps.now().getTime();
|
||||||
|
if (writing || now - lastWrite < PROGRESS_INTERVAL_MS) return;
|
||||||
|
lastWrite = now;
|
||||||
|
writing = update({ progress: progress.progress * RENDER_SHARE })
|
||||||
|
.then((render) => publishRender(deps, spaceId, render))
|
||||||
|
.catch((error: unknown) => {
|
||||||
|
failure ??= error;
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
writing = null;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await deps.renderer.render(plan, { signal, onProgress });
|
||||||
|
} finally {
|
||||||
|
await writing;
|
||||||
|
}
|
||||||
|
if (failure) throw failure;
|
||||||
|
if (signal.aborted) throw new RenderCanceled();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function finish(
|
||||||
|
input: RunInput,
|
||||||
|
plan: RenderPlan,
|
||||||
|
report: GateReport,
|
||||||
|
planId: string,
|
||||||
|
update: (patch: MediaRenderPatch) => Promise<MediaRender>,
|
||||||
|
context: AdapterContext,
|
||||||
|
) {
|
||||||
|
const { deps, spaceId, renderId, renderDir } = input;
|
||||||
|
const outputs: Array<MediaRenderOutput & { path: string; planId: string }> = [];
|
||||||
|
for (const output of plan.outputs) {
|
||||||
|
const fileName = OUTPUT_FILES[output.kind];
|
||||||
|
const relative = `${spaceId}/renders/${renderId}/${fileName}`;
|
||||||
|
const destination = await resolveInside(deps.mediaRoot, relative);
|
||||||
|
await rename(output.path, destination);
|
||||||
|
const { size } = await stat(destination);
|
||||||
|
let artifactId: string | null = null;
|
||||||
|
if (output.kind !== "video") {
|
||||||
|
artifactId = await registerArtifact(input, context, {
|
||||||
|
name: fileName,
|
||||||
|
mimeType: OUTPUT_MIME[output.kind],
|
||||||
|
file: destination,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
outputs.push({
|
||||||
|
kind: output.kind,
|
||||||
|
artifactId,
|
||||||
|
fileName,
|
||||||
|
sizeBytes: size,
|
||||||
|
path: relative,
|
||||||
|
planId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const done = await update({
|
||||||
|
status: "done",
|
||||||
|
progress: 1,
|
||||||
|
outputs,
|
||||||
|
gateReport: report,
|
||||||
|
error: null,
|
||||||
|
});
|
||||||
|
await rm(path.join(renderDir, "work"), { recursive: true, force: true });
|
||||||
|
await publishRender(deps, spaceId, done);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function registerArtifact(
|
||||||
|
input: RunInput,
|
||||||
|
context: AdapterContext,
|
||||||
|
file: { name: string; mimeType: string; file: string },
|
||||||
|
): Promise<string> {
|
||||||
|
const { deps, spaceId } = input;
|
||||||
|
const bytes = new Uint8Array(await readFile(file.file));
|
||||||
|
const stored = await deps.artifacts.put(
|
||||||
|
{ name: file.name, mimeType: file.mimeType, bytes },
|
||||||
|
context,
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
const row = await deps.prisma.artifact.create({
|
||||||
|
data: {
|
||||||
|
spaceId,
|
||||||
|
userId: context.userId,
|
||||||
|
name: file.name,
|
||||||
|
mimeType: file.mimeType,
|
||||||
|
size: bytes.byteLength,
|
||||||
|
hash: createHash("sha256").update(bytes).digest("hex"),
|
||||||
|
storageKey: stored.id,
|
||||||
|
},
|
||||||
|
select: { id: true },
|
||||||
|
});
|
||||||
|
input.createdArtifacts.push({ id: row.id, storageKey: stored.id });
|
||||||
|
return row.id;
|
||||||
|
} catch (error) {
|
||||||
|
await deps.artifacts.remove(stored.id, context).catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
208
packages/testkit/src/fixtures/media/synthetic-media.ts
Normal file
208
packages/testkit/src/fixtures/media/synthetic-media.ts
Normal file
@ -0,0 +1,208 @@
|
|||||||
|
import { execFile, execFileSync } from "node:child_process";
|
||||||
|
import { mkdir } from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import { promisify } from "node:util";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Synthetic media fixtures for the render integration test, generated with
|
||||||
|
* ffmpeg lavfi sources at test time (nothing binary is checked in). No real
|
||||||
|
* people, faces, places, or devices: the "face" is a flat generated disc, the
|
||||||
|
* picture is testsrc2, the sound is a sine tone, and the device tags are
|
||||||
|
* placeholders that every export must drop.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const run = promisify(execFile);
|
||||||
|
|
||||||
|
export function hasFfmpeg(): boolean {
|
||||||
|
try {
|
||||||
|
execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });
|
||||||
|
execFileSync("ffprobe", ["-version"], { stdio: "ignore" });
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ffmpeg(args: string[]): Promise<void> {
|
||||||
|
await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...args]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Tags a phone writes (GPS, make, model, capture time, serial in a handler name). */
|
||||||
|
export const PHONE_TAGS: Record<string, string> = {
|
||||||
|
creation_time: "2020-01-01T10:00:00Z",
|
||||||
|
location: "+00.0000+000.0000/",
|
||||||
|
"com.apple.quicktime.location.ISO6709": "+00.0000+000.0000/",
|
||||||
|
make: "ExampleCam",
|
||||||
|
model: "ExampleCam One",
|
||||||
|
"com.apple.quicktime.software": "ExampleCam OS 1.0",
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface SyntheticClipOptions {
|
||||||
|
seconds: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
fps: number;
|
||||||
|
/** Sine level in dBFS; quiet on purpose so loudness normalization has work to do. */
|
||||||
|
volumeDb: number;
|
||||||
|
/**
|
||||||
|
* Draw a skin-toned disc ("face") centred at this normalized point with this
|
||||||
|
* radius relative to the height. The media emulator reports faces for files
|
||||||
|
* named with `__faces-1__`; the disc sits where its first face box lies.
|
||||||
|
*/
|
||||||
|
face?: { cx: number; cy: number; radius: number };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** testsrc2 + sine, H.264/AAC MP4 with phone-style tags. */
|
||||||
|
export async function writeSyntheticClip(file: string, options: SyntheticClipOptions) {
|
||||||
|
await mkdir(path.dirname(file), { recursive: true });
|
||||||
|
const { seconds, width, height, fps } = options;
|
||||||
|
const inputs = [
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
`testsrc2=size=${width}x${height}:rate=${fps}:duration=${seconds}`,
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
`sine=frequency=330:sample_rate=48000:duration=${seconds}`,
|
||||||
|
];
|
||||||
|
let video = "[0:v]null[v]";
|
||||||
|
if (options.face) {
|
||||||
|
const r = Math.round(options.face.radius * height);
|
||||||
|
const size = r * 2;
|
||||||
|
const x = Math.round(options.face.cx * width - r);
|
||||||
|
const y = Math.round(options.face.cy * height - r);
|
||||||
|
inputs.push("-f", "lavfi", "-i", `color=c=black:size=${size}x${size}:rate=${fps}`);
|
||||||
|
video = [
|
||||||
|
`[2:v]format=rgba,geq=r=230:g=190:b=160:a='if(lte(hypot(X-${r},Y-${r}),${r}),255,0)'[disc]`,
|
||||||
|
`[0:v][disc]overlay=x=${x}:y=${y}:shortest=1[v]`,
|
||||||
|
].join(";");
|
||||||
|
}
|
||||||
|
const tags = Object.entries(PHONE_TAGS).flatMap(([key, value]) => [
|
||||||
|
"-metadata",
|
||||||
|
`${key}=${value}`,
|
||||||
|
]);
|
||||||
|
await ffmpeg([
|
||||||
|
...inputs,
|
||||||
|
"-filter_complex",
|
||||||
|
`${video};[1:a]volume=${options.volumeDb}dB[a]`,
|
||||||
|
"-map",
|
||||||
|
"[v]",
|
||||||
|
"-map",
|
||||||
|
"[a]",
|
||||||
|
"-c:v",
|
||||||
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"ultrafast",
|
||||||
|
"-pix_fmt",
|
||||||
|
"yuv420p",
|
||||||
|
"-c:a",
|
||||||
|
"aac",
|
||||||
|
"-t",
|
||||||
|
String(seconds),
|
||||||
|
"-movflags",
|
||||||
|
"+use_metadata_tags",
|
||||||
|
...tags,
|
||||||
|
"-metadata:s:v",
|
||||||
|
"handler_name=ExampleCam serial 0000",
|
||||||
|
file,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Sine music bed as WAV. */
|
||||||
|
export async function writeMusicBed(file: string, seconds: number): Promise<void> {
|
||||||
|
await mkdir(path.dirname(file), { recursive: true });
|
||||||
|
await ffmpeg([
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
`sine=frequency=220:sample_rate=48000:duration=${seconds}`,
|
||||||
|
"-af",
|
||||||
|
"volume=-18dB",
|
||||||
|
file,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Flat square PNG used as the brand logo and watermark. */
|
||||||
|
export async function writeLogo(file: string): Promise<void> {
|
||||||
|
await mkdir(path.dirname(file), { recursive: true });
|
||||||
|
await ffmpeg(["-f", "lavfi", "-i", "color=c=white:size=96x96", "-frames:v", "1", file]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Decode one frame as 8-bit grayscale at the given size. */
|
||||||
|
export async function grayFrame(
|
||||||
|
file: string,
|
||||||
|
tMs: number,
|
||||||
|
width: number,
|
||||||
|
height: number,
|
||||||
|
): Promise<Uint8Array> {
|
||||||
|
const { stdout } = await run(
|
||||||
|
"ffmpeg",
|
||||||
|
[
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-ss",
|
||||||
|
(tMs / 1000).toFixed(3),
|
||||||
|
"-i",
|
||||||
|
file,
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-vf",
|
||||||
|
`scale=${width}:${height}:flags=area`,
|
||||||
|
"-f",
|
||||||
|
"rawvideo",
|
||||||
|
"-pix_fmt",
|
||||||
|
"gray",
|
||||||
|
"-",
|
||||||
|
],
|
||||||
|
{ encoding: "buffer", maxBuffer: 64 * 1024 * 1024 },
|
||||||
|
);
|
||||||
|
if (stdout.length !== width * height) throw new Error("unexpected frame size");
|
||||||
|
return new Uint8Array(stdout);
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PixelBox {
|
||||||
|
x0: number;
|
||||||
|
y0: number;
|
||||||
|
x1: number;
|
||||||
|
y1: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Variance of the pixels inside a box. */
|
||||||
|
export function variance(frame: Uint8Array, width: number, box: PixelBox): number {
|
||||||
|
let sum = 0;
|
||||||
|
let squares = 0;
|
||||||
|
let count = 0;
|
||||||
|
for (let y = box.y0; y < box.y1; y++) {
|
||||||
|
for (let x = box.x0; x < box.x1; x++) {
|
||||||
|
const value = frame[y * width + x]!;
|
||||||
|
sum += value;
|
||||||
|
squares += value * value;
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const mean = sum / count;
|
||||||
|
return squares / count - mean * mean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Mean absolute difference of two frames outside a box. */
|
||||||
|
export function meanDifferenceOutside(
|
||||||
|
a: Uint8Array,
|
||||||
|
b: Uint8Array,
|
||||||
|
width: number,
|
||||||
|
height: number,
|
||||||
|
box: PixelBox,
|
||||||
|
): number {
|
||||||
|
let total = 0;
|
||||||
|
let count = 0;
|
||||||
|
for (let y = 0; y < height; y++) {
|
||||||
|
for (let x = 0; x < width; x++) {
|
||||||
|
if (x >= box.x0 && x < box.x1 && y >= box.y0 && y < box.y1) continue;
|
||||||
|
total += Math.abs(a[y * width + x]! - b[y * width + x]!);
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return total / count;
|
||||||
|
}
|
||||||
359
packages/testkit/src/media-render.docker.test.ts
Normal file
359
packages/testkit/src/media-render.docker.test.ts
Normal file
@ -0,0 +1,359 @@
|
|||||||
|
import path from "node:path";
|
||||||
|
import type { BoxKeyframe, Edl } from "@rakazo/contracts";
|
||||||
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import type { MediaPrivacyService } from "../../adapters/src/media-context.js";
|
||||||
|
import { EmulatorMediaInferenceProvider } from "../../adapters/src/media-emulator.js";
|
||||||
|
import { FfmpegMediaRenderer } from "../../adapters/src/media-ffmpeg-renderer.js";
|
||||||
|
import type { RenderHarness } from "../../adapters/src/media-render.fixtures.js";
|
||||||
|
import {
|
||||||
|
createFakePrivacy,
|
||||||
|
createRenderHarness,
|
||||||
|
renderTestKit,
|
||||||
|
renderTestProfile,
|
||||||
|
} from "../../adapters/src/media-render.fixtures.js";
|
||||||
|
import {
|
||||||
|
createMediaRenderHandler,
|
||||||
|
createMediaRenderService,
|
||||||
|
} from "../../adapters/src/media-render.js";
|
||||||
|
import { findMetadataLeaks } from "../../media/src/gate.js";
|
||||||
|
import type { PixelBox } from "./fixtures/media/synthetic-media.js";
|
||||||
|
import {
|
||||||
|
grayFrame,
|
||||||
|
hasFfmpeg,
|
||||||
|
meanDifferenceOutside,
|
||||||
|
variance,
|
||||||
|
writeLogo,
|
||||||
|
writeMusicBed,
|
||||||
|
writeSyntheticClip,
|
||||||
|
} from "./fixtures/media/synthetic-media.js";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Real render integration (spec 13, Docker job). Synthetic fixtures are made
|
||||||
|
* with ffmpeg lavfi; inference runs on the deterministic emulator; the render
|
||||||
|
* goes through the real service, job handler, compiler, and ffmpeg renderer,
|
||||||
|
* and every result is checked with ffprobe. Runs wherever ffmpeg is installed
|
||||||
|
* (the CI test job and the media Docker job) and skips cleanly without it.
|
||||||
|
*
|
||||||
|
* Redactions belong to the privacy slice. Its stand-in here applies EDL region
|
||||||
|
* redactions through the emulator's `redact` (the same MediaInferenceProvider
|
||||||
|
* call the privacy service uses) and refuses to return an unredacted source.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const FPS = 30;
|
||||||
|
const FRAME_MS = 1000 / FPS;
|
||||||
|
/** Grayscale variance a blurred region must stay under. */
|
||||||
|
const VARIANCE_THRESHOLD = 15;
|
||||||
|
/**
|
||||||
|
* The generated "face": a flat disc inside every box the media emulator can
|
||||||
|
* report as the first face of a `__faces-1__` file (boxes are seeded by the
|
||||||
|
* file, but always contain this point and radius).
|
||||||
|
*/
|
||||||
|
const FACE = { cx: 0.19, cy: 0.28, radius: 0.05 };
|
||||||
|
|
||||||
|
const vertical = renderTestProfile();
|
||||||
|
const landscape = renderTestProfile({
|
||||||
|
id: "example-landscape-test",
|
||||||
|
name: "Example landscape test",
|
||||||
|
video: {
|
||||||
|
aspectRatios: ["16:9"],
|
||||||
|
width: 640,
|
||||||
|
height: 360,
|
||||||
|
fps: FPS,
|
||||||
|
codec: "h264",
|
||||||
|
maxBitrateKbps: 6000,
|
||||||
|
maxFileSizeMb: 500,
|
||||||
|
},
|
||||||
|
safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 },
|
||||||
|
});
|
||||||
|
|
||||||
|
const motion = {
|
||||||
|
id: "none",
|
||||||
|
templates: () => [],
|
||||||
|
render: async (): Promise<never> => {
|
||||||
|
throw new Error("no motion templates in this plan");
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Privacy stand-in: region redactions through the inference provider, fail closed. */
|
||||||
|
function emulatorPrivacy(
|
||||||
|
harness: RenderHarness,
|
||||||
|
inference: EmulatorMediaInferenceProvider,
|
||||||
|
): MediaPrivacyService {
|
||||||
|
const base = createFakePrivacy(harness.sources);
|
||||||
|
return {
|
||||||
|
...base,
|
||||||
|
async prepareSources(input) {
|
||||||
|
const sources = new Map();
|
||||||
|
for (const clip of input.edl.tracks.video) {
|
||||||
|
const source = harness.sources.get(clip.assetId);
|
||||||
|
if (!source) throw new Error(`no source for ${clip.assetId}`);
|
||||||
|
const regions = input.edl.redactions.regions.filter((r) => r.clipId === clip.id);
|
||||||
|
if (regions.length === 0) {
|
||||||
|
sources.set(clip.assetId, source);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const outPath = path.join(input.workDir, `redacted-${clip.id}.mp4`);
|
||||||
|
const result = await inference.redact({
|
||||||
|
path: source.path,
|
||||||
|
outPath,
|
||||||
|
redactions: regions.map((region) => ({
|
||||||
|
keyframes: region.keyframes,
|
||||||
|
mode: region.treatment === "cover" ? "cover" : "blur",
|
||||||
|
paddingPct: 10,
|
||||||
|
featherPct: 0,
|
||||||
|
})),
|
||||||
|
signal: input.signal,
|
||||||
|
});
|
||||||
|
if (result.framesProcessed <= 0) throw new Error("redaction processed no frames");
|
||||||
|
sources.set(clip.assetId, { ...source, path: result.outPath });
|
||||||
|
}
|
||||||
|
return { sources, aiAlterations: [], fallbacks: [] };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Fixture {
|
||||||
|
harness: RenderHarness;
|
||||||
|
inference: EmulatorMediaInferenceProvider;
|
||||||
|
renderer: FfmpegMediaRenderer;
|
||||||
|
assetId: string;
|
||||||
|
sourcePath: string;
|
||||||
|
musicArtifactId: string;
|
||||||
|
render: (edl: unknown, quality: "draft" | "final") => Promise<{ id: string; dir: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe.skipIf(!hasFfmpeg())("media render integration with real ffmpeg", () => {
|
||||||
|
let fx: Fixture;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
const renderer = new FfmpegMediaRenderer();
|
||||||
|
const harness = await createRenderHarness({
|
||||||
|
profiles: [vertical, landscape],
|
||||||
|
renderer,
|
||||||
|
motion,
|
||||||
|
});
|
||||||
|
const inference = new EmulatorMediaInferenceProvider({ root: harness.mediaRoot });
|
||||||
|
harness.deps.inference = inference;
|
||||||
|
const name = "lake__faces-1__speech__.mp4";
|
||||||
|
const assetId = await harness.createAsset(name);
|
||||||
|
const sourcePath = harness.assetPath(assetId, name);
|
||||||
|
await writeSyntheticClip(sourcePath, {
|
||||||
|
seconds: 6,
|
||||||
|
width: 1280,
|
||||||
|
height: 720,
|
||||||
|
fps: FPS,
|
||||||
|
volumeDb: -32,
|
||||||
|
face: FACE,
|
||||||
|
});
|
||||||
|
harness.setSource(assetId, {
|
||||||
|
path: sourcePath,
|
||||||
|
width: 1280,
|
||||||
|
height: 720,
|
||||||
|
durationMs: 6000,
|
||||||
|
fps: FPS,
|
||||||
|
hasAudio: true,
|
||||||
|
rotation: 0,
|
||||||
|
});
|
||||||
|
const timesMs = Array.from({ length: 13 }, (_, i) => i * 500);
|
||||||
|
await harness.addAnalysis(
|
||||||
|
assetId,
|
||||||
|
"transcript",
|
||||||
|
await inference.transcribe({ path: sourcePath, language: "en" }),
|
||||||
|
);
|
||||||
|
await harness.addAnalysis(
|
||||||
|
assetId,
|
||||||
|
"saliency",
|
||||||
|
await inference.saliency({ path: sourcePath, timesMs }),
|
||||||
|
);
|
||||||
|
|
||||||
|
const brandDir = path.join(harness.mediaRoot, "fixtures");
|
||||||
|
await writeLogo(path.join(brandDir, "logo.png"));
|
||||||
|
await writeMusicBed(path.join(brandDir, "bed.wav"), 6);
|
||||||
|
const logoArtifactId = await harness.addArtifact(
|
||||||
|
path.join(brandDir, "logo.png"),
|
||||||
|
"logo.png",
|
||||||
|
"image/png",
|
||||||
|
);
|
||||||
|
const musicArtifactId = await harness.addArtifact(
|
||||||
|
path.join(brandDir, "bed.wav"),
|
||||||
|
"bed.wav",
|
||||||
|
"audio/wav",
|
||||||
|
);
|
||||||
|
const kit = renderTestKit({ logoArtifactId, musicArtifactId });
|
||||||
|
|
||||||
|
const privacy = emulatorPrivacy(harness, inference);
|
||||||
|
const service = createMediaRenderService(harness.deps);
|
||||||
|
const handler = createMediaRenderHandler(harness.deps, privacy);
|
||||||
|
const actor = { spaceId: harness.spaceId, userId: harness.userId };
|
||||||
|
fx = {
|
||||||
|
harness,
|
||||||
|
inference,
|
||||||
|
renderer,
|
||||||
|
assetId,
|
||||||
|
sourcePath,
|
||||||
|
musicArtifactId,
|
||||||
|
async render(edl, quality) {
|
||||||
|
const projectId = await harness.createProject({
|
||||||
|
assetIds: [assetId],
|
||||||
|
kit,
|
||||||
|
profileId: (edl as Edl).output.profileId,
|
||||||
|
});
|
||||||
|
const plan = await harness.createPlan(projectId, edl);
|
||||||
|
const started = await service.start(actor, { planId: plan.id, quality });
|
||||||
|
await handler({ renderId: started.id });
|
||||||
|
const render = await harness.deps.repos.getRender(harness.spaceId, started.id);
|
||||||
|
expect(render.error).toBeNull();
|
||||||
|
expect(render.status).toBe("done");
|
||||||
|
return {
|
||||||
|
id: render.id,
|
||||||
|
dir: path.join(harness.mediaRoot, harness.spaceId, "renders", render.id),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await fx?.harness.cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders 16:9 to 9:16 with auto-reframe, captions, brand overlay and loudness", async () => {
|
||||||
|
const edl = {
|
||||||
|
edlVersion: 1,
|
||||||
|
output: {
|
||||||
|
profileId: vertical.id,
|
||||||
|
width: vertical.video.width,
|
||||||
|
height: vertical.video.height,
|
||||||
|
fps: FPS,
|
||||||
|
durationTargetMs: 4000,
|
||||||
|
durationHardMinMs: 1000,
|
||||||
|
durationHardMaxMs: 120_000,
|
||||||
|
},
|
||||||
|
tracks: {
|
||||||
|
video: [
|
||||||
|
{ id: "c1", assetId: fx.assetId, inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" },
|
||||||
|
{ id: "c2", assetId: fx.assetId, inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" },
|
||||||
|
],
|
||||||
|
audio: [
|
||||||
|
{
|
||||||
|
id: "bed",
|
||||||
|
kind: "music",
|
||||||
|
artifactId: fx.musicArtifactId,
|
||||||
|
inMs: 0,
|
||||||
|
outMs: 4000,
|
||||||
|
atMs: 0,
|
||||||
|
gainDb: -6,
|
||||||
|
fadeOutMs: 500,
|
||||||
|
duckUnderSpeech: true,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "cut", durationMs: 0 }],
|
||||||
|
captions: { mode: "both", presetId: "clean" },
|
||||||
|
overlays: [
|
||||||
|
{ id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 },
|
||||||
|
{ id: "lt", kind: "lower_third", atMs: 500, durationMs: 1500, text: "Example: 50% off" },
|
||||||
|
],
|
||||||
|
metadata: { title: "Example teaser" },
|
||||||
|
};
|
||||||
|
const render = await fx.render(edl, "final");
|
||||||
|
const video = path.join(render.dir, "video.mp4");
|
||||||
|
const probe = await fx.renderer.probe(video, { measureLoudness: true });
|
||||||
|
|
||||||
|
expect(Math.abs(probe.durationMs - 4000)).toBeLessThanOrEqual(FRAME_MS + 1);
|
||||||
|
expect(probe.width / probe.height).toBeCloseTo(9 / 16, 3);
|
||||||
|
expect(probe.width).toBe(vertical.video.width);
|
||||||
|
expect(probe.fps).toBe(FPS);
|
||||||
|
expect(probe.videoCodec).toBe("h264");
|
||||||
|
expect(Math.abs(probe.integratedLufs! - vertical.audio.targetLufs)).toBeLessThanOrEqual(1);
|
||||||
|
expect(probe.truePeakDb!).toBeLessThanOrEqual(vertical.audio.truePeakDb + 0.5);
|
||||||
|
|
||||||
|
expect(findMetadataLeaks(probe.tags)).toEqual([]);
|
||||||
|
const tagText = JSON.stringify(probe.tags).toLowerCase();
|
||||||
|
for (const leak of ["location", "iso6709", "make", "model", "creation_time", "examplecam"]) {
|
||||||
|
expect(tagText).not.toContain(leak);
|
||||||
|
}
|
||||||
|
expect(probe.tags["format.title"]).toBe("Example teaser");
|
||||||
|
|
||||||
|
const poster = await fx.renderer.probe(path.join(render.dir, "poster.jpg"));
|
||||||
|
expect(findMetadataLeaks(poster.tags)).toEqual([]);
|
||||||
|
|
||||||
|
const stored = await fx.harness.deps.repos.getRender(fx.harness.spaceId, render.id);
|
||||||
|
expect(stored.gateReport?.passed).toBe(true);
|
||||||
|
expect(stored.outputs.map((output) => output.fileName).sort()).toEqual([
|
||||||
|
"captions.srt",
|
||||||
|
"captions.vtt",
|
||||||
|
"poster.jpg",
|
||||||
|
"video.mp4",
|
||||||
|
]);
|
||||||
|
}, 240_000);
|
||||||
|
|
||||||
|
it("tracks the generated face disc with the emulator", async () => {
|
||||||
|
const tracks = await fx.inference.trackFaces({ path: fx.sourcePath, sampleFps: 2 });
|
||||||
|
expect(tracks.tracks).toHaveLength(1);
|
||||||
|
const disc = { x0: FACE.cx - FACE.radius * (9 / 16), x1: FACE.cx + FACE.radius * (9 / 16) };
|
||||||
|
for (const { box } of tracks.tracks[0]!.keyframes) {
|
||||||
|
expect(box.x).toBeLessThanOrEqual(disc.x0);
|
||||||
|
expect(box.x + box.w).toBeGreaterThanOrEqual(disc.x1);
|
||||||
|
expect(box.y).toBeLessThanOrEqual(FACE.cy - FACE.radius);
|
||||||
|
expect(box.y + box.h).toBeGreaterThanOrEqual(FACE.cy + FACE.radius);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("blurs a redaction region below the variance threshold and leaves the rest unchanged", async () => {
|
||||||
|
// A static region over testsrc2's fine checkerboard, held for the whole clip.
|
||||||
|
const box = { x: 0.68, y: 0.64, w: 0.14, h: 0.2 };
|
||||||
|
const keyframes: BoxKeyframe[] = [
|
||||||
|
{ tMs: 0, box },
|
||||||
|
{ tMs: 3000, box },
|
||||||
|
];
|
||||||
|
const edl = (regions: unknown[]) => ({
|
||||||
|
edlVersion: 1,
|
||||||
|
output: {
|
||||||
|
profileId: landscape.id,
|
||||||
|
width: landscape.video.width,
|
||||||
|
height: landscape.video.height,
|
||||||
|
fps: FPS,
|
||||||
|
durationTargetMs: 3000,
|
||||||
|
durationHardMinMs: 1000,
|
||||||
|
durationHardMaxMs: 120_000,
|
||||||
|
},
|
||||||
|
tracks: {
|
||||||
|
video: [{ id: "c1", assetId: fx.assetId, inMs: 0, outMs: 3000, atMs: 0, reframe: "none" }],
|
||||||
|
},
|
||||||
|
redactions: { regions },
|
||||||
|
metadata: {},
|
||||||
|
});
|
||||||
|
const plain = await fx.render(edl([]), "draft");
|
||||||
|
const redacted = await fx.render(
|
||||||
|
edl([{ id: "region", clipId: "c1", keyframes, treatment: "blur" }]),
|
||||||
|
"draft",
|
||||||
|
);
|
||||||
|
|
||||||
|
const W = landscape.video.width;
|
||||||
|
const H = landscape.video.height;
|
||||||
|
const inner: PixelBox = {
|
||||||
|
x0: Math.ceil((box.x + box.w * 0.15) * W),
|
||||||
|
y0: Math.ceil((box.y + box.h * 0.15) * H),
|
||||||
|
x1: Math.floor((box.x + box.w * 0.85) * W),
|
||||||
|
y1: Math.floor((box.y + box.h * 0.85) * H),
|
||||||
|
};
|
||||||
|
// Padding (10 %) and blur bleed at the edge stay out of the comparison.
|
||||||
|
const margin = 0.04;
|
||||||
|
const outer: PixelBox = {
|
||||||
|
x0: Math.floor((box.x - margin) * W),
|
||||||
|
y0: Math.floor((box.y - margin) * H),
|
||||||
|
x1: Math.ceil((box.x + box.w + margin) * W),
|
||||||
|
y1: Math.ceil((box.y + box.h + margin) * H),
|
||||||
|
};
|
||||||
|
for (const tMs of [500, 1500, 2500]) {
|
||||||
|
const before = await grayFrame(path.join(plain.dir, "video.mp4"), tMs, W, H);
|
||||||
|
const after = await grayFrame(path.join(redacted.dir, "video.mp4"), tMs, W, H);
|
||||||
|
const original = variance(before, W, inner);
|
||||||
|
const blurred = variance(after, W, inner);
|
||||||
|
const outside = meanDifferenceOutside(before, after, W, H, outer);
|
||||||
|
expect(original).toBeGreaterThan(VARIANCE_THRESHOLD * 4);
|
||||||
|
expect(blurred).toBeLessThan(VARIANCE_THRESHOLD);
|
||||||
|
expect(outside).toBeLessThan(1.5);
|
||||||
|
}
|
||||||
|
}, 240_000);
|
||||||
|
});
|
||||||
Loading…
x
Reference in New Issue
Block a user