From 3ebe0fe47bc99b2fc21fb97e2cbeb74f12886615 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:44:58 +0000 Subject: [PATCH] feat(media): ffmpeg renderer, render job and real render integration test Fail closed when anonymization cannot prepare every clip, retry a failing hardware encoder in software, verify each ffmpeg output exists, and fail renders whose video or poster still carries source metadata. Cover parsing, encoder detection, two-pass loudness, abort and cleanup, the render job (auto-fix, cancel, outputs) on memory and Postgres, and a synthetic 16:9 to 9:16 render plus a region blur check with ffprobe. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w --- .../src/media-ffmpeg-renderer.test.ts | 584 ++++++++++++++++++ .../adapters/src/media-ffmpeg-renderer.ts | 7 + .../adapters/src/media-render.fixtures.ts | 82 ++- packages/adapters/src/media-render.test.ts | 456 ++++++++++++++ packages/adapters/src/media-render.ts | 151 +++-- .../src/fixtures/media/synthetic-media.ts | 208 +++++++ .../testkit/src/media-render.docker.test.ts | 359 +++++++++++ 7 files changed, 1806 insertions(+), 41 deletions(-) create mode 100644 packages/adapters/src/media-ffmpeg-renderer.test.ts create mode 100644 packages/adapters/src/media-render.test.ts create mode 100644 packages/testkit/src/fixtures/media/synthetic-media.ts create mode 100644 packages/testkit/src/media-render.docker.test.ts diff --git a/packages/adapters/src/media-ffmpeg-renderer.test.ts b/packages/adapters/src/media-ffmpeg-renderer.test.ts new file mode 100644 index 0000000..97db4e2 --- /dev/null +++ b/packages/adapters/src/media-ffmpeg-renderer.test.ts @@ -0,0 +1,584 @@ +import { + chmod, + mkdir, + mkdtemp, + readdir, + readFile, + realpath, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { MediaRenderProgress } from "@rakazo/adapter-kit"; +import type { RenderPlan } from "@rakazo/contracts"; +import { EdlSchema } from "@rakazo/contracts"; +import type { CompileInput } from "@rakazo/media"; +import { compileRenderPlan, findMetadataLeaks } from "@rakazo/media"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + FfmpegMediaRenderer, + findTwoPassLoudnorm, + parseEbur128Summary, + parseFrameRate, + parseLoudnormJson, + parseProbeReport, +} from "./media-ffmpeg-renderer.js"; +import { MediaToolMissingError } from "./media-folder-source-probe.js"; +import { hasMediaTools, renderTestProfile, writeSyntheticClip } from "./media-render.fixtures.js"; + +describe("ffmpeg output parsing", () => { + it("parses frame rates", () => { + expect(parseFrameRate("30000/1001")).toBe(29.97); + expect(parseFrameRate("25/1")).toBe(25); + expect(parseFrameRate("0/0")).toBe(0); + expect(parseFrameRate(undefined)).toBe(0); + expect(parseFrameRate("abc")).toBe(0); + }); + + it("flattens format and stream tags and falls back to the file size", () => { + const report = parseProbeReport( + { + streams: [ + { + codec_type: "video", + codec_name: "h264", + width: 640, + height: 360, + r_frame_rate: "30/1", + }, + { codec_type: "audio", codec_name: "aac", tags: { language: "und" } }, + ], + format: { duration: "2.5", tags: { title: "Example", location: "+00+000/" } }, + }, + 1234, + ); + expect(report).toEqual({ + durationMs: 2500, + width: 640, + height: 360, + fps: 30, + videoCodec: "h264", + audioCodec: "aac", + sizeBytes: 1234, + integratedLufs: null, + truePeakDb: null, + tags: { + "format.title": "Example", + "format.location": "+00+000/", + "stream.1.language": "und", + }, + }); + }); + + it("reads the ebur128 summary and treats the silence gate as no programme", () => { + const stderr = `[Parsed_ebur128_0 @ 0x1] Summary: + + Integrated loudness: + I: -14.2 LUFS + Threshold: -24.3 LUFS + + True peak: + Peak: -1.4 dBFS`; + expect(parseEbur128Summary(stderr)).toEqual({ integratedLufs: -14.2, truePeakDb: -1.4 }); + expect(parseEbur128Summary("Summary:\n I: -70.0 LUFS\n True peak:\n Peak: -inf dBFS")).toEqual({ + integratedLufs: null, + truePeakDb: null, + }); + expect(parseEbur128Summary("no summary")).toEqual({ integratedLufs: null, truePeakDb: null }); + }); + + it("finds exactly one linear loudnorm without measured values", () => { + const graph = "[a]anull[b];[b]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]"; + const site = findTwoPassLoudnorm(graph); + expect(site?.options).toBe("I=-14:TP=-1:LRA=11:linear=true"); + expect(graph.slice(site!.start, site!.end)).toBe(site!.options); + expect(findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true:measured_I=-20[b]")).toBeNull(); + expect(findTwoPassLoudnorm("[a]loudnorm=I=-14[b]")).toBeNull(); + expect( + findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true[b];[c]loudnorm=I=-14:linear=true[d]"), + ).toBeNull(); + expect(findTwoPassLoudnorm("[a]anull[b]")).toBeNull(); + }); + + it("parses the loudnorm measurement block", () => { + const stderr = `[Parsed_loudnorm_3 @ 0x55] +{ +\t"input_i" : "-31.20", +\t"input_tp" : "-20.11", +\t"input_lra" : "0.00", +\t"input_thresh" : "-41.20", +\t"output_i" : "-14.02", +\t"output_tp" : "-1.00", +\t"output_lra" : "0.00", +\t"output_thresh" : "-24.02", +\t"normalization_type" : "dynamic", +\t"target_offset" : "0.02" +}`; + expect(parseLoudnormJson(stderr)).toEqual({ + inputI: -31.2, + inputTp: -20.11, + inputLra: 0, + inputThresh: -41.2, + targetOffset: 0.02, + }); + expect(parseLoudnormJson(stderr.replace('"-31.20"', '"-inf"'))).toBeNull(); + expect(parseLoudnormJson("nothing")).toBeNull(); + }); +}); + +describe("hardware encoder detection", () => { + let dir: string; + beforeAll(async () => { + dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-hw-"))); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + /** A fake ffmpeg that lists encoders and succeeds only for the given test encodes. */ + async function fakeFfmpeg(name: string, working: string[]): Promise { + const file = path.join(dir, name); + const cases = working.map((encoder) => `*" ${encoder} "*) exit 0 ;;`).join("\n "); + await writeFile( + file, + `#!/bin/sh +case " $* " in + *" -encoders "*) + echo " V....D libx264 libx264" + echo " V....D h264_nvenc NVIDIA NVENC H.264 encoder" + echo " V....D hevc_nvenc NVIDIA NVENC hevc encoder" + echo " V....D h264_vaapi H.264/AVC (VAAPI)" + exit 0 ;; + ${cases} +esac +echo "No device available" >&2 +exit 1 +`, + ); + await chmod(file, 0o755); + return file; + } + + it("keeps only listed encoders whose one-frame test encode works", async () => { + const renderer = new FfmpegMediaRenderer({ + ffmpegPath: await fakeFfmpeg("ffmpeg-nvenc", ["h264_nvenc"]), + vaapiDevice: path.join(dir, "no-render-node"), + }); + expect(await renderer.hardwareEncoders()).toEqual(["h264_nvenc"]); + }); + + it("falls back to software when no hardware encoder works or ffmpeg is missing", async () => { + const broken = new FfmpegMediaRenderer({ ffmpegPath: await fakeFfmpeg("ffmpeg-none", []) }); + expect(await broken.hardwareEncoders()).toEqual([]); + const missing = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") }); + expect(await missing.hardwareEncoders()).toEqual([]); + }); + + it("skips VAAPI encoders without a render node", async () => { + const renderer = new FfmpegMediaRenderer({ + ffmpegPath: await fakeFfmpeg("ffmpeg-vaapi", ["h264_vaapi"]), + vaapiDevice: path.join(dir, "no-render-node"), + }); + expect(await renderer.hardwareEncoders()).toEqual([]); + }); + + it.skipIf(!hasMediaTools)("never reports an encoder this host cannot run", async () => { + const renderer = new FfmpegMediaRenderer(); + for (const encoder of await renderer.hardwareEncoders()) { + expect(encoder).toMatch(/^(h264|hevc)_(vaapi|videotoolbox|nvenc)$/); + } + }); +}); + +describe("plan safety", () => { + let dir: string; + beforeAll(async () => { + dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-plan-"))); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("refuses writes outside the work directory", async () => { + const renderer = new FfmpegMediaRenderer({ root: dir }); + const workDir = path.join(dir, "work"); + const plan: RenderPlan = { + quality: "draft", + workDir, + steps: [ + { kind: "write_file", id: "escape", path: path.join(dir, "outside.txt"), content: "x" }, + ], + outputs: [], + }; + await expect(renderer.render(plan, {})).rejects.toThrow("outside"); + await expect(stat(path.join(dir, "outside.txt"))).rejects.toThrow(); + }); + + it("refuses a work directory outside the media root", async () => { + const renderer = new FfmpegMediaRenderer({ root: path.join(dir, "root") }); + await mkdir(path.join(dir, "root"), { recursive: true }); + const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] }; + await expect(renderer.render(plan, {})).rejects.toThrow("outside"); + }); + + it("requires motion, inpaint and identity steps to be prepared first", async () => { + const renderer = new FfmpegMediaRenderer(); + const plan: RenderPlan = { + quality: "final", + workDir: path.join(dir, "motion"), + steps: [ + { + kind: "motion", + id: "intro", + template: { templateId: "intro-minimal", params: {}, durationMs: 1000 }, + width: 320, + height: 180, + fps: 30, + outPath: path.join(dir, "motion", "intro.mov"), + }, + ], + outputs: [], + }; + await expect(renderer.render(plan, {})).rejects.toThrow("was not prepared"); + }); + + it("reports a missing ffmpeg instead of pretending to render", async () => { + const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") }); + const out = path.join(dir, "missing", "out.mp4"); + const plan: RenderPlan = { + quality: "draft", + workDir: path.join(dir, "missing"), + steps: [{ kind: "ffmpeg", id: "render", argv: ["-version"], produces: [out], durationMs: 1 }], + outputs: [{ kind: "video", path: out }], + }; + await expect(renderer.render(plan, {})).rejects.toBeInstanceOf(MediaToolMissingError); + }); + + it("rejects an already aborted render without spawning anything", async () => { + const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") }); + const controller = new AbortController(); + controller.abort(); + const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] }; + await expect(renderer.render(plan, { signal: controller.signal })).rejects.toMatchObject({ + name: "AbortError", + }); + }); +}); + +describe("two-pass loudness", () => { + let dir: string; + let ffmpegPath: string; + let log: string; + const GRAPH = "[0:a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]"; + + beforeAll(async () => { + dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-loudnorm-"))); + log = path.join(dir, "calls.log"); + ffmpegPath = path.join(dir, "ffmpeg"); + // Logs each call; the measurement pass prints loudnorm's JSON, the render + // pass writes its output file (the last argument). + await writeFile( + ffmpegPath, + `#!/bin/sh +echo "$*" >> "${log}" +case " $* " in + *print_format=json*|*.measure.txt*) + printf '[Parsed_loudnorm_0 @ 0x1] \\n{\\n "input_i" : "-31.20",\\n "input_tp" : "-20.11",\\n "input_lra" : "0.00",\\n "input_thresh" : "-41.20",\\n "target_offset" : "0.02"\\n}\\n' >&2 + exit 0 ;; +esac +for last; do :; done +[ "$last" = "-" ] || echo rendered > "$last" +`, + ); + await chmod(ffmpegPath, 0o755); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + async function calls(): Promise { + return (await readFile(log, "utf8")).trim().split("\n"); + } + + it("measures first, then renders with the measured values", async () => { + await rm(log, { force: true }); + const workDir = path.join(dir, "inline"); + const out = path.join(workDir, "out.m4a"); + const plan: RenderPlan = { + quality: "final", + workDir, + steps: [ + { + kind: "ffmpeg", + id: "render", + argv: ["-i", "in.wav", "-filter_complex", GRAPH, "-map", "[aout]", "-t", "2", out], + produces: [out], + durationMs: 2000, + }, + ], + outputs: [], + }; + await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {}); + const [measure, render] = await calls(); + expect(measure).toContain("linear=true:print_format=json"); + expect(measure).toContain("-map [aout] -t 2 -f null -"); + expect(measure).not.toContain(out); + expect(render).toContain( + "linear=true:measured_I=-31.20:measured_TP=-20.11:measured_LRA=0.00:measured_thresh=-41.20:offset=0.02,aresample=48000", + ); + expect(render?.endsWith(out)).toBe(true); + }); + + it("rewrites a filtergraph script and removes the measurement script", async () => { + await rm(log, { force: true }); + const workDir = path.join(dir, "script"); + const script = path.join(workDir, "filtergraph.txt"); + const out = path.join(workDir, "out.m4a"); + const plan: RenderPlan = { + quality: "final", + workDir, + steps: [ + { kind: "write_file", id: "filtergraph", path: script, content: GRAPH }, + { + kind: "ffmpeg", + id: "render", + argv: ["-i", "in.wav", "-filter_complex_script", script, "-map", "[aout]", out], + produces: [out], + durationMs: 2000, + }, + ], + outputs: [], + }; + await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {}); + const [measure] = await calls(); + expect(measure).toContain(path.join(workDir, "filtergraph.measure.txt")); + expect(await readFile(script, "utf8")).toContain("measured_I=-31.20"); + expect((await readdir(workDir)).sort()).toEqual(["filtergraph.txt", "out.m4a"]); + }); + + it("fails a step whose output was never written and removes what it wrote", async () => { + const workDir = path.join(dir, "silent"); + const out = path.join(workDir, "out.m4a"); + const script = path.join(workDir, "notes.txt"); + const plan: RenderPlan = { + quality: "draft", + workDir, + steps: [ + { kind: "write_file", id: "notes", path: script, content: "x" }, + { + kind: "ffmpeg", + id: "render", + // The fake writes its last argument; the declared output stays missing. + argv: ["-i", "in.wav", path.join(workDir, "elsewhere.m4a")], + produces: [out], + durationMs: 1000, + }, + ], + outputs: [], + }; + await expect(new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {})).rejects.toThrow( + "wrote no out.m4a", + ); + await expect(stat(script)).rejects.toThrow(); + }); +}); + +describe.skipIf(!hasMediaTools)("real ffmpeg renders", () => { + let dir: string; + let source: string; + const renderer = () => new FfmpegMediaRenderer({ root: dir }); + const profile = renderTestProfile(); + + beforeAll(async () => { + dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-ffmpeg-"))); + source = path.join(dir, "assets", "source.mp4"); + // Deliberately quiet (-36 dBFS sine) so loudness normalization has real work to do. + await writeSyntheticClip(source, { seconds: 6, width: 320, height: 180, volumeDb: -36 }); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + function compile(workDir: string, overrides: Partial = {}): RenderPlan { + const edl = EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: profile.id, + width: 180, + height: 320, + fps: 30, + durationTargetMs: 4000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" }, + ], + }, + captions: { mode: "both" }, + overlays: [{ id: "t1", kind: "text", atMs: 500, durationMs: 1500, text: "Example: 50%" }], + metadata: { title: "Example teaser" }, + }); + return compileRenderPlan({ + edl, + profile, + brandKit: null, + quality: "draft", + workDir, + sources: new Map([ + [ + "asset-a", + { + path: source, + width: 320, + height: 180, + durationMs: 6000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + ]), + transcripts: new Map([ + [ + "asset-a", + [ + { startMs: 100, endMs: 600, text: "Hello" }, + { startMs: 650, endMs: 1200, text: "there." }, + { startMs: 3100, endMs: 3800, text: "Bye." }, + ], + ], + ]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.3, cy: 0.5, strength: 1 }, + { tMs: 5000, cx: 0.7, cy: 0.5, strength: 1 }, + ], + ], + ]), + motionClips: new Map(), + files: { lutPath: null, watermarkPath: null, fonts: new Map(), audio: new Map() }, + videoEncoder: "libx264", + ...overrides, + }); + } + + it("renders 16:9 to 9:16 with captions, overlay, two-pass loudness and clean metadata", async () => { + const workDir = path.join(dir, "render-ok"); + const plan = compile(workDir); + const progress: MediaRenderProgress[] = []; + await renderer().render(plan, { onProgress: (p) => progress.push(p) }); + + const video = plan.outputs.find((output) => output.kind === "video")!.path; + const report = await renderer().probe(video, { measureLoudness: true }); + expect(report.width).toBe(180); + expect(report.height).toBe(320); + expect(report.fps).toBe(30); + expect(report.videoCodec).toBe("h264"); + expect(report.audioCodec).toBe("aac"); + expect(Math.abs(report.durationMs - 4000)).toBeLessThanOrEqual(1000 / 30 + 1); + expect(report.integratedLufs).not.toBeNull(); + expect(Math.abs(report.integratedLufs! - profile.audio.targetLufs)).toBeLessThanOrEqual(1); + expect(report.truePeakDb!).toBeLessThanOrEqual(profile.audio.truePeakDb + 0.5); + expect(findMetadataLeaks(report.tags)).toEqual([]); + expect(report.tags["format.title"]).toBe("Example teaser"); + const keys = Object.keys(report.tags).join(" ").toLowerCase(); + for (const leak of ["location", "creation_time", "make", "model"]) { + expect(keys).not.toContain(leak); + } + expect(Object.values(report.tags).join(" ")).not.toContain("ExampleCam"); + + const poster = plan.outputs.find((output) => output.kind === "poster"); + if (poster) expect(findMetadataLeaks((await renderer().probe(poster.path)).tags)).toEqual([]); + expect(progress.length).toBeGreaterThan(0); + expect(progress.at(-1)!.progress).toBeCloseTo(1, 5); + for (let i = 1; i < progress.length; i++) { + expect(progress[i]!.progress).toBeGreaterThanOrEqual(progress[i - 1]!.progress - 1e-9); + } + // The measurement pass leaves nothing behind. + expect((await readdir(workDir)).sort()).toEqual( + ["captions.ass", "captions.srt", "captions.vtt", "output.mp4", "poster.jpg"].filter( + (name) => name !== "poster.jpg" || poster, + ), + ); + }); + + it("extracts an EXIF-free PNG frame", async () => { + const out = path.join(dir, "frames", "frame.png"); + await mkdir(path.dirname(out), { recursive: true }); + await renderer().extractFrame({ path: source, tMs: 1000, outPath: out, width: 161 }); + const report = await renderer().probe(out); + expect(report.width).toBe(160); + expect(report.videoCodec).toBe("png"); + expect(findMetadataLeaks(report.tags)).toEqual([]); + await expect( + renderer().extractFrame({ path: source, tMs: 1000, outPath: path.join(dir, "frame.jpg") }), + ).rejects.toThrow("PNG"); + await expect( + renderer().extractFrame({ path: source, tMs: 60_000, outPath: path.join(dir, "late.png") }), + ).rejects.toThrow(); + }); + + it("refuses to probe outside the media root", async () => { + await expect(renderer().probe("/etc/hostname")).rejects.toThrow("outside"); + await expect(renderer().probe("relative.mp4")).rejects.toThrow("absolute"); + }); + + it("aborts a running render, kills ffmpeg and removes partial outputs", async () => { + const workDir = path.join(dir, "render-abort"); + const long = await mkdtemp(path.join(dir, "long-")); + const longSource = path.join(long, "long.mp4"); + await writeSyntheticClip(longSource, { seconds: 30, width: 1280, height: 720 }); + const plan = compile(workDir, { + edl: EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: profile.id, + width: 1080, + height: 1920, + fps: 30, + durationTargetMs: 30_000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 30_000, atMs: 0, reframe: "none" }, + ], + }, + metadata: {}, + }), + sources: new Map([ + [ + "asset-a", + { + path: longSource, + width: 1280, + height: 720, + durationMs: 30_000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + ]), + }); + const controller = new AbortController(); + const started = Date.now(); + const rendering = renderer().render(plan, { + signal: controller.signal, + onProgress: (p) => { + if (p.progress > 0.01) controller.abort(); + }, + }); + await expect(rendering).rejects.toMatchObject({ name: "AbortError" }); + expect(Date.now() - started).toBeLessThan(25_000); + await expect(stat(path.join(workDir, "output.mp4"))).rejects.toThrow(); + }); +}); diff --git a/packages/adapters/src/media-ffmpeg-renderer.ts b/packages/adapters/src/media-ffmpeg-renderer.ts index 46b1487..b763a33 100644 --- a/packages/adapters/src/media-ffmpeg-renderer.ts +++ b/packages/adapters/src/media-ffmpeg-renderer.ts @@ -316,6 +316,13 @@ export class FfmpegMediaRenderer implements MediaRenderer { signal, report: (fraction) => report(step.id, fraction, weight(step)), }); + // Never report success for an output ffmpeg did not actually write. + for (const produced of step.produces) { + const file = await stat(produced).catch(() => null); + if (!file?.isFile() || file.size === 0) { + throw new Error(`step ${step.id} wrote no ${path.basename(produced)}`); + } + } done += weight(step); report(step.id, 0, 0); break; diff --git a/packages/adapters/src/media-render.fixtures.ts b/packages/adapters/src/media-render.fixtures.ts index 6926759..a7cd103 100644 --- a/packages/adapters/src/media-render.fixtures.ts +++ b/packages/adapters/src/media-render.fixtures.ts @@ -1,7 +1,9 @@ +import { execFile, execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { mkdir, mkdtemp, readFile, realpath, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { promisify } from "node:util"; import type { JobPublisher, LinkShortener, @@ -43,6 +45,70 @@ import type { MediaServiceDeps, } from "./media-context.js"; +const run = promisify(execFile); + +function onPath(binary: string): boolean { + try { + execFileSync(binary, ["-version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +/** Real-ffmpeg tests run when both tools are installed and skip otherwise. */ +export const hasMediaTools = onPath("ffmpeg") && onPath("ffprobe"); + +/** + * Synthetic clip: testsrc2 pattern plus a sine tone, H.264/AAC in MP4, with the + * kind of tags a phone writes (GPS, device, capture time) that every export + * must drop. No real people or places. + */ +export async function writeSyntheticClip( + file: string, + options: { seconds: number; width: number; height: number; fps?: number; volumeDb?: number }, +): Promise { + const fps = options.fps ?? 30; + await mkdir(path.dirname(file), { recursive: true }); + await run("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-nostdin", + "-y", + "-f", + "lavfi", + "-i", + `testsrc2=size=${options.width}x${options.height}:rate=${fps}:duration=${options.seconds}`, + "-f", + "lavfi", + "-i", + `sine=frequency=440:sample_rate=48000:duration=${options.seconds}`, + "-af", + `volume=${options.volumeDb ?? -20}dB`, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + "-metadata", + "creation_time=2020-01-01T10:00:00Z", + "-metadata", + "location=+00.0000+000.0000/", + "-metadata", + "make=ExampleCam", + "-metadata", + "model=ExampleCam One", + "-metadata:s:v", + "handler_name=ExampleCam serial 0000", + file, + ]); +} + /* * Test harness for the render service and job: the repos the render slice * reads, either in memory or backed by PostgreSQL (VERIFY_DATABASE), plus a @@ -134,7 +200,11 @@ export interface RenderHarness { addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise; addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise; addArtifact(file: string, name: string, mimeType: string): Promise; - createProject(input: { assetIds: string[]; kit: BrandKit | null; profileId: string }): Promise; + createProject(input: { + assetIds: string[]; + kit: BrandKit | null; + profileId: string; + }): Promise; createPlan(projectId: string, edl: unknown): Promise; planVersions(projectId: string, profileId: string): Promise; artifactCount(): Promise; @@ -271,7 +341,11 @@ interface Backend { createAsset(name: string): Promise; addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise; addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise; - createProject(input: { assetIds: string[]; kit: BrandKit | null; profileId: string }): Promise; + createProject(input: { + assetIds: string[]; + kit: BrandKit | null; + profileId: string; + }): Promise; cleanup(): Promise; } @@ -461,7 +535,9 @@ function memoryBackend(now: () => Date): Backend { }, async listPlanVersions(space: string, projectId: string, profileId: string) { return plans - .filter((p) => p.spaceId === space && p.projectId === projectId && p.profileId === profileId) + .filter( + (p) => p.spaceId === space && p.projectId === projectId && p.profileId === profileId, + ) .sort((a, b) => b.version - a.version) .map(mapMediaEditPlan); }, diff --git a/packages/adapters/src/media-render.test.ts b/packages/adapters/src/media-render.test.ts new file mode 100644 index 0000000..218b58a --- /dev/null +++ b/packages/adapters/src/media-render.test.ts @@ -0,0 +1,456 @@ +import { mkdir, readdir, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { MediaRenderer, MotionGraphicsRenderer } from "@rakazo/adapter-kit"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import type { FfprobeReport, GateIssue, MediaRender, RenderPlan } from "@rakazo/contracts"; +import { findMetadataLeaks } from "@rakazo/media"; +import { afterEach, describe, expect, it } from "vitest"; +import type { MediaPrivacyService } from "./media-context.js"; +import { FfmpegMediaRenderer } from "./media-ffmpeg-renderer.js"; +import type { FakePrivacy, RenderHarness } from "./media-render.fixtures.js"; +import { + createFakePrivacy, + createRenderHarness, + hasMediaTools, + renderTestProfile, + writeSyntheticClip, +} from "./media-render.fixtures.js"; +import { createMediaRenderHandler, createMediaRenderService, pickEncoder } from "./media-render.js"; + +const profile = renderTestProfile(); + +const motion: MotionGraphicsRenderer = { + id: "unused", + templates: () => [], + render: async () => { + throw new Error("no motion in these plans"); + }, +}; + +interface FakeRenderer extends MediaRenderer { + plans: RenderPlan[]; +} + +/** + * Executes write_file steps and writes placeholder outputs; probes report a + * clean, on-target file. `failEncoders` makes renders with those encoders fail. + */ +function fakeRenderer( + options: { + hardware?: string[]; + failEncoders?: string[]; + tags?: Record; + posterTags?: Record; + /** Resolves once a render started; the render then waits for its signal. */ + hold?: { started: () => void }; + } = {}, +): FakeRenderer { + const plans: RenderPlan[] = []; + let durationMs = 0; + return { + id: "fake", + plans, + hardwareEncoders: async () => options.hardware ?? [], + async render(plan, { signal, onProgress } = {}) { + plans.push(plan); + const step = plan.steps.find((s) => s.kind === "ffmpeg" && s.id === "render"); + if (step?.kind !== "ffmpeg") throw new Error("no render step"); + durationMs = step.durationMs; + const encoder = step.argv[step.argv.indexOf("-c:v") + 1]!; + if (options.failEncoders?.includes(encoder)) throw new Error(`${encoder}: device lost`); + if (options.hold) { + options.hold.started(); + await new Promise((_, reject) => { + signal?.addEventListener("abort", () => { + const error = new Error("render aborted"); + error.name = "AbortError"; + reject(error); + }); + }); + } + for (const s of plan.steps) { + if (s.kind === "write_file") { + await mkdir(path.dirname(s.path), { recursive: true }); + await writeFile(s.path, s.content); + } + if (s.kind === "ffmpeg") { + for (const file of s.produces) await writeFile(file, `rendered ${s.id}`); + } + } + onProgress?.({ stepId: "render", progress: 1 }); + }, + async probe(file): Promise { + const poster = file.endsWith(".jpg"); + return { + durationMs: poster ? 0 : durationMs, + width: profile.video.width, + height: profile.video.height, + fps: poster ? 0 : profile.video.fps, + videoCodec: poster ? "mjpeg" : "h264", + audioCodec: poster ? null : "aac", + sizeBytes: (await stat(file)).size, + integratedLufs: poster ? null : -14.2, + truePeakDb: poster ? null : -1.5, + tags: poster ? (options.posterTags ?? {}) : (options.tags ?? {}), + }; + }, + extractFrame: async () => { + throw new Error("not used"); + }, + }; +} + +function edlFor(assetId: string) { + return { + edlVersion: 1, + output: { + profileId: profile.id, + width: profile.video.width, + height: profile.video.height, + fps: 30, + durationTargetMs: 3000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [ + { id: "c1", assetId, inMs: 0, outMs: 1500, atMs: 0, reframe: "auto" }, + { id: "c2", assetId, inMs: 2000, outMs: 3500, atMs: 1500, reframe: "auto" }, + ], + }, + captions: { mode: "both" }, + overlays: [{ id: "t1", kind: "text", atMs: 200, durationMs: 1000, text: "Example" }], + metadata: { title: "Example teaser" }, + }; +} + +interface Setup { + harness: RenderHarness; + privacy: FakePrivacy; + renderer: MediaRenderer; + run: (planQuality?: "draft" | "final") => Promise; + renderDir: (renderId: string) => string; +} + +let cleanups: Array<() => Promise> = []; +afterEach(async () => { + for (const cleanup of cleanups) await cleanup(); + cleanups = []; +}); + +async function setup(options: { + renderer: MediaRenderer; + privacy?: (sources: RenderHarness["sources"]) => MediaPrivacyService; + failWith?: GateIssue; + realSource?: boolean; +}): Promise }> { + const harness = await createRenderHarness({ + profiles: [profile], + renderer: options.renderer, + motion, + }); + cleanups.push(() => harness.cleanup()); + const assetId = await harness.createAsset("source.mp4"); + const file = harness.assetPath(assetId, "source.mp4"); + if (options.realSource) { + await writeSyntheticClip(file, { seconds: 4, width: 320, height: 180, volumeDb: -30 }); + } else { + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, "synthetic"); + } + harness.setSource(assetId, { + path: file, + width: 320, + height: 180, + durationMs: 4000, + fps: 30, + hasAudio: true, + rotation: 0, + }); + await harness.addAnalysis(assetId, "transcript", { + language: "en", + words: [ + { startMs: 100, endMs: 500, text: "Hello", confidence: 0.9 }, + { startMs: 2100, endMs: 2600, text: "again.", confidence: 0.9 }, + ], + }); + await harness.addSegments(assetId, [ + { startMs: 0, endMs: 2000 }, + { startMs: 2000, endMs: 4000 }, + ]); + const projectId = await harness.createProject({ + assetIds: [assetId], + kit: null, + profileId: profile.id, + }); + const plan = await harness.createPlan(projectId, edlFor(assetId)); + const privacy = createFakePrivacy(harness.sources, { failWith: options.failWith }); + const service = createMediaRenderService(harness.deps); + const handler = createMediaRenderHandler( + harness.deps, + options.privacy ? options.privacy(harness.sources) : privacy, + { cancelPollMs: 50 }, + ); + const actor = { spaceId: harness.spaceId, userId: harness.userId }; + return { + harness, + privacy, + renderer: options.renderer, + service, + renderDir: (renderId) => path.join(harness.mediaRoot, harness.spaceId, "renders", renderId), + async run(quality = "draft") { + const render = await service.start(actor, { planId: plan.id, quality }); + await handler({ renderId: render.id }); + return harness.deps.repos.getRender(harness.spaceId, render.id); + }, + }; +} + +async function exists(file: string): Promise { + return stat(file).then( + () => true, + () => false, + ); +} + +describe("media render service", () => { + it("queues a render job and announces it", async () => { + const { harness, service } = await setup({ renderer: fakeRenderer() }); + expect(harness.backend).toBe( + process.env.VERIFY_DATABASE && process.env.DATABASE_URL ? "postgres" : "memory", + ); + const projectId = await harness.createProject({ + assetIds: [], + kit: null, + profileId: profile.id, + }); + const assetId = [...harness.sources.keys()][0]!; + const plan = await harness.createPlan(projectId, edlFor(assetId)); + const actor = { spaceId: harness.spaceId, userId: harness.userId }; + const render = await service.start(actor, { planId: plan.id, quality: "final" }); + expect(render).toMatchObject({ status: "queued", quality: "final", planId: plan.id }); + expect(harness.enqueued).toContain(`media.render:${JSON.stringify({ renderId: render.id })}`); + expect(harness.events).toContainEqual({ + type: "media.render", + renderId: render.id, + projectId, + status: "queued", + progress: 0, + }); + await expect( + service.start({ ...actor, spaceId: "space-other" }, { planId: plan.id, quality: "draft" }), + ).rejects.toThrow(); + }); + + it("renders, gates, stores outputs and serves only recorded files", async () => { + const renderer = fakeRenderer(); + const { harness, service, run, renderDir } = await setup({ renderer }); + const render = await run(); + expect(render.status).toBe("done"); + expect(render.progress).toBe(1); + expect(render.gateReport?.passed).toBe(true); + expect(render.ffprobe?.integratedLufs).toBe(-14.2); + expect(render.outputs.map((output) => output.fileName).sort()).toEqual([ + "captions.srt", + "captions.vtt", + "poster.jpg", + "video.mp4", + ]); + const video = render.outputs.find((output) => output.kind === "video")!; + expect(video.artifactId).toBeNull(); + expect(render.outputs.filter((output) => output.artifactId !== null)).toHaveLength(3); + expect(await harness.artifactCount()).toBe(3); + // Intermediates are gone; only the published outputs stay. + expect((await readdir(renderDir(render.id))).sort()).toEqual([ + "captions.srt", + "captions.vtt", + "poster.jpg", + "video.mp4", + ]); + const actor = { spaceId: harness.spaceId, userId: harness.userId }; + expect(await service.outputPath(actor, render.id, "video.mp4")).toBe( + path.join(renderDir(render.id), "video.mp4"), + ); + expect(await service.outputPath(actor, render.id, "../video.mp4")).toBeNull(); + expect(await service.outputPath(actor, render.id, "work")).toBeNull(); + await expect( + service.outputPath({ ...actor, spaceId: "space-other" }, render.id, "video.mp4"), + ).rejects.toThrow(); + const statuses = harness.events.flatMap((event) => + event.type === "media.render" && event.renderId === render.id ? [event.status] : [], + ); + expect(statuses[0]).toBe("queued"); + expect(statuses).toContain("gating"); + expect(statuses.at(-1)).toBe("done"); + }); + + it("refuses to render when redactions cannot be applied (fail closed)", async () => { + const renderer = fakeRenderer(); + let calls = 0; + const { harness, run, renderDir } = await setup({ + renderer, + privacy: (sources) => ({ + ...createFakePrivacy(sources), + async prepareSources() { + calls += 1; + throw new Error("face track lost for 800 ms"); + }, + }), + }); + const render = await run("final"); + expect(calls).toBe(1); + expect(render.status).toBe("failed"); + expect(render.error).toBe("Anonymization could not be applied. Nothing was rendered."); + expect(render.outputs).toEqual([]); + expect(renderer.plans).toHaveLength(0); + expect(await exists(renderDir(render.id))).toBe(false); + expect(await harness.artifactCount()).toBe(0); + }); + + it("names the missing capability and refuses when a clip was not prepared", async () => { + const unsupported = await setup({ + renderer: fakeRenderer(), + privacy: (sources) => ({ + ...createFakePrivacy(sources), + async prepareSources() { + throw new UnsupportedMediaOperationError("inpaintVideo", "no model"); + }, + }), + }); + expect((await unsupported.run()).error).toBe( + "Anonymization needs inpaintVideo, which is not available. Nothing was rendered.", + ); + + const renderer = fakeRenderer(); + const partial = await setup({ + renderer, + privacy: (sources) => ({ + ...createFakePrivacy(sources), + async prepareSources() { + return { sources: new Map(), aiAlterations: [], fallbacks: [] }; + }, + }), + }); + const render = await partial.run(); + expect(render.status).toBe("failed"); + expect(render.error).toBe("Anonymization did not prepare every clip. Nothing was rendered."); + expect(renderer.plans).toHaveLength(0); + }); + + it("falls back to software when a hardware encoder fails and skips it afterwards", async () => { + const renderer = fakeRenderer({ + hardware: ["hevc_vaapi", "h264_vaapi"], + failEncoders: ["h264_vaapi"], + }); + const { run } = await setup({ renderer }); + const encoders = () => + renderer.plans.map((plan) => { + const step = plan.steps.find((s) => s.kind === "ffmpeg" && s.id === "render"); + return step?.kind === "ffmpeg" ? step.argv[step.argv.indexOf("-c:v") + 1] : null; + }); + expect((await run("final")).status).toBe("done"); + expect(encoders()).toEqual(["h264_vaapi", "libx264"]); + expect((await run("final")).status).toBe("done"); + expect(encoders()).toEqual(["h264_vaapi", "libx264", "libx264"]); + }); + + it("uses software for drafts and a working hardware encoder for finals", async () => { + const renderer = fakeRenderer({ hardware: ["hevc_nvenc", "h264_nvenc"] }); + expect(await pickEncoder(renderer, profile, "draft")).toBe("libx264"); + expect(await pickEncoder(renderer, profile, "final")).toBe("h264_nvenc"); + expect(await pickEncoder(fakeRenderer(), profile, "final")).toBe("libx264"); + const hevc = renderTestProfile({ + video: { ...profile.video, codec: "hevc" }, + }); + expect(await pickEncoder(renderer, hevc, "final")).toBe("hevc_nvenc"); + expect(await pickEncoder(renderer, hevc, "draft")).toBe("libx265"); + }); + + it("fails a render whose video or poster still carries source metadata", async () => { + const leaky = await setup({ + renderer: fakeRenderer({ tags: { "format.location": "+00+000/" } }), + }); + const render = await leaky.run(); + expect(render.status).toBe("failed"); + expect(render.error).toBe("The export still carried source metadata."); + expect(await exists(leaky.renderDir(render.id))).toBe(false); + + const poster = await setup({ + renderer: fakeRenderer({ posterTags: { "format.make": "ExampleCam" } }), + }); + expect((await poster.run()).error).toBe("The poster still carried source metadata."); + }); + + it("auto-fixes a blocked gate at most three times, then blocks", async () => { + const renderer = fakeRenderer(); + const { harness, privacy, run, renderDir } = await setup({ + renderer, + failWith: { + check: "content", + severity: "block", + code: "classifier_violence", + message: "violence score above the profile threshold", + atMs: 0, + endMs: 400, + }, + }); + const render = await run(); + expect(render.status).toBe("blocked"); + expect(render.gateReport?.passed).toBe(false); + expect(privacy.gateCalls.map((call) => call.autoFixRound)).toEqual([0, 1, 2, 3]); + expect(renderer.plans).toHaveLength(4); + const record = await harness.deps.repos.getRenderRecord(harness.spaceId, render.id); + expect(record.autoFixRound).toBe(3); + const versions = await harness.planVersions(render.projectId, profile.id); + expect(versions).toHaveLength(4); + expect(versions[0]!.createdBy).toBe("agent"); + expect(versions[0]!.rationale).toContain("Auto-fix 3/3"); + expect(render.outputs).toEqual([]); + expect(await exists(renderDir(render.id))).toBe(false); + expect(await harness.artifactCount()).toBe(0); + }); + + it("cancels a running render, stops it and removes its files", async () => { + let started!: () => void; + const running = new Promise((resolve) => { + started = resolve; + }); + const renderer = fakeRenderer({ hold: { started: () => started() } }); + const { harness, service, run, renderDir } = await setup({ renderer }); + const result = run(); + await running; + const active = harness.events.findLast((event) => event.type === "media.render"); + const renderId = active?.type === "media.render" ? active.renderId : ""; + const canceled = await service.cancel( + { spaceId: harness.spaceId, userId: harness.userId }, + renderId, + ); + expect(canceled.status).toBe("canceled"); + const render = await result; + expect(render.status).toBe("canceled"); + expect(await exists(renderDir(renderId))).toBe(false); + await expect( + service.cancel({ spaceId: harness.spaceId, userId: harness.userId }, renderId), + ).rejects.toThrow(); + }); +}); + +describe.skipIf(!hasMediaTools)("media render job with real ffmpeg", () => { + it("renders a synthetic project end to end and records a clean probe", async () => { + const renderer = new FfmpegMediaRenderer(); + const { run, renderDir } = await setup({ renderer, realSource: true }); + const render = await run("draft"); + expect(render.error).toBeNull(); + expect(render.status).toBe("done"); + const probe = render.ffprobe!; + expect(probe.width).toBe(profile.video.width); + expect(probe.height).toBe(profile.video.height); + expect(Math.abs(probe.durationMs - 3000)).toBeLessThanOrEqual(1000 / 30 + 1); + expect(Math.abs(probe.integratedLufs! - profile.audio.targetLufs)).toBeLessThanOrEqual(1); + expect(findMetadataLeaks(probe.tags)).toEqual([]); + const video = path.join(renderDir(render.id), "video.mp4"); + const reprobed = await renderer.probe(video); + expect(reprobed.tags["format.title"]).toBe("Example teaser"); + expect(Object.values(reprobed.tags).join(" ")).not.toContain("ExampleCam"); + }, 120_000); +}); diff --git a/packages/adapters/src/media-render.ts b/packages/adapters/src/media-render.ts index 32fd240..259577f 100644 --- a/packages/adapters/src/media-render.ts +++ b/packages/adapters/src/media-render.ts @@ -23,6 +23,7 @@ import { compileRenderPlan, findMetadataLeaks, planAutoFix } from "@rakazo/media import { resolveInside } from "@rakazo/media/node"; import type { MediaActor, + MediaPreparedSources, MediaPrivacyService, MediaRenderService, MediaServiceDeps, @@ -452,15 +453,37 @@ async function runRender(input: RunInput): Promise { const roundDir = path.join(workDir, `round-${round}`); await mkdir(roundDir, { recursive: true }); - const prepared = await privacy.prepareSources({ - actor, - projectId: project.id, - edl, - quality, - workDir: roundDir, - signal, - }); + // Fail closed: no redacted sources, no render. + let prepared: MediaPreparedSources; + try { + prepared = await privacy.prepareSources({ + actor, + projectId: project.id, + edl, + quality, + workDir: roundDir, + signal, + }); + } catch (error) { + checkCanceled(); + getLogger().warn("media render sources not prepared", { + renderId, + reason: error instanceof Error ? error.message : String(error), + }); + throw new MediaRenderFailure( + error instanceof UnsupportedMediaOperationError + ? `Anonymization needs ${error.operation}, which is not available. Nothing was rendered.` + : "Anonymization could not be applied. Nothing was rendered.", + ); + } checkCanceled(); + for (const clip of edl.tracks.video) { + if (!prepared.sources.has(clip.assetId)) { + throw new MediaRenderFailure( + "Anonymization did not prepare every clip. Nothing was rendered.", + ); + } + } for (const clip of edl.tracks.audio) { if (audioFiles.has(clip.artifactId)) continue; @@ -475,37 +498,42 @@ async function runRender(input: RunInput): Promise { const motionClips = await renderMotionClips(deps, edl, kit, brand, roundDir, signal); const { transcripts, subjects } = await analysisInputs(deps, spaceId, edl); - const videoEncoder = await pickEncoder(deps, profile, quality); - - const renderPlan = compileRenderPlan({ - edl, - profile, - brandKit: kit, - quality, - workDir: roundDir, - sources: prepared.sources, - transcripts, - subjects, - motionClips, - files: { - lutPath: brand.lut, - watermarkPath: brand.watermark, - fonts: brand.fonts, - audio: audioFiles, - logoPath: brand.logos[0] ?? null, - }, - videoEncoder, - }); - - await executePlan(input, renderPlan, update); + const compile = (videoEncoder: string) => + compileRenderPlan({ + edl, + profile, + brandKit: kit, + quality, + workDir: roundDir, + sources: prepared.sources, + transcripts, + subjects, + motionClips, + files: { + lutPath: brand.lut, + watermarkPath: brand.watermark, + fonts: brand.fonts, + audio: audioFiles, + logoPath: brand.logos[0] ?? null, + }, + videoEncoder, + }); + const renderPlan = await renderWithFallback(input, profile, quality, compile, update); const video = renderPlan.outputs.find((output) => output.kind === "video"); if (!video) throw new Error("render plan has no video output"); const ffprobe = await deps.renderer.probe(video.path, { measureLoudness: true, signal }); + // A render whose metadata could not be removed is failed, not gated. if (findMetadataLeaks(ffprobe.tags).length > 0) { - // Spec 5: a render whose metadata could not be removed is failed, not gated. throw new MediaRenderFailure("The export still carried source metadata."); } + for (const output of renderPlan.outputs) { + if (output.kind !== "poster") continue; + const poster = await deps.renderer.probe(output.path, { signal }); + if (findMetadataLeaks(poster.tags).length > 0) { + throw new MediaRenderFailure("The poster still carried source metadata."); + } + } const gating = await update({ status: "gating", progress: 0.9, ffprobe }); await publishRender(deps, spaceId, gating); @@ -688,17 +716,64 @@ async function analysisInputs(deps: MediaServiceDeps, spaceId: string, edl: Edl) return { transcripts, subjects }; } +/** Hardware encoders that failed a real render, per renderer, so later renders skip them. */ +const failedEncoders = new WeakMap>(); + +export function softwareEncoder(profile: PlatformProfile): string { + return profile.video.codec === "h264" ? "libx264" : "libx265"; +} + /** Final renders prefer a working hardware encoder of the profile's codec; drafts use software. */ -async function pickEncoder( - deps: MediaServiceDeps, +export async function pickEncoder( + renderer: MediaServiceDeps["renderer"], profile: PlatformProfile, quality: MediaRenderQuality, ): Promise { - const family = profile.video.codec; - const software = family === "h264" ? "libx264" : "libx265"; + const software = softwareEncoder(profile); if (quality === "draft") return software; - const hardware = await deps.renderer.hardwareEncoders(); - return hardware.find((encoder) => encoder.startsWith(`${family}_`)) ?? software; + const failed = failedEncoders.get(renderer); + const hardware = await renderer.hardwareEncoders(); + return ( + hardware.find( + (encoder) => encoder.startsWith(`${profile.video.codec}_`) && !failed?.has(encoder), + ) ?? software + ); +} + +/** + * Run the plan with the preferred encoder. A hardware encoder that fails + * mid-render (driver, device, or session limits) is retried once in software + * and skipped by this process afterwards; software failures are final. + */ +async function renderWithFallback( + input: RunInput, + profile: PlatformProfile, + quality: MediaRenderQuality, + compile: (videoEncoder: string) => RenderPlan, + update: (patch: MediaRenderPatch) => Promise, +): Promise { + const { deps, signal } = input; + const encoder = await pickEncoder(deps.renderer, profile, quality); + const plan = compile(encoder); + const software = softwareEncoder(profile); + try { + await executePlan(input, plan, update); + return plan; + } catch (error) { + if (encoder === software || signal.aborted || error instanceof RenderCanceled) throw error; + if (error instanceof MediaToolMissingError) throw error; + getLogger().warn("media hardware encoder failed; rendering in software", { + renderId: input.renderId, + encoder, + reason: error instanceof Error ? error.message : String(error), + }); + const failed = failedEncoders.get(deps.renderer) ?? new Set(); + failed.add(encoder); + failedEncoders.set(deps.renderer, failed); + } + const fallback = compile(software); + await executePlan(input, fallback, update); + return fallback; } /** Segments the auto-fix may swap in: the project's material, never paid-only. */ diff --git a/packages/testkit/src/fixtures/media/synthetic-media.ts b/packages/testkit/src/fixtures/media/synthetic-media.ts new file mode 100644 index 0000000..da24c66 --- /dev/null +++ b/packages/testkit/src/fixtures/media/synthetic-media.ts @@ -0,0 +1,208 @@ +import { execFile, execFileSync } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +/* + * Synthetic media fixtures for the render integration test, generated with + * ffmpeg lavfi sources at test time (nothing binary is checked in). No real + * people, faces, places, or devices: the "face" is a flat generated disc, the + * picture is testsrc2, the sound is a sine tone, and the device tags are + * placeholders that every export must drop. + */ + +const run = promisify(execFile); + +export function hasFfmpeg(): boolean { + try { + execFileSync("ffmpeg", ["-version"], { stdio: "ignore" }); + execFileSync("ffprobe", ["-version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +async function ffmpeg(args: string[]): Promise { + await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...args]); +} + +/** Tags a phone writes (GPS, make, model, capture time, serial in a handler name). */ +export const PHONE_TAGS: Record = { + creation_time: "2020-01-01T10:00:00Z", + location: "+00.0000+000.0000/", + "com.apple.quicktime.location.ISO6709": "+00.0000+000.0000/", + make: "ExampleCam", + model: "ExampleCam One", + "com.apple.quicktime.software": "ExampleCam OS 1.0", +}; + +export interface SyntheticClipOptions { + seconds: number; + width: number; + height: number; + fps: number; + /** Sine level in dBFS; quiet on purpose so loudness normalization has work to do. */ + volumeDb: number; + /** + * Draw a skin-toned disc ("face") centred at this normalized point with this + * radius relative to the height. The media emulator reports faces for files + * named with `__faces-1__`; the disc sits where its first face box lies. + */ + face?: { cx: number; cy: number; radius: number }; +} + +/** testsrc2 + sine, H.264/AAC MP4 with phone-style tags. */ +export async function writeSyntheticClip(file: string, options: SyntheticClipOptions) { + await mkdir(path.dirname(file), { recursive: true }); + const { seconds, width, height, fps } = options; + const inputs = [ + "-f", + "lavfi", + "-i", + `testsrc2=size=${width}x${height}:rate=${fps}:duration=${seconds}`, + "-f", + "lavfi", + "-i", + `sine=frequency=330:sample_rate=48000:duration=${seconds}`, + ]; + let video = "[0:v]null[v]"; + if (options.face) { + const r = Math.round(options.face.radius * height); + const size = r * 2; + const x = Math.round(options.face.cx * width - r); + const y = Math.round(options.face.cy * height - r); + inputs.push("-f", "lavfi", "-i", `color=c=black:size=${size}x${size}:rate=${fps}`); + video = [ + `[2:v]format=rgba,geq=r=230:g=190:b=160:a='if(lte(hypot(X-${r},Y-${r}),${r}),255,0)'[disc]`, + `[0:v][disc]overlay=x=${x}:y=${y}:shortest=1[v]`, + ].join(";"); + } + const tags = Object.entries(PHONE_TAGS).flatMap(([key, value]) => [ + "-metadata", + `${key}=${value}`, + ]); + await ffmpeg([ + ...inputs, + "-filter_complex", + `${video};[1:a]volume=${options.volumeDb}dB[a]`, + "-map", + "[v]", + "-map", + "[a]", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-t", + String(seconds), + "-movflags", + "+use_metadata_tags", + ...tags, + "-metadata:s:v", + "handler_name=ExampleCam serial 0000", + file, + ]); +} + +/** Sine music bed as WAV. */ +export async function writeMusicBed(file: string, seconds: number): Promise { + await mkdir(path.dirname(file), { recursive: true }); + await ffmpeg([ + "-f", + "lavfi", + "-i", + `sine=frequency=220:sample_rate=48000:duration=${seconds}`, + "-af", + "volume=-18dB", + file, + ]); +} + +/** Flat square PNG used as the brand logo and watermark. */ +export async function writeLogo(file: string): Promise { + await mkdir(path.dirname(file), { recursive: true }); + await ffmpeg(["-f", "lavfi", "-i", "color=c=white:size=96x96", "-frames:v", "1", file]); +} + +/** Decode one frame as 8-bit grayscale at the given size. */ +export async function grayFrame( + file: string, + tMs: number, + width: number, + height: number, +): Promise { + const { stdout } = await run( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-nostdin", + "-ss", + (tMs / 1000).toFixed(3), + "-i", + file, + "-frames:v", + "1", + "-vf", + `scale=${width}:${height}:flags=area`, + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "-", + ], + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + ); + if (stdout.length !== width * height) throw new Error("unexpected frame size"); + return new Uint8Array(stdout); +} + +export interface PixelBox { + x0: number; + y0: number; + x1: number; + y1: number; +} + +/** Variance of the pixels inside a box. */ +export function variance(frame: Uint8Array, width: number, box: PixelBox): number { + let sum = 0; + let squares = 0; + let count = 0; + for (let y = box.y0; y < box.y1; y++) { + for (let x = box.x0; x < box.x1; x++) { + const value = frame[y * width + x]!; + sum += value; + squares += value * value; + count += 1; + } + } + const mean = sum / count; + return squares / count - mean * mean; +} + +/** Mean absolute difference of two frames outside a box. */ +export function meanDifferenceOutside( + a: Uint8Array, + b: Uint8Array, + width: number, + height: number, + box: PixelBox, +): number { + let total = 0; + let count = 0; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + if (x >= box.x0 && x < box.x1 && y >= box.y0 && y < box.y1) continue; + total += Math.abs(a[y * width + x]! - b[y * width + x]!); + count += 1; + } + } + return total / count; +} diff --git a/packages/testkit/src/media-render.docker.test.ts b/packages/testkit/src/media-render.docker.test.ts new file mode 100644 index 0000000..307f740 --- /dev/null +++ b/packages/testkit/src/media-render.docker.test.ts @@ -0,0 +1,359 @@ +import path from "node:path"; +import type { BoxKeyframe, Edl } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { MediaPrivacyService } from "../../adapters/src/media-context.js"; +import { EmulatorMediaInferenceProvider } from "../../adapters/src/media-emulator.js"; +import { FfmpegMediaRenderer } from "../../adapters/src/media-ffmpeg-renderer.js"; +import type { RenderHarness } from "../../adapters/src/media-render.fixtures.js"; +import { + createFakePrivacy, + createRenderHarness, + renderTestKit, + renderTestProfile, +} from "../../adapters/src/media-render.fixtures.js"; +import { + createMediaRenderHandler, + createMediaRenderService, +} from "../../adapters/src/media-render.js"; +import { findMetadataLeaks } from "../../media/src/gate.js"; +import type { PixelBox } from "./fixtures/media/synthetic-media.js"; +import { + grayFrame, + hasFfmpeg, + meanDifferenceOutside, + variance, + writeLogo, + writeMusicBed, + writeSyntheticClip, +} from "./fixtures/media/synthetic-media.js"; + +/* + * Real render integration (spec 13, Docker job). Synthetic fixtures are made + * with ffmpeg lavfi; inference runs on the deterministic emulator; the render + * goes through the real service, job handler, compiler, and ffmpeg renderer, + * and every result is checked with ffprobe. Runs wherever ffmpeg is installed + * (the CI test job and the media Docker job) and skips cleanly without it. + * + * Redactions belong to the privacy slice. Its stand-in here applies EDL region + * redactions through the emulator's `redact` (the same MediaInferenceProvider + * call the privacy service uses) and refuses to return an unredacted source. + */ + +const FPS = 30; +const FRAME_MS = 1000 / FPS; +/** Grayscale variance a blurred region must stay under. */ +const VARIANCE_THRESHOLD = 15; +/** + * The generated "face": a flat disc inside every box the media emulator can + * report as the first face of a `__faces-1__` file (boxes are seeded by the + * file, but always contain this point and radius). + */ +const FACE = { cx: 0.19, cy: 0.28, radius: 0.05 }; + +const vertical = renderTestProfile(); +const landscape = renderTestProfile({ + id: "example-landscape-test", + name: "Example landscape test", + video: { + aspectRatios: ["16:9"], + width: 640, + height: 360, + fps: FPS, + codec: "h264", + maxBitrateKbps: 6000, + maxFileSizeMb: 500, + }, + safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 }, +}); + +const motion = { + id: "none", + templates: () => [], + render: async (): Promise => { + throw new Error("no motion templates in this plan"); + }, +}; + +/** Privacy stand-in: region redactions through the inference provider, fail closed. */ +function emulatorPrivacy( + harness: RenderHarness, + inference: EmulatorMediaInferenceProvider, +): MediaPrivacyService { + const base = createFakePrivacy(harness.sources); + return { + ...base, + async prepareSources(input) { + const sources = new Map(); + for (const clip of input.edl.tracks.video) { + const source = harness.sources.get(clip.assetId); + if (!source) throw new Error(`no source for ${clip.assetId}`); + const regions = input.edl.redactions.regions.filter((r) => r.clipId === clip.id); + if (regions.length === 0) { + sources.set(clip.assetId, source); + continue; + } + const outPath = path.join(input.workDir, `redacted-${clip.id}.mp4`); + const result = await inference.redact({ + path: source.path, + outPath, + redactions: regions.map((region) => ({ + keyframes: region.keyframes, + mode: region.treatment === "cover" ? "cover" : "blur", + paddingPct: 10, + featherPct: 0, + })), + signal: input.signal, + }); + if (result.framesProcessed <= 0) throw new Error("redaction processed no frames"); + sources.set(clip.assetId, { ...source, path: result.outPath }); + } + return { sources, aiAlterations: [], fallbacks: [] }; + }, + }; +} + +interface Fixture { + harness: RenderHarness; + inference: EmulatorMediaInferenceProvider; + renderer: FfmpegMediaRenderer; + assetId: string; + sourcePath: string; + musicArtifactId: string; + render: (edl: unknown, quality: "draft" | "final") => Promise<{ id: string; dir: string }>; +} + +describe.skipIf(!hasFfmpeg())("media render integration with real ffmpeg", () => { + let fx: Fixture; + + beforeAll(async () => { + const renderer = new FfmpegMediaRenderer(); + const harness = await createRenderHarness({ + profiles: [vertical, landscape], + renderer, + motion, + }); + const inference = new EmulatorMediaInferenceProvider({ root: harness.mediaRoot }); + harness.deps.inference = inference; + const name = "lake__faces-1__speech__.mp4"; + const assetId = await harness.createAsset(name); + const sourcePath = harness.assetPath(assetId, name); + await writeSyntheticClip(sourcePath, { + seconds: 6, + width: 1280, + height: 720, + fps: FPS, + volumeDb: -32, + face: FACE, + }); + harness.setSource(assetId, { + path: sourcePath, + width: 1280, + height: 720, + durationMs: 6000, + fps: FPS, + hasAudio: true, + rotation: 0, + }); + const timesMs = Array.from({ length: 13 }, (_, i) => i * 500); + await harness.addAnalysis( + assetId, + "transcript", + await inference.transcribe({ path: sourcePath, language: "en" }), + ); + await harness.addAnalysis( + assetId, + "saliency", + await inference.saliency({ path: sourcePath, timesMs }), + ); + + const brandDir = path.join(harness.mediaRoot, "fixtures"); + await writeLogo(path.join(brandDir, "logo.png")); + await writeMusicBed(path.join(brandDir, "bed.wav"), 6); + const logoArtifactId = await harness.addArtifact( + path.join(brandDir, "logo.png"), + "logo.png", + "image/png", + ); + const musicArtifactId = await harness.addArtifact( + path.join(brandDir, "bed.wav"), + "bed.wav", + "audio/wav", + ); + const kit = renderTestKit({ logoArtifactId, musicArtifactId }); + + const privacy = emulatorPrivacy(harness, inference); + const service = createMediaRenderService(harness.deps); + const handler = createMediaRenderHandler(harness.deps, privacy); + const actor = { spaceId: harness.spaceId, userId: harness.userId }; + fx = { + harness, + inference, + renderer, + assetId, + sourcePath, + musicArtifactId, + async render(edl, quality) { + const projectId = await harness.createProject({ + assetIds: [assetId], + kit, + profileId: (edl as Edl).output.profileId, + }); + const plan = await harness.createPlan(projectId, edl); + const started = await service.start(actor, { planId: plan.id, quality }); + await handler({ renderId: started.id }); + const render = await harness.deps.repos.getRender(harness.spaceId, started.id); + expect(render.error).toBeNull(); + expect(render.status).toBe("done"); + return { + id: render.id, + dir: path.join(harness.mediaRoot, harness.spaceId, "renders", render.id), + }; + }, + }; + }); + + afterAll(async () => { + await fx?.harness.cleanup(); + }); + + it("renders 16:9 to 9:16 with auto-reframe, captions, brand overlay and loudness", async () => { + const edl = { + edlVersion: 1, + output: { + profileId: vertical.id, + width: vertical.video.width, + height: vertical.video.height, + fps: FPS, + durationTargetMs: 4000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [ + { id: "c1", assetId: fx.assetId, inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: fx.assetId, inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: fx.musicArtifactId, + inMs: 0, + outMs: 4000, + atMs: 0, + gainDb: -6, + fadeOutMs: 500, + duckUnderSpeech: true, + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "cut", durationMs: 0 }], + captions: { mode: "both", presetId: "clean" }, + overlays: [ + { id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 }, + { id: "lt", kind: "lower_third", atMs: 500, durationMs: 1500, text: "Example: 50% off" }, + ], + metadata: { title: "Example teaser" }, + }; + const render = await fx.render(edl, "final"); + const video = path.join(render.dir, "video.mp4"); + const probe = await fx.renderer.probe(video, { measureLoudness: true }); + + expect(Math.abs(probe.durationMs - 4000)).toBeLessThanOrEqual(FRAME_MS + 1); + expect(probe.width / probe.height).toBeCloseTo(9 / 16, 3); + expect(probe.width).toBe(vertical.video.width); + expect(probe.fps).toBe(FPS); + expect(probe.videoCodec).toBe("h264"); + expect(Math.abs(probe.integratedLufs! - vertical.audio.targetLufs)).toBeLessThanOrEqual(1); + expect(probe.truePeakDb!).toBeLessThanOrEqual(vertical.audio.truePeakDb + 0.5); + + expect(findMetadataLeaks(probe.tags)).toEqual([]); + const tagText = JSON.stringify(probe.tags).toLowerCase(); + for (const leak of ["location", "iso6709", "make", "model", "creation_time", "examplecam"]) { + expect(tagText).not.toContain(leak); + } + expect(probe.tags["format.title"]).toBe("Example teaser"); + + const poster = await fx.renderer.probe(path.join(render.dir, "poster.jpg")); + expect(findMetadataLeaks(poster.tags)).toEqual([]); + + const stored = await fx.harness.deps.repos.getRender(fx.harness.spaceId, render.id); + expect(stored.gateReport?.passed).toBe(true); + expect(stored.outputs.map((output) => output.fileName).sort()).toEqual([ + "captions.srt", + "captions.vtt", + "poster.jpg", + "video.mp4", + ]); + }, 240_000); + + it("tracks the generated face disc with the emulator", async () => { + const tracks = await fx.inference.trackFaces({ path: fx.sourcePath, sampleFps: 2 }); + expect(tracks.tracks).toHaveLength(1); + const disc = { x0: FACE.cx - FACE.radius * (9 / 16), x1: FACE.cx + FACE.radius * (9 / 16) }; + for (const { box } of tracks.tracks[0]!.keyframes) { + expect(box.x).toBeLessThanOrEqual(disc.x0); + expect(box.x + box.w).toBeGreaterThanOrEqual(disc.x1); + expect(box.y).toBeLessThanOrEqual(FACE.cy - FACE.radius); + expect(box.y + box.h).toBeGreaterThanOrEqual(FACE.cy + FACE.radius); + } + }); + + it("blurs a redaction region below the variance threshold and leaves the rest unchanged", async () => { + // A static region over testsrc2's fine checkerboard, held for the whole clip. + const box = { x: 0.68, y: 0.64, w: 0.14, h: 0.2 }; + const keyframes: BoxKeyframe[] = [ + { tMs: 0, box }, + { tMs: 3000, box }, + ]; + const edl = (regions: unknown[]) => ({ + edlVersion: 1, + output: { + profileId: landscape.id, + width: landscape.video.width, + height: landscape.video.height, + fps: FPS, + durationTargetMs: 3000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [{ id: "c1", assetId: fx.assetId, inMs: 0, outMs: 3000, atMs: 0, reframe: "none" }], + }, + redactions: { regions }, + metadata: {}, + }); + const plain = await fx.render(edl([]), "draft"); + const redacted = await fx.render( + edl([{ id: "region", clipId: "c1", keyframes, treatment: "blur" }]), + "draft", + ); + + const W = landscape.video.width; + const H = landscape.video.height; + const inner: PixelBox = { + x0: Math.ceil((box.x + box.w * 0.15) * W), + y0: Math.ceil((box.y + box.h * 0.15) * H), + x1: Math.floor((box.x + box.w * 0.85) * W), + y1: Math.floor((box.y + box.h * 0.85) * H), + }; + // Padding (10 %) and blur bleed at the edge stay out of the comparison. + const margin = 0.04; + const outer: PixelBox = { + x0: Math.floor((box.x - margin) * W), + y0: Math.floor((box.y - margin) * H), + x1: Math.ceil((box.x + box.w + margin) * W), + y1: Math.ceil((box.y + box.h + margin) * H), + }; + for (const tMs of [500, 1500, 2500]) { + const before = await grayFrame(path.join(plain.dir, "video.mp4"), tMs, W, H); + const after = await grayFrame(path.join(redacted.dir, "video.mp4"), tMs, W, H); + const original = variance(before, W, inner); + const blurred = variance(after, W, inner); + const outside = meanDifferenceOutside(before, after, W, H, outer); + expect(original).toBeGreaterThan(VARIANCE_THRESHOLD * 4); + expect(blurred).toBeLessThan(VARIANCE_THRESHOLD); + expect(outside).toBeLessThan(1.5); + } + }, 240_000); +});