From 8f38c2211947f93745a0ec7a2263b5ab8307c9c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:24:51 +0000 Subject: [PATCH 1/2] wip(media): w2-render partial work (handover, unverified) Stopped mid-implementation for the handover to the project thread. Not typechecked or tested; do not merge as is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- .../adapters/src/media-ffmpeg-renderer.ts | 599 +++++++++++++ .../adapters/src/media-render.fixtures.ts | 676 ++++++++++++++ packages/adapters/src/media-render.ts | 835 ++++++++++++++++++ 3 files changed, 2110 insertions(+) create mode 100644 packages/adapters/src/media-ffmpeg-renderer.ts create mode 100644 packages/adapters/src/media-render.fixtures.ts create mode 100644 packages/adapters/src/media-render.ts diff --git a/packages/adapters/src/media-ffmpeg-renderer.ts b/packages/adapters/src/media-ffmpeg-renderer.ts new file mode 100644 index 0000000..46b1487 --- /dev/null +++ b/packages/adapters/src/media-ffmpeg-renderer.ts @@ -0,0 +1,599 @@ +import { spawn } from "node:child_process"; +import { access, mkdir, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { MediaRenderer, MediaRenderProgress } from "@rakazo/adapter-kit"; +import type { FfprobeReport, RenderPlan, RenderStep } from "@rakazo/contracts"; +import { getLogger } from "@rakazo/logging"; +import { assertInside } from "@rakazo/media/node"; +import { MediaToolMissingError, runMediaTool } from "./media-folder-source-probe.js"; + +/* + * Local ffmpeg executor for compiled render plans. Every process is spawned + * with an argv array and no shell. Plans come from compileRenderPlan, which + * already escapes every value; this module only adds runtime options + * (progress, hardware device) and the loudness measurement pass. + * + * Loudness: when the render step carries a single `loudnorm` with + * linear=true and no measured values, a first pass measures the program + * (print_format=json, null muxer) and the filter is rewritten with the + * measured_* options, so the second pass normalizes linearly to the target. + * When the measurement cannot be parsed the step runs single-pass (dynamic). + */ + +const HARDWARE_CANDIDATES = [ + "h264_vaapi", + "hevc_vaapi", + "h264_videotoolbox", + "hevc_videotoolbox", + "h264_nvenc", + "hevc_nvenc", +] as const; +const DEFAULT_VAAPI_DEVICE = "/dev/dri/renderD128"; +const DEFAULT_KILL_GRACE_MS = 2000; +const STDERR_TAIL_BYTES = 64 * 1024; + +export interface FfmpegMediaRendererOptions { + ffmpegPath?: string; + ffprobePath?: string; + /** When set, plan work dirs and probed or extracted paths must lie inside it. */ + root?: string; + vaapiDevice?: string; + /** Time between SIGTERM and SIGKILL when a render is aborted. */ + killGraceMs?: number; +} + +function abortError(signal: AbortSignal | undefined): Error { + const reason = signal?.reason; + if (reason instanceof Error && reason.name === "AbortError") return reason; + const error = new Error("render aborted"); + error.name = "AbortError"; + return error; +} + +/** "30000/1001" → 29.97; "0/0" → 0. */ +export function parseFrameRate(value: string | undefined): number { + if (!value) return 0; + const [num, den] = value.split("/").map(Number); + if (num === undefined || !Number.isFinite(num)) return 0; + const rate = den === undefined ? num : den > 0 ? num / den : 0; + return Number.isFinite(rate) ? Math.round(rate * 1000) / 1000 : 0; +} + +interface ProbeJson { + streams?: Array<{ + codec_type?: string; + codec_name?: string; + width?: number; + height?: number; + r_frame_rate?: string; + tags?: Record; + }>; + format?: { duration?: string; size?: string; tags?: Record }; +} + +/** ffprobe JSON → report without loudness. Tags are flattened as `format.x` / `stream.N.x`. */ +export function parseProbeReport(json: ProbeJson, fileSize: number): FfprobeReport { + const streams = json.streams ?? []; + const video = streams.find((stream) => stream.codec_type === "video"); + const audio = streams.find((stream) => stream.codec_type === "audio"); + const tags: Record = {}; + for (const [key, value] of Object.entries(json.format?.tags ?? {})) { + tags[`format.${key}`] = String(value); + } + streams.forEach((stream, index) => { + for (const [key, value] of Object.entries(stream.tags ?? {})) { + tags[`stream.${index}.${key}`] = String(value); + } + }); + const seconds = Number(json.format?.duration); + const size = Number(json.format?.size); + return { + durationMs: Number.isFinite(seconds) && seconds > 0 ? Math.round(seconds * 1000) : 0, + width: video?.width ?? 0, + height: video?.height ?? 0, + fps: parseFrameRate(video?.r_frame_rate), + videoCodec: video?.codec_name ?? "none", + audioCodec: audio?.codec_name ?? null, + sizeBytes: Number.isFinite(size) && size > 0 ? Math.round(size) : fileSize, + integratedLufs: null, + truePeakDb: null, + tags, + }; +} + +/** Integrated loudness and true peak from the ebur128 summary on stderr. */ +export function parseEbur128Summary(stderr: string): { + integratedLufs: number | null; + truePeakDb: number | null; +} { + const at = stderr.lastIndexOf("Summary:"); + if (at < 0) return { integratedLufs: null, truePeakDb: null }; + const summary = stderr.slice(at); + const integrated = /I:\s+(-?\d+(?:\.\d+)?) LUFS/.exec(summary); + const peak = /True peak:\s+Peak:\s+(-?\d+(?:\.\d+)?|-inf) dBFS/.exec(summary); + const lufs = integrated ? Number(integrated[1]) : null; + const tp = peak && peak[1] !== "-inf" ? Number(peak[1]) : null; + return { + // ebur128 reports -70 LUFS (its gate) for silence; that is "no programme". + integratedLufs: lufs !== null && Number.isFinite(lufs) && lufs > -70 ? lufs : null, + truePeakDb: tp !== null && Number.isFinite(tp) ? tp : null, + }; +} + +// --------------------------------------------------------------------------- +// Two-pass loudnorm +// --------------------------------------------------------------------------- + +const LOUDNORM = /loudnorm=((?:[^,;[\]\\]|\\.)*)/g; + +interface LoudnormSite { + /** Index of the options string inside the graph. */ + start: number; + end: number; + options: string; +} + +/** The single linear loudnorm without measured values, or null. */ +export function findTwoPassLoudnorm(graph: string): LoudnormSite | null { + const matches = [...graph.matchAll(LOUDNORM)]; + if (matches.length !== 1) return null; + const match = matches[0]!; + const options = match[1] ?? ""; + const parts = options.split(":"); + if (!parts.includes("linear=true")) return null; + if (parts.some((part) => part.startsWith("measured_"))) return null; + const start = (match.index ?? 0) + "loudnorm=".length; + return { start, end: start + options.length, options }; +} + +export interface LoudnormMeasurement { + inputI: number; + inputTp: number; + inputLra: number; + inputThresh: number; + targetOffset: number; +} + +/** The JSON block loudnorm prints after its summary line; null when absent or not finite. */ +export function parseLoudnormJson(stderr: string): LoudnormMeasurement | null { + const at = stderr.lastIndexOf("Parsed_loudnorm"); + if (at < 0) return null; + const open = stderr.indexOf("{", at); + const close = stderr.indexOf("}", open); + if (open < 0 || close < 0) return null; + let parsed: Record; + try { + parsed = JSON.parse(stderr.slice(open, close + 1)) as Record; + } catch { + return null; + } + const read = (key: string) => { + const value = Number(parsed[key]); + return Number.isFinite(value) ? value : null; + }; + const inputI = read("input_i"); + const inputTp = read("input_tp"); + const inputLra = read("input_lra"); + const inputThresh = read("input_thresh"); + const targetOffset = read("target_offset"); + if ( + inputI === null || + inputTp === null || + inputLra === null || + inputThresh === null || + targetOffset === null + ) { + return null; + } + return { inputI, inputTp, inputLra, inputThresh, targetOffset }; +} + +function withLoudnormOptions(graph: string, site: LoudnormSite, extra: string): string { + return `${graph.slice(0, site.start)}${site.options}:${extra}${graph.slice(site.end)}`; +} + +function measuredOptions(m: LoudnormMeasurement): string { + const f = (value: number) => value.toFixed(2); + return [ + `measured_I=${f(m.inputI)}`, + `measured_TP=${f(m.inputTp)}`, + `measured_LRA=${f(m.inputLra)}`, + `measured_thresh=${f(m.inputThresh)}`, + `offset=${f(m.targetOffset)}`, + ].join(":"); +} + +// --------------------------------------------------------------------------- +// Renderer +// --------------------------------------------------------------------------- + +interface RunOptions { + signal?: AbortSignal; + /** Called with the output time reached, in ms. */ + onTime?: (ms: number) => void; +} + +export class FfmpegMediaRenderer implements MediaRenderer { + readonly id = "ffmpeg"; + private readonly ffmpegPath: string; + private readonly ffprobePath: string; + private readonly vaapiDevice: string; + private readonly killGraceMs: number; + private encoders: Promise | null = null; + + constructor(private readonly options: FfmpegMediaRendererOptions = {}) { + this.ffmpegPath = options.ffmpegPath ?? "ffmpeg"; + this.ffprobePath = options.ffprobePath ?? "ffprobe"; + this.vaapiDevice = options.vaapiDevice ?? DEFAULT_VAAPI_DEVICE; + this.killGraceMs = options.killGraceMs ?? DEFAULT_KILL_GRACE_MS; + } + + hardwareEncoders(): Promise { + this.encoders ??= this.detectEncoders().catch((error) => { + getLogger().warn("media hardware encoder detection failed", { + reason: error instanceof Error ? error.message : String(error), + }); + return []; + }); + return this.encoders; + } + + private async detectEncoders(): Promise { + const { stdout } = await runMediaTool(this.ffmpegPath, ["-hide_banner", "-encoders"]); + const listed = new Set( + stdout + .toString("utf8") + .split("\n") + .map((line) => line.trim().split(/\s+/)[1]) + .filter((name): name is string => Boolean(name)), + ); + const hasVaapiDevice = await access(this.vaapiDevice).then( + () => true, + () => false, + ); + const usable: string[] = []; + for (const encoder of HARDWARE_CANDIDATES) { + if (!listed.has(encoder)) continue; + const vaapi = encoder.endsWith("_vaapi"); + if (vaapi && !hasVaapiDevice) continue; + // One-frame test encode: listed encoders often lack the device or driver. + const args = ["-hide_banner", "-v", "error", "-nostdin"]; + if (vaapi) args.push("-vaapi_device", this.vaapiDevice); + args.push("-f", "lavfi", "-i", "color=c=black:s=256x256:r=30:d=0.1", "-frames:v", "1"); + if (vaapi) args.push("-vf", "format=nv12,hwupload"); + args.push("-c:v", encoder, "-f", "null", "-"); + try { + await runMediaTool(this.ffmpegPath, args); + usable.push(encoder); + } catch { + // Not usable on this host. + } + } + return usable; + } + + async render( + plan: RenderPlan, + options: { signal?: AbortSignal; onProgress?: (progress: MediaRenderProgress) => void } = {}, + ): Promise { + const { signal, onProgress } = options; + if (signal?.aborted) throw abortError(signal); + const workDir = await this.workDir(plan.workDir); + const weight = (step: RenderStep) => + step.kind === "ffmpeg" ? Math.max(step.durationMs, 1) : 0; + const total = plan.steps.reduce((sum, step) => sum + weight(step), 0) || 1; + let done = 0; + const report = (stepId: string, fraction: number, stepWeight: number) => { + const value = (done + Math.min(1, Math.max(0, fraction)) * stepWeight) / total; + onProgress?.({ stepId, progress: Math.min(1, Math.max(0, value)) }); + }; + const written: string[] = []; + try { + for (const step of plan.steps) { + if (signal?.aborted) throw abortError(signal); + switch (step.kind) { + case "write_file": { + const file = await this.insideWorkDir(workDir, step.path); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, step.content, "utf8"); + written.push(file); + break; + } + case "motion": + case "inpaint": + case "synthetic_identity": { + const ready = await access(step.outPath).then( + () => true, + () => false, + ); + if (!ready) throw new Error(`${step.kind} step ${step.id} was not prepared`); + break; + } + case "ffmpeg": { + for (const produced of step.produces) await this.insideWorkDir(workDir, produced); + written.push(...step.produces); + await this.runStep(step, plan, workDir, { + signal, + report: (fraction) => report(step.id, fraction, weight(step)), + }); + done += weight(step); + report(step.id, 0, 0); + break; + } + } + } + } catch (error) { + await Promise.all(written.map((file) => rm(file, { force: true }).catch(() => undefined))); + if (signal?.aborted) throw abortError(signal); + throw error; + } + } + + private async runStep( + step: Extract, + plan: RenderPlan, + workDir: string, + options: { signal?: AbortSignal; report: (fraction: number) => void }, + ): Promise { + let argv = [...step.argv]; + const vaapi = argv.some((arg, i) => argv[i - 1] === "-c:v" && arg.endsWith("_vaapi")); + const prefix = vaapi ? ["-vaapi_device", this.vaapiDevice] : []; + const durationMs = Math.max(step.durationMs, 1); + const twoPass = await this.loudnormPlan(argv, plan, workDir); + if (twoPass) { + let measurement: LoudnormMeasurement | null = null; + try { + const { stderr } = await this.run([...prefix, ...twoPass.measureArgv], { + signal: options.signal, + onTime: (ms) => options.report((ms / durationMs) * 0.5), + }); + measurement = parseLoudnormJson(stderr); + } finally { + if (twoPass.measureScript) await rm(twoPass.measureScript, { force: true }); + } + if (measurement) { + argv = await twoPass.apply(measuredOptions(measurement)); + } else { + getLogger().warn("loudnorm measurement unreadable; rendering single pass", { + stepId: step.id, + }); + } + } + const offset = twoPass ? 0.5 : 0; + const scale = twoPass ? 0.5 : 1; + await this.run([...prefix, ...argv], { + signal: options.signal, + onTime: (ms) => options.report(offset + (ms / durationMs) * scale), + }); + options.report(1); + } + + /** + * Measurement argv and a function that writes the measured filter into the + * render argv (or its filtergraph script), or null when the step needs no + * measurement pass. + */ + private async loudnormPlan( + argv: string[], + plan: RenderPlan, + workDir: string, + ): Promise<{ + measureArgv: string[]; + measureScript: string | null; + apply: (extra: string) => Promise; + } | null> { + const inline = argv.indexOf("-filter_complex"); + const scriptAt = argv.indexOf("-filter_complex_script"); + const flagAt = inline >= 0 ? inline : scriptAt; + if (flagAt < 0 || flagAt + 1 >= argv.length) return null; + let graph: string; + let scriptPath: string | null = null; + if (inline >= 0) { + graph = argv[inline + 1]!; + } else { + scriptPath = await this.insideWorkDir(workDir, argv[scriptAt + 1]!); + let content: string | null = null; + for (const step of plan.steps) { + if (step.kind !== "write_file") continue; + if ((await this.insideWorkDir(workDir, step.path)) === scriptPath) content = step.content; + } + if (content === null) return null; + graph = content; + } + const site = findTwoPassLoudnorm(graph); + if (!site) return null; + + const tail = argv.slice(flagAt + 2); + const maps: string[] = []; + let duration: string | null = null; + for (let i = 0; i < tail.length - 1; i++) { + if (tail[i] === "-map") maps.push("-map", tail[i + 1]!); + if (tail[i] === "-t") duration = tail[i + 1]!; + } + const measureGraph = withLoudnormOptions(graph, site, "print_format=json"); + let measureScript: string | null = null; + let graphArg = measureGraph; + if (scriptPath) { + measureScript = path.join(path.dirname(scriptPath), "filtergraph.measure.txt"); + await writeFile(measureScript, measureGraph, "utf8"); + graphArg = measureScript; + } + const measureArgv = [ + ...argv.slice(0, flagAt + 1), + graphArg, + ...maps, + ...(duration ? ["-t", duration] : []), + "-f", + "null", + "-", + ]; + return { + measureArgv, + measureScript, + apply: async (extra) => { + const next = withLoudnormOptions(graph, site, extra); + const out = [...argv]; + if (scriptPath) await writeFile(scriptPath, next, "utf8"); + else out[flagAt + 1] = next; + return out; + }, + }; + } + + /** Spawn ffmpeg with progress on stdout. Rejects with the stderr tail on failure. */ + private run(args: string[], options: RunOptions): Promise<{ stderr: string }> { + const { signal } = options; + if (signal?.aborted) return Promise.reject(abortError(signal)); + return new Promise((resolve, reject) => { + const child = spawn(this.ffmpegPath, ["-progress", "pipe:1", "-nostats", ...args], { + stdio: ["ignore", "pipe", "pipe"], + shell: false, + windowsHide: true, + }); + let stderr = ""; + let pending = ""; + let killTimer: NodeJS.Timeout | null = null; + const onAbort = () => { + child.kill("SIGTERM"); + killTimer = setTimeout(() => child.kill("SIGKILL"), this.killGraceMs); + killTimer.unref(); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + pending += chunk; + const lines = pending.split("\n"); + pending = lines.pop() ?? ""; + for (const line of lines) { + const match = /^out_time_(us|ms)=(\d+)/.exec(line.trim()); + // Both keys are microseconds in ffmpeg 5+ (out_time_ms is misnamed). + if (match) options.onTime?.(Number(match[2]) / 1000); + } + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + if (stderr.length > STDERR_TAIL_BYTES * 2) stderr = stderr.slice(-STDERR_TAIL_BYTES); + }); + child.on("error", (error: NodeJS.ErrnoException) => { + signal?.removeEventListener("abort", onAbort); + reject(error.code === "ENOENT" ? new MediaToolMissingError(this.ffmpegPath) : error); + }); + child.on("close", (code) => { + signal?.removeEventListener("abort", onAbort); + if (killTimer) clearTimeout(killTimer); + if (signal?.aborted) { + reject(abortError(signal)); + return; + } + if (code !== 0) { + const tail = stderr.trim().split("\n").slice(-4).join(" "); + reject(new Error(`ffmpeg failed (${code}): ${tail}`)); + return; + } + resolve({ stderr }); + }); + }); + } + + async probe( + file: string, + options: { measureLoudness?: boolean; signal?: AbortSignal } = {}, + ): Promise { + const target = await this.checked(file); + const { stdout } = await runMediaTool( + this.ffprobePath, + ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "--", target], + { signal: options.signal }, + ); + const report = parseProbeReport( + JSON.parse(stdout.toString("utf8")) as ProbeJson, + (await stat(target)).size, + ); + if (!options.measureLoudness || report.audioCodec === null) return report; + const { stderr } = await runMediaTool( + this.ffmpegPath, + [ + "-hide_banner", + "-nostats", + "-nostdin", + "-i", + target, + "-map", + "0:a:0", + "-af", + "ebur128=peak=true", + "-f", + "null", + "-", + ], + { signal: options.signal }, + ); + return { ...report, ...parseEbur128Summary(stderr) }; + } + + async extractFrame(input: { + path: string; + tMs: number; + outPath: string; + width?: number; + signal?: AbortSignal; + }): Promise { + const source = await this.checked(input.path); + if (path.extname(input.outPath).toLowerCase() !== ".png") { + throw new Error("frames are extracted as PNG"); + } + const outPath = await this.checked(input.outPath); + if (!Number.isFinite(input.tMs) || input.tMs < 0) throw new Error("invalid frame time"); + const args = [ + "-hide_banner", + "-v", + "error", + "-nostdin", + "-y", + "-ss", + (input.tMs / 1000).toFixed(3), + "-i", + source, + "-frames:v", + "1", + ]; + if (input.width !== undefined) { + const width = Math.max(2, Math.floor(input.width / 2) * 2); + args.push("-vf", `scale=${width}:-2`); + } + args.push( + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-f", + "image2", + "-c:v", + "png", + outPath, + ); + await runMediaTool(this.ffmpegPath, args, { signal: input.signal }); + const written = await stat(outPath).catch(() => null); + if (!written || written.size === 0) throw new Error("no frame at the requested time"); + } + + private async checked(file: string): Promise { + if (!path.isAbsolute(file)) throw new Error("media paths must be absolute"); + return this.options.root ? assertInside(this.options.root, file) : path.resolve(file); + } + + private async workDir(dir: string): Promise { + const resolved = await this.checked(dir); + await mkdir(resolved, { recursive: true }); + return resolved; + } + + /** Steps may only write inside the plan's work directory. */ + private async insideWorkDir(workDir: string, file: string): Promise { + if (!path.isAbsolute(file)) throw new Error("plan paths must be absolute"); + return assertInside(workDir, file); + } +} diff --git a/packages/adapters/src/media-render.fixtures.ts b/packages/adapters/src/media-render.fixtures.ts new file mode 100644 index 0000000..6926759 --- /dev/null +++ b/packages/adapters/src/media-render.fixtures.ts @@ -0,0 +1,676 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { + JobPublisher, + LinkShortener, + MediaInferenceProvider, + MediaPublisher, + MediaRenderer, + MotionGraphicsRenderer, +} from "@rakazo/adapter-kit"; +import type { + BrandKit, + Edl, + FfprobeReport, + GateIssue, + GateReport, + MediaAnalysisKind, + MediaEditPlan, + MediaRender, + PlatformProfile, +} from "@rakazo/contracts"; +import { BrandKitSchema, EdlSchema, PlatformProfileSchema } from "@rakazo/contracts"; +import type { MediaRenderPatch, MediaRepos, PrismaClient } from "@rakazo/db"; +import { + ACTIVE_RENDER_STATUSES, + bootstrapUserSpace, + createDb, + createMediaRepos, + IsolationError, + MediaStateConflictError, + mapMediaEditPlan, + mapMediaRender, +} from "@rakazo/db"; +import type { CompileSource } from "@rakazo/media"; +import { evaluateGate } from "@rakazo/media"; +import { LocalArtifactStore } from "./artifacts.js"; +import type { + MediaPreparedSources, + MediaPrivacyService, + MediaRealtimeEvent, + MediaServiceDeps, +} from "./media-context.js"; + +/* + * Test harness for the render service and job: the repos the render slice + * reads, either in memory or backed by PostgreSQL (VERIFY_DATABASE), plus a + * pass-through privacy service. Everything is synthetic. + */ + +export function renderTestProfile(overrides: Record = {}): PlatformProfile { + return PlatformProfileSchema.parse({ + id: "example-vertical-test", + name: "Example vertical test", + kind: "own_site", + reviewedAt: "2026-09-01", + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "h264", + maxBitrateKbps: 8000, + maxFileSizeMb: 500, + }, + duration: { hardMinMs: 1000, hardMaxMs: 120_000, hookWindowMs: 2000 }, + safeZones: { topPct: 8, bottomPct: 15, leftPct: 4, rightPct: 10 }, + captions: { recommended: true, maxLines: 2 }, + audio: { targetLufs: -14, truePeakDb: -1 }, + text: { captionMaxChars: 500, hashtagLimit: 5, linkPolicy: "allowed" }, + content: { thresholds: { violence: 0.5 } }, + disclosure: { aiAlteredLabelRequired: true }, + consent: { requirePerformerConsent: false, requireAgeVerification: false }, + export: {}, + ...overrides, + }); +} + +export function renderTestKit(input: { + logoArtifactId: string; + musicArtifactId: string; +}): BrandKit { + return BrandKitSchema.parse({ + name: "Studio Example", + colors: { + primary: "#224488", + secondary: "#446688", + accent: "#FFAA00", + text: "#FFFFFF", + background: "#101010", + }, + logoArtifactIds: [input.logoArtifactId], + watermark: { + artifactId: input.logoArtifactId, + position: "top_right", + opacity: 0.8, + scale: 0.2, + }, + captionPresets: [ + { + id: "clean", + name: "Clean", + template: "clean", + fontFamily: "DejaVu Sans", + sizePct: 5, + textColor: "#FFFFFF", + }, + ], + musicBeds: [ + { artifactId: input.musicArtifactId, title: "Bed", license: "own", commercialUse: true }, + ], + }); +} + +export interface HarnessAsset { + assetId: string; + source: CompileSource; +} + +export interface RenderHarness { + backend: "memory" | "postgres"; + mediaRoot: string; + spaceId: string; + userId: string; + deps: MediaServiceDeps; + events: MediaRealtimeEvent[]; + enqueued: string[]; + /** Register an asset; the caller writes the file at `assetPath(assetId, fileName)`. */ + createAsset(name: string): Promise; + assetPath(assetId: string, fileName: string): string; + setSource(assetId: string, source: CompileSource): void; + sources: Map; + addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise; + addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise; + addArtifact(file: string, name: string, mimeType: string): Promise; + createProject(input: { assetIds: string[]; kit: BrandKit | null; profileId: string }): Promise; + createPlan(projectId: string, edl: unknown): Promise; + planVersions(projectId: string, profileId: string): Promise; + artifactCount(): Promise; + cleanup(): Promise; +} + +export interface HarnessOptions { + profiles: PlatformProfile[]; + renderer: MediaRenderer; + motion: MotionGraphicsRenderer; + inference?: MediaInferenceProvider | null; + now?: () => Date; +} + +const unusedPublisher: MediaPublisher = { + id: "unused", + publish: async () => { + throw new Error("not used by render tests"); + }, +}; + +/** PostgreSQL when VERIFY_DATABASE and DATABASE_URL are set, otherwise in memory. */ +export async function createRenderHarness(options: HarnessOptions): Promise { + const dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-render-"))); + const mediaRoot = path.join(dir, "media"); + await mkdir(mediaRoot, { recursive: true }); + const artifacts = new LocalArtifactStore(path.join(dir, "store")); + const events: MediaRealtimeEvent[] = []; + const enqueued: string[] = []; + const jobs: JobPublisher = { + enqueue: async (job) => { + enqueued.push(`${job.name}:${JSON.stringify(job.payload)}`); + }, + cancel: async () => undefined, + close: async () => undefined, + }; + const databaseUrl = process.env.DATABASE_URL; + const backend = + process.env.VERIFY_DATABASE && databaseUrl + ? await postgresBackend(databaseUrl) + : memoryBackend(options.now ?? (() => new Date())); + const sources = new Map(); + const deps: MediaServiceDeps = { + prisma: backend.prisma, + repos: backend.repos, + jobs, + inference: options.inference ?? null, + renderer: options.renderer, + motion: options.motion, + publisher: unusedPublisher, + shortener: null as LinkShortener | null, + sources: {}, + mediaRoot, + profiles: async () => ({ + profiles: new Map( + options.profiles.map((profile) => [profile.id, { profile, origin: "builtin" as const }]), + ), + errors: [], + }), + artifacts, + model: { generate: async () => null }, + publish: async (_spaceId, event) => { + events.push(event); + }, + now: options.now ?? (() => new Date()), + }; + const context = (userId: string) => ({ + operationId: "test", + traceId: "test", + spaceId: backend.spaceId, + userId, + signal: new AbortController().signal, + }); + + return { + backend: backend.kind, + mediaRoot, + spaceId: backend.spaceId, + userId: backend.userId, + deps, + events, + enqueued, + sources, + createAsset: (name) => backend.createAsset(name), + assetPath: (assetId, fileName) => + path.join(mediaRoot, backend.spaceId, "assets", assetId, fileName), + setSource: (assetId, source) => sources.set(assetId, source), + addAnalysis: (assetId, kind, data) => backend.addAnalysis(assetId, kind, data), + addSegments: (assetId, ranges) => backend.addSegments(assetId, ranges), + async addArtifact(file, name, mimeType) { + const bytes = new Uint8Array(await readFile(file)); + const stored = await artifacts.put({ name, mimeType, bytes }, context(backend.userId)); + const row = await backend.prisma.artifact.create({ + data: { + spaceId: backend.spaceId, + userId: backend.userId, + name, + mimeType, + size: bytes.byteLength, + hash: "test", + storageKey: stored.id, + }, + select: { id: true }, + }); + return row.id; + }, + createProject: (input) => backend.createProject(input), + createPlan: (projectId, edl) => { + const parsed = EdlSchema.parse(edl); + return backend.repos.createPlanVersion(backend.spaceId, { + projectId, + profileId: parsed.output.profileId, + edl: parsed, + createdBy: "user", + rationale: null, + }); + }, + planVersions: (projectId, profileId) => + backend.repos.listPlanVersions(backend.spaceId, projectId, profileId), + artifactCount: () => backend.prisma.artifact.count({ where: { spaceId: backend.spaceId } }), + async cleanup() { + await backend.cleanup(); + await rm(dir, { recursive: true, force: true }); + }, + }; +} + +interface Backend { + kind: "memory" | "postgres"; + prisma: PrismaClient; + repos: MediaRepos; + spaceId: string; + userId: string; + createAsset(name: string): Promise; + addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise; + addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise; + createProject(input: { assetIds: string[]; kit: BrandKit | null; profileId: string }): Promise; + cleanup(): Promise; +} + +async function postgresBackend(databaseUrl: string): Promise { + const db = createDb(databaseUrl, { poolMax: 4 }); + const prisma = db.prisma; + const repos = createMediaRepos(prisma); + const userId = `mr${randomUUID().replaceAll("-", "")}`; + await prisma.user.create({ data: { id: userId, name: "Test", email: `${userId}@rakazo.test` } }); + const { spaceId } = await bootstrapUserSpace( + prisma, + { id: userId }, + { signupsEnabled: "true", signupAllowlist: "" }, + { claimDeploymentOwner: false }, + ); + const source = await repos.createSource(spaceId, { + kind: "folder", + label: "Synthetic", + ownerUserId: userId, + config: {}, + }); + let analysisVersion = 0; + return { + kind: "postgres", + prisma, + repos, + spaceId, + userId, + async createAsset(name) { + const result = await repos.upsertSourceItems(spaceId, source.id, [ + { + externalId: name, + kind: "video", + capturedAt: new Date(Date.UTC(2026, 0, 1)), + durationMs: 8000, + width: 640, + height: 360, + fps: 30, + gpsLat: null, + gpsLon: null, + deviceModel: null, + albumNames: [], + isFavorite: false, + fileName: name, + sizeBytes: 1, + revision: "r1", + }, + ]); + return result.changedAssetIds[0]!; + }, + async addAnalysis(assetId, kind, data) { + analysisVersion += 1; + await repos.upsertAnalysis(spaceId, assetId, { + kind, + modelId: "emulator", + modelVersion: `v${analysisVersion}`, + data: data as never, + }); + }, + async addSegments(assetId, ranges) { + await repos.replaceSegments( + spaceId, + assetId, + 1, + ranges.map((range) => ({ kind: "shot", ...range })), + ); + }, + async createProject(input) { + const kit = input.kit ? await repos.upsertBrandKit(spaceId, { kit: input.kit }) : null; + const project = await repos.createProject(spaceId, userId, { + title: "Synthetic project", + goal: "teaser", + profileIds: [input.profileId], + assetIds: input.assetIds, + brandKitId: kit?.id ?? null, + }); + return project.id; + }, + async cleanup() { + const memberships = await prisma.member.findMany({ + where: { userId }, + select: { organizationId: true }, + }); + await prisma.organization.deleteMany({ + where: { id: { in: memberships.map((member) => member.organizationId) } }, + }); + await prisma.user.deleteMany({ where: { id: userId } }); + await prisma.$disconnect(); + await db.pool.end(); + }, + }; +} + +interface RenderRow { + id: string; + spaceId: string; + projectId: string; + planId: string; + profileId: string; + quality: string; + status: string; + progress: number; + outputs: unknown[]; + gateReport: unknown; + ffprobe: unknown; + autoFixRound: number; + error: string | null; + createdBy: string; + createdAt: Date; + updatedAt: Date; +} + +/** In-memory stand-in for the repos the render slice uses, with the same state rules. */ +function memoryBackend(now: () => Date): Backend { + const spaceId = "space-memory"; + const userId = "user-memory"; + let counter = 0; + const id = (prefix: string) => `${prefix}${(++counter).toString().padStart(6, "0")}`; + const assets = new Set(); + const analyses = new Map(); + const segments: Array<{ + id: string; + assetId: string; + startMs: number; + endMs: number; + paidOnly: boolean; + }> = []; + const kits = new Map(); + const projects = new Map< + string, + { id: string; assetIds: string[]; collectionIds: string[]; brandKitId: string | null } + >(); + const plans: Array[0]> = []; + const renders = new Map(); + const artifacts: Array & { id: string; spaceId: string }> = []; + + const requireRender = (space: string, renderId: string) => { + const row = renders.get(renderId); + if (!row || row.spaceId !== space) throw new IsolationError(); + return row; + }; + const requirePlan = (space: string, planId: string) => { + const row = plans.find((plan) => plan.id === planId && plan.spaceId === space); + if (!row) throw new IsolationError(); + return row; + }; + const toRender = (row: RenderRow): MediaRender => + mapMediaRender(structuredClone(row) as unknown as Parameters[0]); + + const repos = { + async mediaSpaceOf(kind: string, renderId: string) { + return kind === "render" ? (renders.get(renderId)?.spaceId ?? null) : null; + }, + async getPlan(space: string, planId: string) { + return mapMediaEditPlan(requirePlan(space, planId)); + }, + async createPlanVersion( + space: string, + input: { + projectId: string; + profileId: string; + edl: unknown; + createdBy: "agent" | "user"; + rationale: string | null; + }, + ) { + const edl = EdlSchema.parse(input.edl); + if (edl.output.profileId !== input.profileId) throw new Error("profile mismatch"); + if (space !== spaceId || !projects.has(input.projectId)) throw new IsolationError(); + const version = + plans + .filter((p) => p.projectId === input.projectId && p.profileId === input.profileId) + .reduce((max, p) => Math.max(max, p.version), 0) + 1; + const row = { + id: id("plan"), + spaceId: space, + projectId: input.projectId, + profileId: input.profileId, + version, + edl: structuredClone(edl), + createdBy: input.createdBy, + rationale: input.rationale, + createdAt: now(), + } as unknown as Parameters[0]; + plans.push(row); + return mapMediaEditPlan(row); + }, + async listPlanVersions(space: string, projectId: string, profileId: string) { + return plans + .filter((p) => p.spaceId === space && p.projectId === projectId && p.profileId === profileId) + .sort((a, b) => b.version - a.version) + .map(mapMediaEditPlan); + }, + async getProjectRecord(space: string, projectId: string) { + const project = projects.get(projectId); + if (!project || space !== spaceId) throw new IsolationError(); + return { ...project, createdBy: userId }; + }, + async getBrandKit(space: string, kitId: string) { + const kit = kits.get(kitId); + if (!kit || space !== spaceId) throw new IsolationError(); + return { id: kitId, kit, createdAt: now().toISOString(), updatedAt: now().toISOString() }; + }, + async collectionAssetIds() { + return []; + }, + async listSegments(space: string, assetIds: string[]) { + if (space !== spaceId) return []; + return segments.filter((segment) => assetIds.includes(segment.assetId)); + }, + async latestAnalysis(space: string, assetId: string, kind: string) { + if (space !== spaceId || !assets.has(assetId)) throw new IsolationError(); + const data = analyses.get(`${assetId}:${kind}`); + return data === undefined ? null : { data }; + }, + async createRender( + space: string, + input: { planId: string; quality: string; createdBy: string }, + ) { + const plan = requirePlan(space, input.planId); + const row: RenderRow = { + id: id("render"), + spaceId: space, + projectId: plan.projectId, + planId: plan.id, + profileId: plan.profileId, + quality: input.quality, + status: "queued", + progress: 0, + outputs: [], + gateReport: null, + ffprobe: null, + autoFixRound: 0, + error: null, + createdBy: input.createdBy, + createdAt: now(), + updatedAt: now(), + }; + renders.set(row.id, row); + return toRender(row); + }, + async getRender(space: string, renderId: string) { + return toRender(requireRender(space, renderId)); + }, + async getRenderRecord(space: string, renderId: string) { + return structuredClone(requireRender(space, renderId)); + }, + async claimRender(space: string, renderId: string) { + const row = requireRender(space, renderId); + if (row.status !== "queued") return null; + Object.assign(row, { status: "rendering", progress: 0, error: null, updatedAt: now() }); + return toRender(row); + }, + async updateRender( + space: string, + renderId: string, + patch: MediaRenderPatch, + from = ACTIVE_RENDER_STATUSES, + ) { + const row = requireRender(space, renderId); + if (!from.includes(row.status as never)) return null; + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) continue; + (row as unknown as Record)[key] = + key === "progress" ? Math.min(1, Math.max(0, value as number)) : structuredClone(value); + } + row.updatedAt = now(); + return toRender(row); + }, + async cancelRender(space: string, renderId: string) { + const row = requireRender(space, renderId); + if (!ACTIVE_RENDER_STATUSES.includes(row.status as never)) { + throw new MediaStateConflictError(`Render is already ${row.status}`); + } + row.status = "canceled"; + row.updatedAt = now(); + return toRender(row); + }, + }; + + const prisma = { + artifact: { + async findFirst(args: { where: { id: string; spaceId: string } }) { + return ( + artifacts.find((a) => a.id === args.where.id && a.spaceId === args.where.spaceId) ?? null + ); + }, + async create(args: { data: Record & { spaceId: string } }) { + const row = { ...args.data, id: id("artifact") }; + artifacts.push(row); + return row; + }, + async deleteMany(args: { where: { id: string; spaceId: string } }) { + const index = artifacts.findIndex( + (a) => a.id === args.where.id && a.spaceId === args.where.spaceId, + ); + if (index >= 0) artifacts.splice(index, 1); + return { count: index >= 0 ? 1 : 0 }; + }, + async count(args: { where: { spaceId: string } }) { + return artifacts.filter((a) => a.spaceId === args.where.spaceId).length; + }, + }, + }; + + return { + kind: "memory", + prisma: prisma as unknown as PrismaClient, + repos: repos as unknown as MediaRepos, + spaceId, + userId, + async createAsset() { + const assetId = id("asset"); + assets.add(assetId); + return assetId; + }, + async addAnalysis(assetId, kind, data) { + analyses.set(`${assetId}:${kind}`, data); + }, + async addSegments(assetId, ranges) { + for (const range of ranges) { + segments.push({ id: id("segment"), assetId, ...range, paidOnly: false }); + } + }, + async createProject(input) { + let brandKitId: string | null = null; + if (input.kit) { + brandKitId = id("kit"); + kits.set(brandKitId, input.kit); + } + const projectId = id("project"); + projects.set(projectId, { + id: projectId, + assetIds: input.assetIds, + collectionIds: [], + brandKitId, + }); + return projectId; + }, + async cleanup() {}, + }; +} + +// --------------------------------------------------------------------------- +// Privacy stand-in +// --------------------------------------------------------------------------- + +export interface FakePrivacy extends MediaPrivacyService { + gateCalls: Array<{ autoFixRound: number; edl: Edl; ffprobe: FfprobeReport }>; +} + +/** + * Pass-through sources (no redactions in these plans) and the deterministic + * gate on the probed output. `failWith` adds a blocking content issue to every + * report, like a classifier hit at the start of the output. + */ +export function createFakePrivacy( + sources: Map, + options: { failWith?: GateIssue; now?: () => Date } = {}, +): FakePrivacy { + const gateCalls: FakePrivacy["gateCalls"] = []; + const unsupported = async (): Promise => { + throw new Error("not used by render tests"); + }; + return { + gateCalls, + async prepareSources(input): Promise { + const prepared = new Map(); + for (const clip of input.edl.tracks.video) { + const source = sources.get(clip.assetId); + if (!source) throw new Error(`no source for ${clip.assetId}`); + prepared.set(clip.assetId, source); + } + return { sources: prepared, aiAlterations: [], fallbacks: [] }; + }, + async gate(input): Promise { + gateCalls.push({ autoFixRound: input.autoFixRound, edl: input.edl, ffprobe: input.ffprobe }); + const report = evaluateGate({ + profile: input.profile, + edl: input.edl, + ffprobe: input.ffprobe, + content: [], + faces: [], + trackingGaps: [], + marks: [], + people: [], + copy: input.copy, + aiAlterations: [], + bundleHasDisclosure: input.bundleHasDisclosure, + watermarkBox: null, + sampledFrames: 0, + autoFixRound: input.autoFixRound, + now: (options.now ?? (() => new Date()))(), + }); + if (!options.failWith) return report; + return { ...report, passed: false, issues: [...report.issues, options.failWith] }; + }, + createMark: unsupported, + updateMark: unsupported, + removePerson: unsupported, + }; +} diff --git a/packages/adapters/src/media-render.ts b/packages/adapters/src/media-render.ts new file mode 100644 index 0000000..32fd240 --- /dev/null +++ b/packages/adapters/src/media-render.ts @@ -0,0 +1,835 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { AdapterContext, MediaJobHandlers, MediaRenderProgress } from "@rakazo/adapter-kit"; +import { mediaRenderJob, UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import type { + AiAlteration, + BrandKit, + Edl, + GateReport, + MediaRender, + MediaRenderOutput, + MediaRenderQuality, + PlatformProfile, + RenderPlan, +} from "@rakazo/contracts"; +import { MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import type { MediaRenderPatch } from "@rakazo/db"; +import { InvalidMediaInputError } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import type { CaptionWord } from "@rakazo/media"; +import { compileRenderPlan, findMetadataLeaks, planAutoFix } from "@rakazo/media"; +import { resolveInside } from "@rakazo/media/node"; +import type { + MediaActor, + MediaPrivacyService, + MediaRenderService, + MediaServiceDeps, +} from "./media-context.js"; +import { MediaToolMissingError } from "./media-folder-source-probe.js"; + +/* + * Render service (API side) and render job (worker side). + * + * A render runs: claim → prepare redacted sources (privacy slice) → pre-render + * motion clips → compile → ffmpeg → probe → gate. A blocked gate gets up to + * MAX_AUTO_FIX_ROUNDS compliance-only repairs (planAutoFix); each repair is + * stored as a new plan version by the agent and rendered again under the same + * render id. Only a passed gate moves outputs out of the work directory and + * marks the render done; every other outcome removes the render's files. + * + * Layout: `/renders//{video.mp4,poster.jpg,captions.srt, + * captions.vtt}` with intermediates in `work/` beside them. + * + * Cancel: the API marks the render canceled and aborts it when this process + * runs it. A worker in another process notices through a failed status update + * (updateRender returns null once canceled) or the periodic status check. + */ + +/** Renders running in this process, so a cancel can kill ffmpeg immediately. */ +export type RunningMediaRenders = Map; + +const runningRenders: RunningMediaRenders = new Map(); + +const PROGRESS_INTERVAL_MS = 1000; +const CANCEL_POLL_MS = 2000; +/** Share of the progress bar spent rendering; probing and gating take the rest. */ +const RENDER_SHARE = 0.85; + +const OUTPUT_FILES = { + video: "video.mp4", + poster: "poster.jpg", + subtitles_srt: "captions.srt", + subtitles_vtt: "captions.vtt", +} as const; + +const OUTPUT_MIME: Record = { + video: "video/mp4", + poster: "image/jpeg", + subtitles_srt: "application/x-subrip", + subtitles_vtt: "text/vtt", +}; + +const EXTENSION_BY_MIME: Record = { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/svg+xml": ".svg", + "image/webp": ".webp", + "audio/mpeg": ".mp3", + "audio/wav": ".wav", + "audio/x-wav": ".wav", + "audio/aac": ".aac", + "audio/mp4": ".m4a", + "audio/ogg": ".ogg", + "audio/flac": ".flac", + "font/ttf": ".ttf", + "font/otf": ".otf", + "font/collection": ".ttc", +}; + +/** A failure whose message is safe to show the user (no paths, no tool output). */ +export class MediaRenderFailure extends Error { + constructor(message: string) { + super(message); + this.name = "MediaRenderFailure"; + } +} + +class RenderCanceled extends Error { + constructor() { + super("render canceled"); + this.name = "RenderCanceled"; + } +} + +function userMessage(error: unknown): string { + if (error instanceof MediaRenderFailure) return error.message; + if (error instanceof UnsupportedMediaOperationError) { + return `${error.operation} is not available on this server.`; + } + if (error instanceof MediaToolMissingError) return "ffmpeg is not installed on the media worker."; + return "Rendering failed. Check the plan and try again."; +} + +function renderEvent(render: MediaRender) { + return { + type: "media.render" as const, + renderId: render.id, + projectId: render.projectId, + status: render.status, + progress: render.progress, + }; +} + +async function publishRender(deps: MediaServiceDeps, spaceId: string, render: MediaRender) { + try { + await deps.publish(spaceId, renderEvent(render)); + } catch (error) { + getLogger().warn("media render event not published", { + renderId: render.id, + reason: error instanceof Error ? error.message : String(error), + }); + } +} + +// --------------------------------------------------------------------------- +// Service +// --------------------------------------------------------------------------- + +export function createMediaRenderService( + deps: MediaServiceDeps, + options: { running?: RunningMediaRenders } = {}, +): MediaRenderService { + const running = options.running ?? runningRenders; + return { + async start(actor, input) { + const plan = await deps.repos.getPlan(actor.spaceId, input.planId); + const { profiles } = await deps.profiles(); + if (!profiles.has(plan.profileId)) { + throw new InvalidMediaInputError("The plan's platform profile is not available"); + } + const render = await deps.repos.createRender(actor.spaceId, { + planId: plan.id, + quality: input.quality, + createdBy: actor.userId, + }); + await deps.jobs.enqueue(mediaRenderJob(render.id)); + await publishRender(deps, actor.spaceId, render); + return render; + }, + + async cancel(actor, renderId) { + const render = await deps.repos.cancelRender(actor.spaceId, renderId); + running.get(renderId)?.abort(); + await deps.jobs.cancel(mediaRenderJob(renderId).replaceKey!).catch((error: unknown) => { + getLogger().warn("media render job not dequeued", { + renderId, + reason: error instanceof Error ? error.message : String(error), + }); + }); + await publishRender(deps, actor.spaceId, render); + return render; + }, + + async outputPath(actor, renderId, fileName) { + const record = await deps.repos.getRenderRecord(actor.spaceId, renderId); + if (record.status !== "done") return null; + if (path.basename(fileName) !== fileName || fileName.startsWith(".")) return null; + const outputs = Array.isArray(record.outputs) ? record.outputs : []; + const recorded = outputs.some( + (output) => + output !== null && + typeof output === "object" && + !Array.isArray(output) && + output.fileName === fileName, + ); + if (!recorded) return null; + // Derive the location instead of trusting a stored path. + const file = await resolveInside( + deps.mediaRoot, + `${actor.spaceId}/renders/${renderId}/${fileName}`, + ); + const found = await stat(file).catch(() => null); + return found?.isFile() ? file : null; + }, + }; +} + +// --------------------------------------------------------------------------- +// Job handler +// --------------------------------------------------------------------------- + +interface BrandFiles { + /** Font artifact id → file. */ + fonts: Map; + fontList: Array<{ family: string; path: string }>; + logos: string[]; + watermark: string | null; + lut: string | null; +} + +function safeExtension(name: string, mimeType: string): string { + const fromName = path.extname(name).toLowerCase(); + if (/^\.[a-z0-9]{1,5}$/.test(fromName)) return fromName; + return EXTENSION_BY_MIME[mimeType.toLowerCase()] ?? ".bin"; +} + +const even = (value: number) => Math.max(2, Math.floor(value / 2) * 2); + +function finiteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +/** Transcript words from a stored TranscribeResult. */ +export function transcriptWords(data: unknown): CaptionWord[] { + const words = (data as { words?: unknown } | null)?.words; + if (!Array.isArray(words)) return []; + return words.flatMap((word) => { + const w = word as { startMs?: unknown; endMs?: unknown; text?: unknown }; + if (!finiteNumber(w.startMs) || !finiteNumber(w.endMs) || typeof w.text !== "string") { + return []; + } + return [{ startMs: w.startMs, endMs: w.endMs, text: w.text }]; + }); +} + +type Subject = { tMs: number; cx: number; cy: number; strength: number }; + +/** Reframe subjects from a stored saliency result, or face centres from detections. */ +export function subjectPoints(saliency: unknown, faces: unknown): Subject[] { + const points = (saliency as { points?: unknown } | null)?.points; + if (Array.isArray(points)) { + const parsed = points.flatMap((point) => { + const p = point as Partial; + return finiteNumber(p.tMs) && + finiteNumber(p.cx) && + finiteNumber(p.cy) && + finiteNumber(p.strength) + ? [{ tMs: p.tMs, cx: p.cx, cy: p.cy, strength: p.strength }] + : []; + }); + if (parsed.length > 0) return parsed; + } + const frames = (faces as { frames?: unknown } | null)?.frames; + if (!Array.isArray(frames)) return []; + return frames.flatMap((frame) => { + const f = frame as { tMs?: unknown; faces?: unknown }; + if (!finiteNumber(f.tMs) || !Array.isArray(f.faces)) return []; + const tMs = f.tMs; + return f.faces.flatMap((face) => { + const box = (face as { box?: { x?: unknown; y?: unknown; w?: unknown; h?: unknown } }).box; + const confidence = (face as { confidence?: unknown }).confidence; + if (!box || !finiteNumber(box.x) || !finiteNumber(box.y)) return []; + if (!finiteNumber(box.w) || !finiteNumber(box.h)) return []; + return [ + { + tMs, + cx: box.x + box.w / 2, + cy: box.y + box.h / 2, + strength: finiteNumber(confidence) ? confidence : 1, + }, + ]; + }); + }); +} + +export interface MediaRenderHandlerOptions { + running?: RunningMediaRenders; + /** How often a running render checks whether another process canceled it. */ + cancelPollMs?: number; +} + +/** + * Worker handler for `media.render`. Never throws for a render failure: the + * render record carries the outcome, so a job retry has nothing to redo. + */ +export function createMediaRenderHandler( + deps: MediaServiceDeps, + privacy: MediaPrivacyService, + options: MediaRenderHandlerOptions = {}, +): MediaJobHandlers["media.render"] { + const running = options.running ?? runningRenders; + const pollMs = options.cancelPollMs ?? CANCEL_POLL_MS; + const { repos } = deps; + + return async ({ renderId }) => { + const spaceId = await repos.mediaSpaceOf("render", renderId); + if (!spaceId) return; + const claimed = await repos.claimRender(spaceId, renderId); + if (!claimed) return; + await publishRender(deps, spaceId, claimed); + + const controller = new AbortController(); + running.set(renderId, controller); + const poll = setInterval(() => { + repos + .getRender(spaceId, renderId) + .then((render) => { + if (render.status === "canceled") controller.abort(); + }) + .catch(() => undefined); + }, pollMs); + poll.unref?.(); + const createdArtifacts: Array<{ id: string; storageKey: string }> = []; + let renderDir: string | null = null; + let userId = ""; + + try { + renderDir = await resolveInside(deps.mediaRoot, `${spaceId}/renders/${renderId}`); + const record = await repos.getRenderRecord(spaceId, renderId); + userId = record.createdBy; + await runRender({ + deps, + privacy, + spaceId, + renderId, + renderDir, + record, + signal: controller.signal, + abort: () => controller.abort(), + createdArtifacts, + }); + } catch (error) { + if (renderDir) await rm(renderDir, { recursive: true, force: true }).catch(() => undefined); + await removeArtifacts(deps, spaceId, userId, renderId, createdArtifacts); + const current = await repos.getRender(spaceId, renderId).catch(() => null); + if (current?.status === "canceled") { + await publishRender(deps, spaceId, current); + return; + } + getLogger().error("media render failed", { + renderId, + reason: error instanceof Error ? error.message : String(error), + }); + const failed = await repos + .updateRender(spaceId, renderId, { status: "failed", error: userMessage(error) }) + .catch(() => null); + if (failed) await publishRender(deps, spaceId, failed); + } finally { + clearInterval(poll); + running.delete(renderId); + } + }; +} + +function artifactContext(spaceId: string, userId: string, renderId: string): AdapterContext { + return { + operationId: `media-render:${renderId}`, + traceId: `media-render:${renderId}`, + spaceId, + userId, + signal: new AbortController().signal, + }; +} + +async function removeArtifacts( + deps: MediaServiceDeps, + spaceId: string, + userId: string, + renderId: string, + created: Array<{ id: string; storageKey: string }>, +) { + for (const artifact of created) { + await deps.prisma.artifact + .deleteMany({ where: { id: artifact.id, spaceId } }) + .catch(() => undefined); + await deps.artifacts + .remove(artifact.storageKey, artifactContext(spaceId, userId, renderId)) + .catch(() => undefined); + } +} + +type RenderRecord = Awaited>; + +interface RunInput { + deps: MediaServiceDeps; + privacy: MediaPrivacyService; + spaceId: string; + renderId: string; + renderDir: string; + record: RenderRecord; + signal: AbortSignal; + abort: () => void; + createdArtifacts: Array<{ id: string; storageKey: string }>; +} + +async function runRender(input: RunInput): Promise { + const { deps, privacy, spaceId, renderId, renderDir, record, signal } = input; + const { repos } = deps; + const actor: MediaActor = { spaceId, userId: record.createdBy }; + const context = artifactContext(spaceId, actor.userId, renderId); + + /** Apply a patch while the render is active; a null result means it was canceled. */ + const update = async (patch: MediaRenderPatch) => { + const updated = await repos.updateRender(spaceId, renderId, patch); + if (!updated) { + input.abort(); + throw new RenderCanceled(); + } + return updated; + }; + const checkCanceled = () => { + if (signal.aborted) throw new RenderCanceled(); + }; + + let plan = await repos.getPlan(spaceId, record.planId); + const project = await repos.getProjectRecord(spaceId, record.projectId); + const entry = (await deps.profiles()).profiles.get(record.profileId); + if (!entry) throw new MediaRenderFailure("The platform profile is no longer available."); + const profile = entry.profile; + const kit = project.brandKitId + ? (await repos.getBrandKit(spaceId, project.brandKitId)).kit + : null; + const quality = record.quality as MediaRenderQuality; + + // A retried job starts from a clean work directory. + const workDir = path.join(renderDir, "work"); + await rm(renderDir, { recursive: true, force: true }); + await mkdir(workDir, { recursive: true }); + + const fetchArtifact = async (artifactId: string, dir: string, baseName: string) => { + const row = await deps.prisma.artifact.findFirst({ + where: { id: artifactId, spaceId }, + select: { storageKey: true, name: true, mimeType: true }, + }); + if (!row) return null; + const bytes = await deps.artifacts.get(row.storageKey, context); + await mkdir(dir, { recursive: true }); + const file = path.join(dir, `${baseName}${safeExtension(row.name, row.mimeType)}`); + await writeFile(file, bytes); + return file; + }; + const brand = await loadBrandFiles(kit, path.join(workDir, "brand"), fetchArtifact); + const audioFiles = new Map(); + + const alternatives = await alternativeSegments(deps, spaceId, project, plan.edl); + let round = record.autoFixRound; + let edl = plan.edl; + + for (;;) { + checkCanceled(); + const roundDir = path.join(workDir, `round-${round}`); + await mkdir(roundDir, { recursive: true }); + + const prepared = await privacy.prepareSources({ + actor, + projectId: project.id, + edl, + quality, + workDir: roundDir, + signal, + }); + checkCanceled(); + + for (const clip of edl.tracks.audio) { + if (audioFiles.has(clip.artifactId)) continue; + const file = await fetchArtifact( + clip.artifactId, + path.join(workDir, "audio"), + `audio-${audioFiles.size + 1}`, + ); + if (!file) throw new MediaRenderFailure("An audio file of the plan is no longer available."); + audioFiles.set(clip.artifactId, file); + } + + const motionClips = await renderMotionClips(deps, edl, kit, brand, roundDir, signal); + const { transcripts, subjects } = await analysisInputs(deps, spaceId, edl); + const videoEncoder = await pickEncoder(deps, profile, quality); + + const renderPlan = compileRenderPlan({ + edl, + profile, + brandKit: kit, + quality, + workDir: roundDir, + sources: prepared.sources, + transcripts, + subjects, + motionClips, + files: { + lutPath: brand.lut, + watermarkPath: brand.watermark, + fonts: brand.fonts, + audio: audioFiles, + logoPath: brand.logos[0] ?? null, + }, + videoEncoder, + }); + + await executePlan(input, renderPlan, update); + + const video = renderPlan.outputs.find((output) => output.kind === "video"); + if (!video) throw new Error("render plan has no video output"); + const ffprobe = await deps.renderer.probe(video.path, { measureLoudness: true, signal }); + if (findMetadataLeaks(ffprobe.tags).length > 0) { + // Spec 5: a render whose metadata could not be removed is failed, not gated. + throw new MediaRenderFailure("The export still carried source metadata."); + } + const gating = await update({ status: "gating", progress: 0.9, ffprobe }); + await publishRender(deps, spaceId, gating); + + const aiAlterations = [ + ...new Set([...edl.metadata.aiAlterations, ...prepared.aiAlterations]), + ]; + const rendered: Edl = { + ...edl, + metadata: { + ...edl.metadata, + aiAlterations, + aiAltered: edl.metadata.aiAltered || aiAlterations.length > 0, + }, + }; + const report = await privacy.gate({ + actor, + projectId: project.id, + edl: rendered, + profile, + outputPath: video.path, + ffprobe, + copy: { + title: edl.metadata.title ?? null, + caption: edl.metadata.description ?? null, + tags: edl.metadata.tags, + }, + // The publish bundle always carries the AI disclosure file and checklist + // entry when the profile requires a label and the output is altered. + bundleHasDisclosure: profile.disclosure.aiAlteredLabelRequired && aiAlterations.length > 0, + autoFixRound: round, + signal, + }); + checkCanceled(); + + if (report.passed) { + await finish(input, renderPlan, report, plan.id, update, context); + return; + } + + const fix = + round < MAX_AUTO_FIX_ROUNDS + ? planAutoFix(edl, { ...report, autoFixRound: round }, alternatives) + : null; + if (!fix) { + await rm(renderDir, { recursive: true, force: true }); + const blocked = await update({ status: "blocked", progress: 1, gateReport: report }); + await publishRender(deps, spaceId, blocked); + return; + } + const codes = [ + ...new Set(report.issues.filter((i) => i.severity === "block").map((i) => i.code)), + ]; + plan = await repos.createPlanVersion(spaceId, { + projectId: project.id, + profileId: plan.profileId, + edl: fix.edl, + createdBy: "agent", + rationale: `Auto-fix ${round + 1}/${MAX_AUTO_FIX_ROUNDS} after the gate blocked (${codes.join( + ", ", + )}): ${fix.changes.join("; ")}`, + }); + round += 1; + edl = plan.edl; + await rm(roundDir, { recursive: true, force: true }); + const retry = await update({ + status: "rendering", + progress: 0, + autoFixRound: round, + gateReport: report, + ffprobe: null, + }); + await publishRender(deps, spaceId, retry); + } +} + +async function loadBrandFiles( + kit: BrandKit | null, + dir: string, + fetchArtifact: (artifactId: string, dir: string, baseName: string) => Promise, +): Promise { + const files: BrandFiles = { + fonts: new Map(), + fontList: [], + logos: [], + watermark: null, + lut: null, + }; + if (!kit) return files; + // Missing files are skipped here; the compiler fails only if the plan needs one. + const missing = (what: string) => getLogger().warn("brand kit file unavailable", { what }); + for (const [index, font] of kit.fonts.entries()) { + const file = await fetchArtifact(font.artifactId, dir, `font-${index + 1}`); + if (!file) { + missing("font"); + continue; + } + files.fonts.set(font.artifactId, file); + files.fontList.push({ family: font.family, path: file }); + } + for (const [index, logo] of kit.logoArtifactIds.entries()) { + const file = await fetchArtifact(logo, dir, `logo-${index + 1}`); + if (file) files.logos.push(file); + else missing("logo"); + } + const watermarkId = kit.watermark?.artifactId; + if (watermarkId) { + files.watermark = await fetchArtifact(watermarkId, dir, "watermark"); + if (!files.watermark) missing("watermark"); + } else if (kit.watermark) { + files.watermark = files.logos[0] ?? null; + } + if (kit.lutArtifactId) { + files.lut = await fetchArtifact(kit.lutArtifactId, dir, "look"); + if (!files.lut) missing("lut"); + } + return files; +} + +async function renderMotionClips( + deps: MediaServiceDeps, + edl: Edl, + kit: BrandKit | null, + brand: BrandFiles, + dir: string, + signal: AbortSignal, +): Promise> { + const W = edl.output.width; + const H = edl.output.height; + const jobs: Array<{ key: string; template: NonNullable; w: number; h: number }> = + []; + if (edl.intro && edl.intro.durationMs > 0) + jobs.push({ key: "intro", template: edl.intro, w: W, h: H }); + if (edl.outro && edl.outro.durationMs > 0) + jobs.push({ key: "outro", template: edl.outro, w: W, h: H }); + for (const overlay of edl.overlays) { + if (!overlay.motion) continue; + const box = overlay.box; + jobs.push({ + key: overlay.id, + template: overlay.motion, + w: box ? even(box.w * W) : W, + h: box ? even(box.h * H) : H, + }); + } + const clips = new Map(); + if (jobs.length === 0) return clips; + if (!kit) throw new MediaRenderFailure("Motion templates need a brand kit on the project."); + for (const job of jobs) { + // Keys are SafeId values ([a-z0-9_-]), so they are safe file names. + const result = await deps.motion.render({ + template: job.template, + brandKit: kit, + assets: { fonts: brand.fontList, logoPaths: brand.logos }, + width: job.w, + height: job.h, + fps: edl.output.fps, + outPath: path.join(dir, `motion-${job.key}.mov`), + signal, + }); + clips.set(job.key, result.path); + } + return clips; +} + +async function analysisInputs(deps: MediaServiceDeps, spaceId: string, edl: Edl) { + const transcripts = new Map(); + const subjects = new Map(); + const assetIds = [...new Set(edl.tracks.video.map((clip) => clip.assetId))]; + for (const assetId of assetIds) { + const [transcript, saliency, faces] = await Promise.all([ + deps.repos.latestAnalysis(spaceId, assetId, "transcript"), + deps.repos.latestAnalysis(spaceId, assetId, "saliency"), + deps.repos.latestAnalysis(spaceId, assetId, "faces"), + ]); + const words = transcriptWords(transcript?.data ?? null); + if (words.length > 0) transcripts.set(assetId, words); + const points = subjectPoints(saliency?.data ?? null, faces?.data ?? null); + if (points.length > 0) subjects.set(assetId, points); + } + return { transcripts, subjects }; +} + +/** Final renders prefer a working hardware encoder of the profile's codec; drafts use software. */ +async function pickEncoder( + deps: MediaServiceDeps, + profile: PlatformProfile, + quality: MediaRenderQuality, +): Promise { + const family = profile.video.codec; + const software = family === "h264" ? "libx264" : "libx265"; + if (quality === "draft") return software; + const hardware = await deps.renderer.hardwareEncoders(); + return hardware.find((encoder) => encoder.startsWith(`${family}_`)) ?? software; +} + +/** Segments the auto-fix may swap in: the project's material, never paid-only. */ +async function alternativeSegments( + deps: MediaServiceDeps, + spaceId: string, + project: { assetIds: string[]; collectionIds: string[] }, + edl: Edl, +) { + const fromCollections = await deps.repos.collectionAssetIds(spaceId, project.collectionIds); + const assetIds = [ + ...new Set([ + ...project.assetIds, + ...fromCollections, + ...edl.tracks.video.map((clip) => clip.assetId), + ]), + ]; + const segments = await deps.repos.listSegments(spaceId, assetIds); + return segments + .filter((segment) => !segment.paidOnly) + .map((segment) => ({ + segmentId: segment.id, + assetId: segment.assetId, + startMs: segment.startMs, + endMs: segment.endMs, + })); +} + +async function executePlan( + input: RunInput, + plan: RenderPlan, + update: (patch: MediaRenderPatch) => Promise, +) { + const { deps, spaceId, signal } = input; + let lastWrite = Number.NEGATIVE_INFINITY; + let writing: Promise | null = null; + let failure: unknown = null; + const onProgress = (progress: MediaRenderProgress) => { + const now = deps.now().getTime(); + if (writing || now - lastWrite < PROGRESS_INTERVAL_MS) return; + lastWrite = now; + writing = update({ progress: progress.progress * RENDER_SHARE }) + .then((render) => publishRender(deps, spaceId, render)) + .catch((error: unknown) => { + failure ??= error; + }) + .finally(() => { + writing = null; + }); + }; + try { + await deps.renderer.render(plan, { signal, onProgress }); + } finally { + await writing; + } + if (failure) throw failure; + if (signal.aborted) throw new RenderCanceled(); +} + +async function finish( + input: RunInput, + plan: RenderPlan, + report: GateReport, + planId: string, + update: (patch: MediaRenderPatch) => Promise, + context: AdapterContext, +) { + const { deps, spaceId, renderId, renderDir } = input; + const outputs: Array = []; + for (const output of plan.outputs) { + const fileName = OUTPUT_FILES[output.kind]; + const relative = `${spaceId}/renders/${renderId}/${fileName}`; + const destination = await resolveInside(deps.mediaRoot, relative); + await rename(output.path, destination); + const { size } = await stat(destination); + let artifactId: string | null = null; + if (output.kind !== "video") { + artifactId = await registerArtifact(input, context, { + name: fileName, + mimeType: OUTPUT_MIME[output.kind], + file: destination, + }); + } + outputs.push({ + kind: output.kind, + artifactId, + fileName, + sizeBytes: size, + path: relative, + planId, + }); + } + const done = await update({ + status: "done", + progress: 1, + outputs, + gateReport: report, + error: null, + }); + await rm(path.join(renderDir, "work"), { recursive: true, force: true }); + await publishRender(deps, spaceId, done); +} + +async function registerArtifact( + input: RunInput, + context: AdapterContext, + file: { name: string; mimeType: string; file: string }, +): Promise { + const { deps, spaceId } = input; + const bytes = new Uint8Array(await readFile(file.file)); + const stored = await deps.artifacts.put( + { name: file.name, mimeType: file.mimeType, bytes }, + context, + ); + try { + const row = await deps.prisma.artifact.create({ + data: { + spaceId, + userId: context.userId, + name: file.name, + mimeType: file.mimeType, + size: bytes.byteLength, + hash: createHash("sha256").update(bytes).digest("hex"), + storageKey: stored.id, + }, + select: { id: true }, + }); + input.createdArtifacts.push({ id: row.id, storageKey: stored.id }); + return row.id; + } catch (error) { + await deps.artifacts.remove(stored.id, context).catch(() => undefined); + throw error; + } +} From 3ebe0fe47bc99b2fc21fb97e2cbeb74f12886615 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:44:58 +0000 Subject: [PATCH 2/2] feat(media): ffmpeg renderer, render job and real render integration test Fail closed when anonymization cannot prepare every clip, retry a failing hardware encoder in software, verify each ffmpeg output exists, and fail renders whose video or poster still carries source metadata. Cover parsing, encoder detection, two-pass loudness, abort and cleanup, the render job (auto-fix, cancel, outputs) on memory and Postgres, and a synthetic 16:9 to 9:16 render plus a region blur check with ffprobe. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w --- .../src/media-ffmpeg-renderer.test.ts | 584 ++++++++++++++++++ .../adapters/src/media-ffmpeg-renderer.ts | 7 + .../adapters/src/media-render.fixtures.ts | 82 ++- packages/adapters/src/media-render.test.ts | 456 ++++++++++++++ packages/adapters/src/media-render.ts | 151 +++-- .../src/fixtures/media/synthetic-media.ts | 208 +++++++ .../testkit/src/media-render.docker.test.ts | 359 +++++++++++ 7 files changed, 1806 insertions(+), 41 deletions(-) create mode 100644 packages/adapters/src/media-ffmpeg-renderer.test.ts create mode 100644 packages/adapters/src/media-render.test.ts create mode 100644 packages/testkit/src/fixtures/media/synthetic-media.ts create mode 100644 packages/testkit/src/media-render.docker.test.ts diff --git a/packages/adapters/src/media-ffmpeg-renderer.test.ts b/packages/adapters/src/media-ffmpeg-renderer.test.ts new file mode 100644 index 0000000..97db4e2 --- /dev/null +++ b/packages/adapters/src/media-ffmpeg-renderer.test.ts @@ -0,0 +1,584 @@ +import { + chmod, + mkdir, + mkdtemp, + readdir, + readFile, + realpath, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { MediaRenderProgress } from "@rakazo/adapter-kit"; +import type { RenderPlan } from "@rakazo/contracts"; +import { EdlSchema } from "@rakazo/contracts"; +import type { CompileInput } from "@rakazo/media"; +import { compileRenderPlan, findMetadataLeaks } from "@rakazo/media"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + FfmpegMediaRenderer, + findTwoPassLoudnorm, + parseEbur128Summary, + parseFrameRate, + parseLoudnormJson, + parseProbeReport, +} from "./media-ffmpeg-renderer.js"; +import { MediaToolMissingError } from "./media-folder-source-probe.js"; +import { hasMediaTools, renderTestProfile, writeSyntheticClip } from "./media-render.fixtures.js"; + +describe("ffmpeg output parsing", () => { + it("parses frame rates", () => { + expect(parseFrameRate("30000/1001")).toBe(29.97); + expect(parseFrameRate("25/1")).toBe(25); + expect(parseFrameRate("0/0")).toBe(0); + expect(parseFrameRate(undefined)).toBe(0); + expect(parseFrameRate("abc")).toBe(0); + }); + + it("flattens format and stream tags and falls back to the file size", () => { + const report = parseProbeReport( + { + streams: [ + { + codec_type: "video", + codec_name: "h264", + width: 640, + height: 360, + r_frame_rate: "30/1", + }, + { codec_type: "audio", codec_name: "aac", tags: { language: "und" } }, + ], + format: { duration: "2.5", tags: { title: "Example", location: "+00+000/" } }, + }, + 1234, + ); + expect(report).toEqual({ + durationMs: 2500, + width: 640, + height: 360, + fps: 30, + videoCodec: "h264", + audioCodec: "aac", + sizeBytes: 1234, + integratedLufs: null, + truePeakDb: null, + tags: { + "format.title": "Example", + "format.location": "+00+000/", + "stream.1.language": "und", + }, + }); + }); + + it("reads the ebur128 summary and treats the silence gate as no programme", () => { + const stderr = `[Parsed_ebur128_0 @ 0x1] Summary: + + Integrated loudness: + I: -14.2 LUFS + Threshold: -24.3 LUFS + + True peak: + Peak: -1.4 dBFS`; + expect(parseEbur128Summary(stderr)).toEqual({ integratedLufs: -14.2, truePeakDb: -1.4 }); + expect(parseEbur128Summary("Summary:\n I: -70.0 LUFS\n True peak:\n Peak: -inf dBFS")).toEqual({ + integratedLufs: null, + truePeakDb: null, + }); + expect(parseEbur128Summary("no summary")).toEqual({ integratedLufs: null, truePeakDb: null }); + }); + + it("finds exactly one linear loudnorm without measured values", () => { + const graph = "[a]anull[b];[b]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]"; + const site = findTwoPassLoudnorm(graph); + expect(site?.options).toBe("I=-14:TP=-1:LRA=11:linear=true"); + expect(graph.slice(site!.start, site!.end)).toBe(site!.options); + expect(findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true:measured_I=-20[b]")).toBeNull(); + expect(findTwoPassLoudnorm("[a]loudnorm=I=-14[b]")).toBeNull(); + expect( + findTwoPassLoudnorm("[a]loudnorm=I=-14:linear=true[b];[c]loudnorm=I=-14:linear=true[d]"), + ).toBeNull(); + expect(findTwoPassLoudnorm("[a]anull[b]")).toBeNull(); + }); + + it("parses the loudnorm measurement block", () => { + const stderr = `[Parsed_loudnorm_3 @ 0x55] +{ +\t"input_i" : "-31.20", +\t"input_tp" : "-20.11", +\t"input_lra" : "0.00", +\t"input_thresh" : "-41.20", +\t"output_i" : "-14.02", +\t"output_tp" : "-1.00", +\t"output_lra" : "0.00", +\t"output_thresh" : "-24.02", +\t"normalization_type" : "dynamic", +\t"target_offset" : "0.02" +}`; + expect(parseLoudnormJson(stderr)).toEqual({ + inputI: -31.2, + inputTp: -20.11, + inputLra: 0, + inputThresh: -41.2, + targetOffset: 0.02, + }); + expect(parseLoudnormJson(stderr.replace('"-31.20"', '"-inf"'))).toBeNull(); + expect(parseLoudnormJson("nothing")).toBeNull(); + }); +}); + +describe("hardware encoder detection", () => { + let dir: string; + beforeAll(async () => { + dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-hw-"))); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + /** A fake ffmpeg that lists encoders and succeeds only for the given test encodes. */ + async function fakeFfmpeg(name: string, working: string[]): Promise { + const file = path.join(dir, name); + const cases = working.map((encoder) => `*" ${encoder} "*) exit 0 ;;`).join("\n "); + await writeFile( + file, + `#!/bin/sh +case " $* " in + *" -encoders "*) + echo " V....D libx264 libx264" + echo " V....D h264_nvenc NVIDIA NVENC H.264 encoder" + echo " V....D hevc_nvenc NVIDIA NVENC hevc encoder" + echo " V....D h264_vaapi H.264/AVC (VAAPI)" + exit 0 ;; + ${cases} +esac +echo "No device available" >&2 +exit 1 +`, + ); + await chmod(file, 0o755); + return file; + } + + it("keeps only listed encoders whose one-frame test encode works", async () => { + const renderer = new FfmpegMediaRenderer({ + ffmpegPath: await fakeFfmpeg("ffmpeg-nvenc", ["h264_nvenc"]), + vaapiDevice: path.join(dir, "no-render-node"), + }); + expect(await renderer.hardwareEncoders()).toEqual(["h264_nvenc"]); + }); + + it("falls back to software when no hardware encoder works or ffmpeg is missing", async () => { + const broken = new FfmpegMediaRenderer({ ffmpegPath: await fakeFfmpeg("ffmpeg-none", []) }); + expect(await broken.hardwareEncoders()).toEqual([]); + const missing = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") }); + expect(await missing.hardwareEncoders()).toEqual([]); + }); + + it("skips VAAPI encoders without a render node", async () => { + const renderer = new FfmpegMediaRenderer({ + ffmpegPath: await fakeFfmpeg("ffmpeg-vaapi", ["h264_vaapi"]), + vaapiDevice: path.join(dir, "no-render-node"), + }); + expect(await renderer.hardwareEncoders()).toEqual([]); + }); + + it.skipIf(!hasMediaTools)("never reports an encoder this host cannot run", async () => { + const renderer = new FfmpegMediaRenderer(); + for (const encoder of await renderer.hardwareEncoders()) { + expect(encoder).toMatch(/^(h264|hevc)_(vaapi|videotoolbox|nvenc)$/); + } + }); +}); + +describe("plan safety", () => { + let dir: string; + beforeAll(async () => { + dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-plan-"))); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("refuses writes outside the work directory", async () => { + const renderer = new FfmpegMediaRenderer({ root: dir }); + const workDir = path.join(dir, "work"); + const plan: RenderPlan = { + quality: "draft", + workDir, + steps: [ + { kind: "write_file", id: "escape", path: path.join(dir, "outside.txt"), content: "x" }, + ], + outputs: [], + }; + await expect(renderer.render(plan, {})).rejects.toThrow("outside"); + await expect(stat(path.join(dir, "outside.txt"))).rejects.toThrow(); + }); + + it("refuses a work directory outside the media root", async () => { + const renderer = new FfmpegMediaRenderer({ root: path.join(dir, "root") }); + await mkdir(path.join(dir, "root"), { recursive: true }); + const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] }; + await expect(renderer.render(plan, {})).rejects.toThrow("outside"); + }); + + it("requires motion, inpaint and identity steps to be prepared first", async () => { + const renderer = new FfmpegMediaRenderer(); + const plan: RenderPlan = { + quality: "final", + workDir: path.join(dir, "motion"), + steps: [ + { + kind: "motion", + id: "intro", + template: { templateId: "intro-minimal", params: {}, durationMs: 1000 }, + width: 320, + height: 180, + fps: 30, + outPath: path.join(dir, "motion", "intro.mov"), + }, + ], + outputs: [], + }; + await expect(renderer.render(plan, {})).rejects.toThrow("was not prepared"); + }); + + it("reports a missing ffmpeg instead of pretending to render", async () => { + const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") }); + const out = path.join(dir, "missing", "out.mp4"); + const plan: RenderPlan = { + quality: "draft", + workDir: path.join(dir, "missing"), + steps: [{ kind: "ffmpeg", id: "render", argv: ["-version"], produces: [out], durationMs: 1 }], + outputs: [{ kind: "video", path: out }], + }; + await expect(renderer.render(plan, {})).rejects.toBeInstanceOf(MediaToolMissingError); + }); + + it("rejects an already aborted render without spawning anything", async () => { + const renderer = new FfmpegMediaRenderer({ ffmpegPath: path.join(dir, "not-installed") }); + const controller = new AbortController(); + controller.abort(); + const plan: RenderPlan = { quality: "draft", workDir: dir, steps: [], outputs: [] }; + await expect(renderer.render(plan, { signal: controller.signal })).rejects.toMatchObject({ + name: "AbortError", + }); + }); +}); + +describe("two-pass loudness", () => { + let dir: string; + let ffmpegPath: string; + let log: string; + const GRAPH = "[0:a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000[aout]"; + + beforeAll(async () => { + dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-loudnorm-"))); + log = path.join(dir, "calls.log"); + ffmpegPath = path.join(dir, "ffmpeg"); + // Logs each call; the measurement pass prints loudnorm's JSON, the render + // pass writes its output file (the last argument). + await writeFile( + ffmpegPath, + `#!/bin/sh +echo "$*" >> "${log}" +case " $* " in + *print_format=json*|*.measure.txt*) + printf '[Parsed_loudnorm_0 @ 0x1] \\n{\\n "input_i" : "-31.20",\\n "input_tp" : "-20.11",\\n "input_lra" : "0.00",\\n "input_thresh" : "-41.20",\\n "target_offset" : "0.02"\\n}\\n' >&2 + exit 0 ;; +esac +for last; do :; done +[ "$last" = "-" ] || echo rendered > "$last" +`, + ); + await chmod(ffmpegPath, 0o755); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + async function calls(): Promise { + return (await readFile(log, "utf8")).trim().split("\n"); + } + + it("measures first, then renders with the measured values", async () => { + await rm(log, { force: true }); + const workDir = path.join(dir, "inline"); + const out = path.join(workDir, "out.m4a"); + const plan: RenderPlan = { + quality: "final", + workDir, + steps: [ + { + kind: "ffmpeg", + id: "render", + argv: ["-i", "in.wav", "-filter_complex", GRAPH, "-map", "[aout]", "-t", "2", out], + produces: [out], + durationMs: 2000, + }, + ], + outputs: [], + }; + await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {}); + const [measure, render] = await calls(); + expect(measure).toContain("linear=true:print_format=json"); + expect(measure).toContain("-map [aout] -t 2 -f null -"); + expect(measure).not.toContain(out); + expect(render).toContain( + "linear=true:measured_I=-31.20:measured_TP=-20.11:measured_LRA=0.00:measured_thresh=-41.20:offset=0.02,aresample=48000", + ); + expect(render?.endsWith(out)).toBe(true); + }); + + it("rewrites a filtergraph script and removes the measurement script", async () => { + await rm(log, { force: true }); + const workDir = path.join(dir, "script"); + const script = path.join(workDir, "filtergraph.txt"); + const out = path.join(workDir, "out.m4a"); + const plan: RenderPlan = { + quality: "final", + workDir, + steps: [ + { kind: "write_file", id: "filtergraph", path: script, content: GRAPH }, + { + kind: "ffmpeg", + id: "render", + argv: ["-i", "in.wav", "-filter_complex_script", script, "-map", "[aout]", out], + produces: [out], + durationMs: 2000, + }, + ], + outputs: [], + }; + await new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {}); + const [measure] = await calls(); + expect(measure).toContain(path.join(workDir, "filtergraph.measure.txt")); + expect(await readFile(script, "utf8")).toContain("measured_I=-31.20"); + expect((await readdir(workDir)).sort()).toEqual(["filtergraph.txt", "out.m4a"]); + }); + + it("fails a step whose output was never written and removes what it wrote", async () => { + const workDir = path.join(dir, "silent"); + const out = path.join(workDir, "out.m4a"); + const script = path.join(workDir, "notes.txt"); + const plan: RenderPlan = { + quality: "draft", + workDir, + steps: [ + { kind: "write_file", id: "notes", path: script, content: "x" }, + { + kind: "ffmpeg", + id: "render", + // The fake writes its last argument; the declared output stays missing. + argv: ["-i", "in.wav", path.join(workDir, "elsewhere.m4a")], + produces: [out], + durationMs: 1000, + }, + ], + outputs: [], + }; + await expect(new FfmpegMediaRenderer({ ffmpegPath }).render(plan, {})).rejects.toThrow( + "wrote no out.m4a", + ); + await expect(stat(script)).rejects.toThrow(); + }); +}); + +describe.skipIf(!hasMediaTools)("real ffmpeg renders", () => { + let dir: string; + let source: string; + const renderer = () => new FfmpegMediaRenderer({ root: dir }); + const profile = renderTestProfile(); + + beforeAll(async () => { + dir = await realpath(await mkdtemp(path.join(tmpdir(), "media-ffmpeg-"))); + source = path.join(dir, "assets", "source.mp4"); + // Deliberately quiet (-36 dBFS sine) so loudness normalization has real work to do. + await writeSyntheticClip(source, { seconds: 6, width: 320, height: 180, volumeDb: -36 }); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + function compile(workDir: string, overrides: Partial = {}): RenderPlan { + const edl = EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: profile.id, + width: 180, + height: 320, + fps: 30, + durationTargetMs: 4000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" }, + ], + }, + captions: { mode: "both" }, + overlays: [{ id: "t1", kind: "text", atMs: 500, durationMs: 1500, text: "Example: 50%" }], + metadata: { title: "Example teaser" }, + }); + return compileRenderPlan({ + edl, + profile, + brandKit: null, + quality: "draft", + workDir, + sources: new Map([ + [ + "asset-a", + { + path: source, + width: 320, + height: 180, + durationMs: 6000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + ]), + transcripts: new Map([ + [ + "asset-a", + [ + { startMs: 100, endMs: 600, text: "Hello" }, + { startMs: 650, endMs: 1200, text: "there." }, + { startMs: 3100, endMs: 3800, text: "Bye." }, + ], + ], + ]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.3, cy: 0.5, strength: 1 }, + { tMs: 5000, cx: 0.7, cy: 0.5, strength: 1 }, + ], + ], + ]), + motionClips: new Map(), + files: { lutPath: null, watermarkPath: null, fonts: new Map(), audio: new Map() }, + videoEncoder: "libx264", + ...overrides, + }); + } + + it("renders 16:9 to 9:16 with captions, overlay, two-pass loudness and clean metadata", async () => { + const workDir = path.join(dir, "render-ok"); + const plan = compile(workDir); + const progress: MediaRenderProgress[] = []; + await renderer().render(plan, { onProgress: (p) => progress.push(p) }); + + const video = plan.outputs.find((output) => output.kind === "video")!.path; + const report = await renderer().probe(video, { measureLoudness: true }); + expect(report.width).toBe(180); + expect(report.height).toBe(320); + expect(report.fps).toBe(30); + expect(report.videoCodec).toBe("h264"); + expect(report.audioCodec).toBe("aac"); + expect(Math.abs(report.durationMs - 4000)).toBeLessThanOrEqual(1000 / 30 + 1); + expect(report.integratedLufs).not.toBeNull(); + expect(Math.abs(report.integratedLufs! - profile.audio.targetLufs)).toBeLessThanOrEqual(1); + expect(report.truePeakDb!).toBeLessThanOrEqual(profile.audio.truePeakDb + 0.5); + expect(findMetadataLeaks(report.tags)).toEqual([]); + expect(report.tags["format.title"]).toBe("Example teaser"); + const keys = Object.keys(report.tags).join(" ").toLowerCase(); + for (const leak of ["location", "creation_time", "make", "model"]) { + expect(keys).not.toContain(leak); + } + expect(Object.values(report.tags).join(" ")).not.toContain("ExampleCam"); + + const poster = plan.outputs.find((output) => output.kind === "poster"); + if (poster) expect(findMetadataLeaks((await renderer().probe(poster.path)).tags)).toEqual([]); + expect(progress.length).toBeGreaterThan(0); + expect(progress.at(-1)!.progress).toBeCloseTo(1, 5); + for (let i = 1; i < progress.length; i++) { + expect(progress[i]!.progress).toBeGreaterThanOrEqual(progress[i - 1]!.progress - 1e-9); + } + // The measurement pass leaves nothing behind. + expect((await readdir(workDir)).sort()).toEqual( + ["captions.ass", "captions.srt", "captions.vtt", "output.mp4", "poster.jpg"].filter( + (name) => name !== "poster.jpg" || poster, + ), + ); + }); + + it("extracts an EXIF-free PNG frame", async () => { + const out = path.join(dir, "frames", "frame.png"); + await mkdir(path.dirname(out), { recursive: true }); + await renderer().extractFrame({ path: source, tMs: 1000, outPath: out, width: 161 }); + const report = await renderer().probe(out); + expect(report.width).toBe(160); + expect(report.videoCodec).toBe("png"); + expect(findMetadataLeaks(report.tags)).toEqual([]); + await expect( + renderer().extractFrame({ path: source, tMs: 1000, outPath: path.join(dir, "frame.jpg") }), + ).rejects.toThrow("PNG"); + await expect( + renderer().extractFrame({ path: source, tMs: 60_000, outPath: path.join(dir, "late.png") }), + ).rejects.toThrow(); + }); + + it("refuses to probe outside the media root", async () => { + await expect(renderer().probe("/etc/hostname")).rejects.toThrow("outside"); + await expect(renderer().probe("relative.mp4")).rejects.toThrow("absolute"); + }); + + it("aborts a running render, kills ffmpeg and removes partial outputs", async () => { + const workDir = path.join(dir, "render-abort"); + const long = await mkdtemp(path.join(dir, "long-")); + const longSource = path.join(long, "long.mp4"); + await writeSyntheticClip(longSource, { seconds: 30, width: 1280, height: 720 }); + const plan = compile(workDir, { + edl: EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: profile.id, + width: 1080, + height: 1920, + fps: 30, + durationTargetMs: 30_000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 30_000, atMs: 0, reframe: "none" }, + ], + }, + metadata: {}, + }), + sources: new Map([ + [ + "asset-a", + { + path: longSource, + width: 1280, + height: 720, + durationMs: 30_000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + ]), + }); + const controller = new AbortController(); + const started = Date.now(); + const rendering = renderer().render(plan, { + signal: controller.signal, + onProgress: (p) => { + if (p.progress > 0.01) controller.abort(); + }, + }); + await expect(rendering).rejects.toMatchObject({ name: "AbortError" }); + expect(Date.now() - started).toBeLessThan(25_000); + await expect(stat(path.join(workDir, "output.mp4"))).rejects.toThrow(); + }); +}); diff --git a/packages/adapters/src/media-ffmpeg-renderer.ts b/packages/adapters/src/media-ffmpeg-renderer.ts index 46b1487..b763a33 100644 --- a/packages/adapters/src/media-ffmpeg-renderer.ts +++ b/packages/adapters/src/media-ffmpeg-renderer.ts @@ -316,6 +316,13 @@ export class FfmpegMediaRenderer implements MediaRenderer { signal, report: (fraction) => report(step.id, fraction, weight(step)), }); + // Never report success for an output ffmpeg did not actually write. + for (const produced of step.produces) { + const file = await stat(produced).catch(() => null); + if (!file?.isFile() || file.size === 0) { + throw new Error(`step ${step.id} wrote no ${path.basename(produced)}`); + } + } done += weight(step); report(step.id, 0, 0); break; diff --git a/packages/adapters/src/media-render.fixtures.ts b/packages/adapters/src/media-render.fixtures.ts index 6926759..a7cd103 100644 --- a/packages/adapters/src/media-render.fixtures.ts +++ b/packages/adapters/src/media-render.fixtures.ts @@ -1,7 +1,9 @@ +import { execFile, execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { mkdir, mkdtemp, readFile, realpath, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { promisify } from "node:util"; import type { JobPublisher, LinkShortener, @@ -43,6 +45,70 @@ import type { MediaServiceDeps, } from "./media-context.js"; +const run = promisify(execFile); + +function onPath(binary: string): boolean { + try { + execFileSync(binary, ["-version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +/** Real-ffmpeg tests run when both tools are installed and skip otherwise. */ +export const hasMediaTools = onPath("ffmpeg") && onPath("ffprobe"); + +/** + * Synthetic clip: testsrc2 pattern plus a sine tone, H.264/AAC in MP4, with the + * kind of tags a phone writes (GPS, device, capture time) that every export + * must drop. No real people or places. + */ +export async function writeSyntheticClip( + file: string, + options: { seconds: number; width: number; height: number; fps?: number; volumeDb?: number }, +): Promise { + const fps = options.fps ?? 30; + await mkdir(path.dirname(file), { recursive: true }); + await run("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-nostdin", + "-y", + "-f", + "lavfi", + "-i", + `testsrc2=size=${options.width}x${options.height}:rate=${fps}:duration=${options.seconds}`, + "-f", + "lavfi", + "-i", + `sine=frequency=440:sample_rate=48000:duration=${options.seconds}`, + "-af", + `volume=${options.volumeDb ?? -20}dB`, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + "-metadata", + "creation_time=2020-01-01T10:00:00Z", + "-metadata", + "location=+00.0000+000.0000/", + "-metadata", + "make=ExampleCam", + "-metadata", + "model=ExampleCam One", + "-metadata:s:v", + "handler_name=ExampleCam serial 0000", + file, + ]); +} + /* * Test harness for the render service and job: the repos the render slice * reads, either in memory or backed by PostgreSQL (VERIFY_DATABASE), plus a @@ -134,7 +200,11 @@ export interface RenderHarness { addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise; addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise; addArtifact(file: string, name: string, mimeType: string): Promise; - createProject(input: { assetIds: string[]; kit: BrandKit | null; profileId: string }): Promise; + createProject(input: { + assetIds: string[]; + kit: BrandKit | null; + profileId: string; + }): Promise; createPlan(projectId: string, edl: unknown): Promise; planVersions(projectId: string, profileId: string): Promise; artifactCount(): Promise; @@ -271,7 +341,11 @@ interface Backend { createAsset(name: string): Promise; addAnalysis(assetId: string, kind: MediaAnalysisKind, data: unknown): Promise; addSegments(assetId: string, ranges: Array<{ startMs: number; endMs: number }>): Promise; - createProject(input: { assetIds: string[]; kit: BrandKit | null; profileId: string }): Promise; + createProject(input: { + assetIds: string[]; + kit: BrandKit | null; + profileId: string; + }): Promise; cleanup(): Promise; } @@ -461,7 +535,9 @@ function memoryBackend(now: () => Date): Backend { }, async listPlanVersions(space: string, projectId: string, profileId: string) { return plans - .filter((p) => p.spaceId === space && p.projectId === projectId && p.profileId === profileId) + .filter( + (p) => p.spaceId === space && p.projectId === projectId && p.profileId === profileId, + ) .sort((a, b) => b.version - a.version) .map(mapMediaEditPlan); }, diff --git a/packages/adapters/src/media-render.test.ts b/packages/adapters/src/media-render.test.ts new file mode 100644 index 0000000..218b58a --- /dev/null +++ b/packages/adapters/src/media-render.test.ts @@ -0,0 +1,456 @@ +import { mkdir, readdir, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { MediaRenderer, MotionGraphicsRenderer } from "@rakazo/adapter-kit"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import type { FfprobeReport, GateIssue, MediaRender, RenderPlan } from "@rakazo/contracts"; +import { findMetadataLeaks } from "@rakazo/media"; +import { afterEach, describe, expect, it } from "vitest"; +import type { MediaPrivacyService } from "./media-context.js"; +import { FfmpegMediaRenderer } from "./media-ffmpeg-renderer.js"; +import type { FakePrivacy, RenderHarness } from "./media-render.fixtures.js"; +import { + createFakePrivacy, + createRenderHarness, + hasMediaTools, + renderTestProfile, + writeSyntheticClip, +} from "./media-render.fixtures.js"; +import { createMediaRenderHandler, createMediaRenderService, pickEncoder } from "./media-render.js"; + +const profile = renderTestProfile(); + +const motion: MotionGraphicsRenderer = { + id: "unused", + templates: () => [], + render: async () => { + throw new Error("no motion in these plans"); + }, +}; + +interface FakeRenderer extends MediaRenderer { + plans: RenderPlan[]; +} + +/** + * Executes write_file steps and writes placeholder outputs; probes report a + * clean, on-target file. `failEncoders` makes renders with those encoders fail. + */ +function fakeRenderer( + options: { + hardware?: string[]; + failEncoders?: string[]; + tags?: Record; + posterTags?: Record; + /** Resolves once a render started; the render then waits for its signal. */ + hold?: { started: () => void }; + } = {}, +): FakeRenderer { + const plans: RenderPlan[] = []; + let durationMs = 0; + return { + id: "fake", + plans, + hardwareEncoders: async () => options.hardware ?? [], + async render(plan, { signal, onProgress } = {}) { + plans.push(plan); + const step = plan.steps.find((s) => s.kind === "ffmpeg" && s.id === "render"); + if (step?.kind !== "ffmpeg") throw new Error("no render step"); + durationMs = step.durationMs; + const encoder = step.argv[step.argv.indexOf("-c:v") + 1]!; + if (options.failEncoders?.includes(encoder)) throw new Error(`${encoder}: device lost`); + if (options.hold) { + options.hold.started(); + await new Promise((_, reject) => { + signal?.addEventListener("abort", () => { + const error = new Error("render aborted"); + error.name = "AbortError"; + reject(error); + }); + }); + } + for (const s of plan.steps) { + if (s.kind === "write_file") { + await mkdir(path.dirname(s.path), { recursive: true }); + await writeFile(s.path, s.content); + } + if (s.kind === "ffmpeg") { + for (const file of s.produces) await writeFile(file, `rendered ${s.id}`); + } + } + onProgress?.({ stepId: "render", progress: 1 }); + }, + async probe(file): Promise { + const poster = file.endsWith(".jpg"); + return { + durationMs: poster ? 0 : durationMs, + width: profile.video.width, + height: profile.video.height, + fps: poster ? 0 : profile.video.fps, + videoCodec: poster ? "mjpeg" : "h264", + audioCodec: poster ? null : "aac", + sizeBytes: (await stat(file)).size, + integratedLufs: poster ? null : -14.2, + truePeakDb: poster ? null : -1.5, + tags: poster ? (options.posterTags ?? {}) : (options.tags ?? {}), + }; + }, + extractFrame: async () => { + throw new Error("not used"); + }, + }; +} + +function edlFor(assetId: string) { + return { + edlVersion: 1, + output: { + profileId: profile.id, + width: profile.video.width, + height: profile.video.height, + fps: 30, + durationTargetMs: 3000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [ + { id: "c1", assetId, inMs: 0, outMs: 1500, atMs: 0, reframe: "auto" }, + { id: "c2", assetId, inMs: 2000, outMs: 3500, atMs: 1500, reframe: "auto" }, + ], + }, + captions: { mode: "both" }, + overlays: [{ id: "t1", kind: "text", atMs: 200, durationMs: 1000, text: "Example" }], + metadata: { title: "Example teaser" }, + }; +} + +interface Setup { + harness: RenderHarness; + privacy: FakePrivacy; + renderer: MediaRenderer; + run: (planQuality?: "draft" | "final") => Promise; + renderDir: (renderId: string) => string; +} + +let cleanups: Array<() => Promise> = []; +afterEach(async () => { + for (const cleanup of cleanups) await cleanup(); + cleanups = []; +}); + +async function setup(options: { + renderer: MediaRenderer; + privacy?: (sources: RenderHarness["sources"]) => MediaPrivacyService; + failWith?: GateIssue; + realSource?: boolean; +}): Promise }> { + const harness = await createRenderHarness({ + profiles: [profile], + renderer: options.renderer, + motion, + }); + cleanups.push(() => harness.cleanup()); + const assetId = await harness.createAsset("source.mp4"); + const file = harness.assetPath(assetId, "source.mp4"); + if (options.realSource) { + await writeSyntheticClip(file, { seconds: 4, width: 320, height: 180, volumeDb: -30 }); + } else { + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, "synthetic"); + } + harness.setSource(assetId, { + path: file, + width: 320, + height: 180, + durationMs: 4000, + fps: 30, + hasAudio: true, + rotation: 0, + }); + await harness.addAnalysis(assetId, "transcript", { + language: "en", + words: [ + { startMs: 100, endMs: 500, text: "Hello", confidence: 0.9 }, + { startMs: 2100, endMs: 2600, text: "again.", confidence: 0.9 }, + ], + }); + await harness.addSegments(assetId, [ + { startMs: 0, endMs: 2000 }, + { startMs: 2000, endMs: 4000 }, + ]); + const projectId = await harness.createProject({ + assetIds: [assetId], + kit: null, + profileId: profile.id, + }); + const plan = await harness.createPlan(projectId, edlFor(assetId)); + const privacy = createFakePrivacy(harness.sources, { failWith: options.failWith }); + const service = createMediaRenderService(harness.deps); + const handler = createMediaRenderHandler( + harness.deps, + options.privacy ? options.privacy(harness.sources) : privacy, + { cancelPollMs: 50 }, + ); + const actor = { spaceId: harness.spaceId, userId: harness.userId }; + return { + harness, + privacy, + renderer: options.renderer, + service, + renderDir: (renderId) => path.join(harness.mediaRoot, harness.spaceId, "renders", renderId), + async run(quality = "draft") { + const render = await service.start(actor, { planId: plan.id, quality }); + await handler({ renderId: render.id }); + return harness.deps.repos.getRender(harness.spaceId, render.id); + }, + }; +} + +async function exists(file: string): Promise { + return stat(file).then( + () => true, + () => false, + ); +} + +describe("media render service", () => { + it("queues a render job and announces it", async () => { + const { harness, service } = await setup({ renderer: fakeRenderer() }); + expect(harness.backend).toBe( + process.env.VERIFY_DATABASE && process.env.DATABASE_URL ? "postgres" : "memory", + ); + const projectId = await harness.createProject({ + assetIds: [], + kit: null, + profileId: profile.id, + }); + const assetId = [...harness.sources.keys()][0]!; + const plan = await harness.createPlan(projectId, edlFor(assetId)); + const actor = { spaceId: harness.spaceId, userId: harness.userId }; + const render = await service.start(actor, { planId: plan.id, quality: "final" }); + expect(render).toMatchObject({ status: "queued", quality: "final", planId: plan.id }); + expect(harness.enqueued).toContain(`media.render:${JSON.stringify({ renderId: render.id })}`); + expect(harness.events).toContainEqual({ + type: "media.render", + renderId: render.id, + projectId, + status: "queued", + progress: 0, + }); + await expect( + service.start({ ...actor, spaceId: "space-other" }, { planId: plan.id, quality: "draft" }), + ).rejects.toThrow(); + }); + + it("renders, gates, stores outputs and serves only recorded files", async () => { + const renderer = fakeRenderer(); + const { harness, service, run, renderDir } = await setup({ renderer }); + const render = await run(); + expect(render.status).toBe("done"); + expect(render.progress).toBe(1); + expect(render.gateReport?.passed).toBe(true); + expect(render.ffprobe?.integratedLufs).toBe(-14.2); + expect(render.outputs.map((output) => output.fileName).sort()).toEqual([ + "captions.srt", + "captions.vtt", + "poster.jpg", + "video.mp4", + ]); + const video = render.outputs.find((output) => output.kind === "video")!; + expect(video.artifactId).toBeNull(); + expect(render.outputs.filter((output) => output.artifactId !== null)).toHaveLength(3); + expect(await harness.artifactCount()).toBe(3); + // Intermediates are gone; only the published outputs stay. + expect((await readdir(renderDir(render.id))).sort()).toEqual([ + "captions.srt", + "captions.vtt", + "poster.jpg", + "video.mp4", + ]); + const actor = { spaceId: harness.spaceId, userId: harness.userId }; + expect(await service.outputPath(actor, render.id, "video.mp4")).toBe( + path.join(renderDir(render.id), "video.mp4"), + ); + expect(await service.outputPath(actor, render.id, "../video.mp4")).toBeNull(); + expect(await service.outputPath(actor, render.id, "work")).toBeNull(); + await expect( + service.outputPath({ ...actor, spaceId: "space-other" }, render.id, "video.mp4"), + ).rejects.toThrow(); + const statuses = harness.events.flatMap((event) => + event.type === "media.render" && event.renderId === render.id ? [event.status] : [], + ); + expect(statuses[0]).toBe("queued"); + expect(statuses).toContain("gating"); + expect(statuses.at(-1)).toBe("done"); + }); + + it("refuses to render when redactions cannot be applied (fail closed)", async () => { + const renderer = fakeRenderer(); + let calls = 0; + const { harness, run, renderDir } = await setup({ + renderer, + privacy: (sources) => ({ + ...createFakePrivacy(sources), + async prepareSources() { + calls += 1; + throw new Error("face track lost for 800 ms"); + }, + }), + }); + const render = await run("final"); + expect(calls).toBe(1); + expect(render.status).toBe("failed"); + expect(render.error).toBe("Anonymization could not be applied. Nothing was rendered."); + expect(render.outputs).toEqual([]); + expect(renderer.plans).toHaveLength(0); + expect(await exists(renderDir(render.id))).toBe(false); + expect(await harness.artifactCount()).toBe(0); + }); + + it("names the missing capability and refuses when a clip was not prepared", async () => { + const unsupported = await setup({ + renderer: fakeRenderer(), + privacy: (sources) => ({ + ...createFakePrivacy(sources), + async prepareSources() { + throw new UnsupportedMediaOperationError("inpaintVideo", "no model"); + }, + }), + }); + expect((await unsupported.run()).error).toBe( + "Anonymization needs inpaintVideo, which is not available. Nothing was rendered.", + ); + + const renderer = fakeRenderer(); + const partial = await setup({ + renderer, + privacy: (sources) => ({ + ...createFakePrivacy(sources), + async prepareSources() { + return { sources: new Map(), aiAlterations: [], fallbacks: [] }; + }, + }), + }); + const render = await partial.run(); + expect(render.status).toBe("failed"); + expect(render.error).toBe("Anonymization did not prepare every clip. Nothing was rendered."); + expect(renderer.plans).toHaveLength(0); + }); + + it("falls back to software when a hardware encoder fails and skips it afterwards", async () => { + const renderer = fakeRenderer({ + hardware: ["hevc_vaapi", "h264_vaapi"], + failEncoders: ["h264_vaapi"], + }); + const { run } = await setup({ renderer }); + const encoders = () => + renderer.plans.map((plan) => { + const step = plan.steps.find((s) => s.kind === "ffmpeg" && s.id === "render"); + return step?.kind === "ffmpeg" ? step.argv[step.argv.indexOf("-c:v") + 1] : null; + }); + expect((await run("final")).status).toBe("done"); + expect(encoders()).toEqual(["h264_vaapi", "libx264"]); + expect((await run("final")).status).toBe("done"); + expect(encoders()).toEqual(["h264_vaapi", "libx264", "libx264"]); + }); + + it("uses software for drafts and a working hardware encoder for finals", async () => { + const renderer = fakeRenderer({ hardware: ["hevc_nvenc", "h264_nvenc"] }); + expect(await pickEncoder(renderer, profile, "draft")).toBe("libx264"); + expect(await pickEncoder(renderer, profile, "final")).toBe("h264_nvenc"); + expect(await pickEncoder(fakeRenderer(), profile, "final")).toBe("libx264"); + const hevc = renderTestProfile({ + video: { ...profile.video, codec: "hevc" }, + }); + expect(await pickEncoder(renderer, hevc, "final")).toBe("hevc_nvenc"); + expect(await pickEncoder(renderer, hevc, "draft")).toBe("libx265"); + }); + + it("fails a render whose video or poster still carries source metadata", async () => { + const leaky = await setup({ + renderer: fakeRenderer({ tags: { "format.location": "+00+000/" } }), + }); + const render = await leaky.run(); + expect(render.status).toBe("failed"); + expect(render.error).toBe("The export still carried source metadata."); + expect(await exists(leaky.renderDir(render.id))).toBe(false); + + const poster = await setup({ + renderer: fakeRenderer({ posterTags: { "format.make": "ExampleCam" } }), + }); + expect((await poster.run()).error).toBe("The poster still carried source metadata."); + }); + + it("auto-fixes a blocked gate at most three times, then blocks", async () => { + const renderer = fakeRenderer(); + const { harness, privacy, run, renderDir } = await setup({ + renderer, + failWith: { + check: "content", + severity: "block", + code: "classifier_violence", + message: "violence score above the profile threshold", + atMs: 0, + endMs: 400, + }, + }); + const render = await run(); + expect(render.status).toBe("blocked"); + expect(render.gateReport?.passed).toBe(false); + expect(privacy.gateCalls.map((call) => call.autoFixRound)).toEqual([0, 1, 2, 3]); + expect(renderer.plans).toHaveLength(4); + const record = await harness.deps.repos.getRenderRecord(harness.spaceId, render.id); + expect(record.autoFixRound).toBe(3); + const versions = await harness.planVersions(render.projectId, profile.id); + expect(versions).toHaveLength(4); + expect(versions[0]!.createdBy).toBe("agent"); + expect(versions[0]!.rationale).toContain("Auto-fix 3/3"); + expect(render.outputs).toEqual([]); + expect(await exists(renderDir(render.id))).toBe(false); + expect(await harness.artifactCount()).toBe(0); + }); + + it("cancels a running render, stops it and removes its files", async () => { + let started!: () => void; + const running = new Promise((resolve) => { + started = resolve; + }); + const renderer = fakeRenderer({ hold: { started: () => started() } }); + const { harness, service, run, renderDir } = await setup({ renderer }); + const result = run(); + await running; + const active = harness.events.findLast((event) => event.type === "media.render"); + const renderId = active?.type === "media.render" ? active.renderId : ""; + const canceled = await service.cancel( + { spaceId: harness.spaceId, userId: harness.userId }, + renderId, + ); + expect(canceled.status).toBe("canceled"); + const render = await result; + expect(render.status).toBe("canceled"); + expect(await exists(renderDir(renderId))).toBe(false); + await expect( + service.cancel({ spaceId: harness.spaceId, userId: harness.userId }, renderId), + ).rejects.toThrow(); + }); +}); + +describe.skipIf(!hasMediaTools)("media render job with real ffmpeg", () => { + it("renders a synthetic project end to end and records a clean probe", async () => { + const renderer = new FfmpegMediaRenderer(); + const { run, renderDir } = await setup({ renderer, realSource: true }); + const render = await run("draft"); + expect(render.error).toBeNull(); + expect(render.status).toBe("done"); + const probe = render.ffprobe!; + expect(probe.width).toBe(profile.video.width); + expect(probe.height).toBe(profile.video.height); + expect(Math.abs(probe.durationMs - 3000)).toBeLessThanOrEqual(1000 / 30 + 1); + expect(Math.abs(probe.integratedLufs! - profile.audio.targetLufs)).toBeLessThanOrEqual(1); + expect(findMetadataLeaks(probe.tags)).toEqual([]); + const video = path.join(renderDir(render.id), "video.mp4"); + const reprobed = await renderer.probe(video); + expect(reprobed.tags["format.title"]).toBe("Example teaser"); + expect(Object.values(reprobed.tags).join(" ")).not.toContain("ExampleCam"); + }, 120_000); +}); diff --git a/packages/adapters/src/media-render.ts b/packages/adapters/src/media-render.ts index 32fd240..259577f 100644 --- a/packages/adapters/src/media-render.ts +++ b/packages/adapters/src/media-render.ts @@ -23,6 +23,7 @@ import { compileRenderPlan, findMetadataLeaks, planAutoFix } from "@rakazo/media import { resolveInside } from "@rakazo/media/node"; import type { MediaActor, + MediaPreparedSources, MediaPrivacyService, MediaRenderService, MediaServiceDeps, @@ -452,15 +453,37 @@ async function runRender(input: RunInput): Promise { const roundDir = path.join(workDir, `round-${round}`); await mkdir(roundDir, { recursive: true }); - const prepared = await privacy.prepareSources({ - actor, - projectId: project.id, - edl, - quality, - workDir: roundDir, - signal, - }); + // Fail closed: no redacted sources, no render. + let prepared: MediaPreparedSources; + try { + prepared = await privacy.prepareSources({ + actor, + projectId: project.id, + edl, + quality, + workDir: roundDir, + signal, + }); + } catch (error) { + checkCanceled(); + getLogger().warn("media render sources not prepared", { + renderId, + reason: error instanceof Error ? error.message : String(error), + }); + throw new MediaRenderFailure( + error instanceof UnsupportedMediaOperationError + ? `Anonymization needs ${error.operation}, which is not available. Nothing was rendered.` + : "Anonymization could not be applied. Nothing was rendered.", + ); + } checkCanceled(); + for (const clip of edl.tracks.video) { + if (!prepared.sources.has(clip.assetId)) { + throw new MediaRenderFailure( + "Anonymization did not prepare every clip. Nothing was rendered.", + ); + } + } for (const clip of edl.tracks.audio) { if (audioFiles.has(clip.artifactId)) continue; @@ -475,37 +498,42 @@ async function runRender(input: RunInput): Promise { const motionClips = await renderMotionClips(deps, edl, kit, brand, roundDir, signal); const { transcripts, subjects } = await analysisInputs(deps, spaceId, edl); - const videoEncoder = await pickEncoder(deps, profile, quality); - - const renderPlan = compileRenderPlan({ - edl, - profile, - brandKit: kit, - quality, - workDir: roundDir, - sources: prepared.sources, - transcripts, - subjects, - motionClips, - files: { - lutPath: brand.lut, - watermarkPath: brand.watermark, - fonts: brand.fonts, - audio: audioFiles, - logoPath: brand.logos[0] ?? null, - }, - videoEncoder, - }); - - await executePlan(input, renderPlan, update); + const compile = (videoEncoder: string) => + compileRenderPlan({ + edl, + profile, + brandKit: kit, + quality, + workDir: roundDir, + sources: prepared.sources, + transcripts, + subjects, + motionClips, + files: { + lutPath: brand.lut, + watermarkPath: brand.watermark, + fonts: brand.fonts, + audio: audioFiles, + logoPath: brand.logos[0] ?? null, + }, + videoEncoder, + }); + const renderPlan = await renderWithFallback(input, profile, quality, compile, update); const video = renderPlan.outputs.find((output) => output.kind === "video"); if (!video) throw new Error("render plan has no video output"); const ffprobe = await deps.renderer.probe(video.path, { measureLoudness: true, signal }); + // A render whose metadata could not be removed is failed, not gated. if (findMetadataLeaks(ffprobe.tags).length > 0) { - // Spec 5: a render whose metadata could not be removed is failed, not gated. throw new MediaRenderFailure("The export still carried source metadata."); } + for (const output of renderPlan.outputs) { + if (output.kind !== "poster") continue; + const poster = await deps.renderer.probe(output.path, { signal }); + if (findMetadataLeaks(poster.tags).length > 0) { + throw new MediaRenderFailure("The poster still carried source metadata."); + } + } const gating = await update({ status: "gating", progress: 0.9, ffprobe }); await publishRender(deps, spaceId, gating); @@ -688,17 +716,64 @@ async function analysisInputs(deps: MediaServiceDeps, spaceId: string, edl: Edl) return { transcripts, subjects }; } +/** Hardware encoders that failed a real render, per renderer, so later renders skip them. */ +const failedEncoders = new WeakMap>(); + +export function softwareEncoder(profile: PlatformProfile): string { + return profile.video.codec === "h264" ? "libx264" : "libx265"; +} + /** Final renders prefer a working hardware encoder of the profile's codec; drafts use software. */ -async function pickEncoder( - deps: MediaServiceDeps, +export async function pickEncoder( + renderer: MediaServiceDeps["renderer"], profile: PlatformProfile, quality: MediaRenderQuality, ): Promise { - const family = profile.video.codec; - const software = family === "h264" ? "libx264" : "libx265"; + const software = softwareEncoder(profile); if (quality === "draft") return software; - const hardware = await deps.renderer.hardwareEncoders(); - return hardware.find((encoder) => encoder.startsWith(`${family}_`)) ?? software; + const failed = failedEncoders.get(renderer); + const hardware = await renderer.hardwareEncoders(); + return ( + hardware.find( + (encoder) => encoder.startsWith(`${profile.video.codec}_`) && !failed?.has(encoder), + ) ?? software + ); +} + +/** + * Run the plan with the preferred encoder. A hardware encoder that fails + * mid-render (driver, device, or session limits) is retried once in software + * and skipped by this process afterwards; software failures are final. + */ +async function renderWithFallback( + input: RunInput, + profile: PlatformProfile, + quality: MediaRenderQuality, + compile: (videoEncoder: string) => RenderPlan, + update: (patch: MediaRenderPatch) => Promise, +): Promise { + const { deps, signal } = input; + const encoder = await pickEncoder(deps.renderer, profile, quality); + const plan = compile(encoder); + const software = softwareEncoder(profile); + try { + await executePlan(input, plan, update); + return plan; + } catch (error) { + if (encoder === software || signal.aborted || error instanceof RenderCanceled) throw error; + if (error instanceof MediaToolMissingError) throw error; + getLogger().warn("media hardware encoder failed; rendering in software", { + renderId: input.renderId, + encoder, + reason: error instanceof Error ? error.message : String(error), + }); + const failed = failedEncoders.get(deps.renderer) ?? new Set(); + failed.add(encoder); + failedEncoders.set(deps.renderer, failed); + } + const fallback = compile(software); + await executePlan(input, fallback, update); + return fallback; } /** Segments the auto-fix may swap in: the project's material, never paid-only. */ diff --git a/packages/testkit/src/fixtures/media/synthetic-media.ts b/packages/testkit/src/fixtures/media/synthetic-media.ts new file mode 100644 index 0000000..da24c66 --- /dev/null +++ b/packages/testkit/src/fixtures/media/synthetic-media.ts @@ -0,0 +1,208 @@ +import { execFile, execFileSync } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +/* + * Synthetic media fixtures for the render integration test, generated with + * ffmpeg lavfi sources at test time (nothing binary is checked in). No real + * people, faces, places, or devices: the "face" is a flat generated disc, the + * picture is testsrc2, the sound is a sine tone, and the device tags are + * placeholders that every export must drop. + */ + +const run = promisify(execFile); + +export function hasFfmpeg(): boolean { + try { + execFileSync("ffmpeg", ["-version"], { stdio: "ignore" }); + execFileSync("ffprobe", ["-version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +async function ffmpeg(args: string[]): Promise { + await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...args]); +} + +/** Tags a phone writes (GPS, make, model, capture time, serial in a handler name). */ +export const PHONE_TAGS: Record = { + creation_time: "2020-01-01T10:00:00Z", + location: "+00.0000+000.0000/", + "com.apple.quicktime.location.ISO6709": "+00.0000+000.0000/", + make: "ExampleCam", + model: "ExampleCam One", + "com.apple.quicktime.software": "ExampleCam OS 1.0", +}; + +export interface SyntheticClipOptions { + seconds: number; + width: number; + height: number; + fps: number; + /** Sine level in dBFS; quiet on purpose so loudness normalization has work to do. */ + volumeDb: number; + /** + * Draw a skin-toned disc ("face") centred at this normalized point with this + * radius relative to the height. The media emulator reports faces for files + * named with `__faces-1__`; the disc sits where its first face box lies. + */ + face?: { cx: number; cy: number; radius: number }; +} + +/** testsrc2 + sine, H.264/AAC MP4 with phone-style tags. */ +export async function writeSyntheticClip(file: string, options: SyntheticClipOptions) { + await mkdir(path.dirname(file), { recursive: true }); + const { seconds, width, height, fps } = options; + const inputs = [ + "-f", + "lavfi", + "-i", + `testsrc2=size=${width}x${height}:rate=${fps}:duration=${seconds}`, + "-f", + "lavfi", + "-i", + `sine=frequency=330:sample_rate=48000:duration=${seconds}`, + ]; + let video = "[0:v]null[v]"; + if (options.face) { + const r = Math.round(options.face.radius * height); + const size = r * 2; + const x = Math.round(options.face.cx * width - r); + const y = Math.round(options.face.cy * height - r); + inputs.push("-f", "lavfi", "-i", `color=c=black:size=${size}x${size}:rate=${fps}`); + video = [ + `[2:v]format=rgba,geq=r=230:g=190:b=160:a='if(lte(hypot(X-${r},Y-${r}),${r}),255,0)'[disc]`, + `[0:v][disc]overlay=x=${x}:y=${y}:shortest=1[v]`, + ].join(";"); + } + const tags = Object.entries(PHONE_TAGS).flatMap(([key, value]) => [ + "-metadata", + `${key}=${value}`, + ]); + await ffmpeg([ + ...inputs, + "-filter_complex", + `${video};[1:a]volume=${options.volumeDb}dB[a]`, + "-map", + "[v]", + "-map", + "[a]", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-t", + String(seconds), + "-movflags", + "+use_metadata_tags", + ...tags, + "-metadata:s:v", + "handler_name=ExampleCam serial 0000", + file, + ]); +} + +/** Sine music bed as WAV. */ +export async function writeMusicBed(file: string, seconds: number): Promise { + await mkdir(path.dirname(file), { recursive: true }); + await ffmpeg([ + "-f", + "lavfi", + "-i", + `sine=frequency=220:sample_rate=48000:duration=${seconds}`, + "-af", + "volume=-18dB", + file, + ]); +} + +/** Flat square PNG used as the brand logo and watermark. */ +export async function writeLogo(file: string): Promise { + await mkdir(path.dirname(file), { recursive: true }); + await ffmpeg(["-f", "lavfi", "-i", "color=c=white:size=96x96", "-frames:v", "1", file]); +} + +/** Decode one frame as 8-bit grayscale at the given size. */ +export async function grayFrame( + file: string, + tMs: number, + width: number, + height: number, +): Promise { + const { stdout } = await run( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-nostdin", + "-ss", + (tMs / 1000).toFixed(3), + "-i", + file, + "-frames:v", + "1", + "-vf", + `scale=${width}:${height}:flags=area`, + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "-", + ], + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + ); + if (stdout.length !== width * height) throw new Error("unexpected frame size"); + return new Uint8Array(stdout); +} + +export interface PixelBox { + x0: number; + y0: number; + x1: number; + y1: number; +} + +/** Variance of the pixels inside a box. */ +export function variance(frame: Uint8Array, width: number, box: PixelBox): number { + let sum = 0; + let squares = 0; + let count = 0; + for (let y = box.y0; y < box.y1; y++) { + for (let x = box.x0; x < box.x1; x++) { + const value = frame[y * width + x]!; + sum += value; + squares += value * value; + count += 1; + } + } + const mean = sum / count; + return squares / count - mean * mean; +} + +/** Mean absolute difference of two frames outside a box. */ +export function meanDifferenceOutside( + a: Uint8Array, + b: Uint8Array, + width: number, + height: number, + box: PixelBox, +): number { + let total = 0; + let count = 0; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + if (x >= box.x0 && x < box.x1 && y >= box.y0 && y < box.y1) continue; + total += Math.abs(a[y * width + x]! - b[y * width + x]!); + count += 1; + } + } + return total / count; +} diff --git a/packages/testkit/src/media-render.docker.test.ts b/packages/testkit/src/media-render.docker.test.ts new file mode 100644 index 0000000..307f740 --- /dev/null +++ b/packages/testkit/src/media-render.docker.test.ts @@ -0,0 +1,359 @@ +import path from "node:path"; +import type { BoxKeyframe, Edl } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { MediaPrivacyService } from "../../adapters/src/media-context.js"; +import { EmulatorMediaInferenceProvider } from "../../adapters/src/media-emulator.js"; +import { FfmpegMediaRenderer } from "../../adapters/src/media-ffmpeg-renderer.js"; +import type { RenderHarness } from "../../adapters/src/media-render.fixtures.js"; +import { + createFakePrivacy, + createRenderHarness, + renderTestKit, + renderTestProfile, +} from "../../adapters/src/media-render.fixtures.js"; +import { + createMediaRenderHandler, + createMediaRenderService, +} from "../../adapters/src/media-render.js"; +import { findMetadataLeaks } from "../../media/src/gate.js"; +import type { PixelBox } from "./fixtures/media/synthetic-media.js"; +import { + grayFrame, + hasFfmpeg, + meanDifferenceOutside, + variance, + writeLogo, + writeMusicBed, + writeSyntheticClip, +} from "./fixtures/media/synthetic-media.js"; + +/* + * Real render integration (spec 13, Docker job). Synthetic fixtures are made + * with ffmpeg lavfi; inference runs on the deterministic emulator; the render + * goes through the real service, job handler, compiler, and ffmpeg renderer, + * and every result is checked with ffprobe. Runs wherever ffmpeg is installed + * (the CI test job and the media Docker job) and skips cleanly without it. + * + * Redactions belong to the privacy slice. Its stand-in here applies EDL region + * redactions through the emulator's `redact` (the same MediaInferenceProvider + * call the privacy service uses) and refuses to return an unredacted source. + */ + +const FPS = 30; +const FRAME_MS = 1000 / FPS; +/** Grayscale variance a blurred region must stay under. */ +const VARIANCE_THRESHOLD = 15; +/** + * The generated "face": a flat disc inside every box the media emulator can + * report as the first face of a `__faces-1__` file (boxes are seeded by the + * file, but always contain this point and radius). + */ +const FACE = { cx: 0.19, cy: 0.28, radius: 0.05 }; + +const vertical = renderTestProfile(); +const landscape = renderTestProfile({ + id: "example-landscape-test", + name: "Example landscape test", + video: { + aspectRatios: ["16:9"], + width: 640, + height: 360, + fps: FPS, + codec: "h264", + maxBitrateKbps: 6000, + maxFileSizeMb: 500, + }, + safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 }, +}); + +const motion = { + id: "none", + templates: () => [], + render: async (): Promise => { + throw new Error("no motion templates in this plan"); + }, +}; + +/** Privacy stand-in: region redactions through the inference provider, fail closed. */ +function emulatorPrivacy( + harness: RenderHarness, + inference: EmulatorMediaInferenceProvider, +): MediaPrivacyService { + const base = createFakePrivacy(harness.sources); + return { + ...base, + async prepareSources(input) { + const sources = new Map(); + for (const clip of input.edl.tracks.video) { + const source = harness.sources.get(clip.assetId); + if (!source) throw new Error(`no source for ${clip.assetId}`); + const regions = input.edl.redactions.regions.filter((r) => r.clipId === clip.id); + if (regions.length === 0) { + sources.set(clip.assetId, source); + continue; + } + const outPath = path.join(input.workDir, `redacted-${clip.id}.mp4`); + const result = await inference.redact({ + path: source.path, + outPath, + redactions: regions.map((region) => ({ + keyframes: region.keyframes, + mode: region.treatment === "cover" ? "cover" : "blur", + paddingPct: 10, + featherPct: 0, + })), + signal: input.signal, + }); + if (result.framesProcessed <= 0) throw new Error("redaction processed no frames"); + sources.set(clip.assetId, { ...source, path: result.outPath }); + } + return { sources, aiAlterations: [], fallbacks: [] }; + }, + }; +} + +interface Fixture { + harness: RenderHarness; + inference: EmulatorMediaInferenceProvider; + renderer: FfmpegMediaRenderer; + assetId: string; + sourcePath: string; + musicArtifactId: string; + render: (edl: unknown, quality: "draft" | "final") => Promise<{ id: string; dir: string }>; +} + +describe.skipIf(!hasFfmpeg())("media render integration with real ffmpeg", () => { + let fx: Fixture; + + beforeAll(async () => { + const renderer = new FfmpegMediaRenderer(); + const harness = await createRenderHarness({ + profiles: [vertical, landscape], + renderer, + motion, + }); + const inference = new EmulatorMediaInferenceProvider({ root: harness.mediaRoot }); + harness.deps.inference = inference; + const name = "lake__faces-1__speech__.mp4"; + const assetId = await harness.createAsset(name); + const sourcePath = harness.assetPath(assetId, name); + await writeSyntheticClip(sourcePath, { + seconds: 6, + width: 1280, + height: 720, + fps: FPS, + volumeDb: -32, + face: FACE, + }); + harness.setSource(assetId, { + path: sourcePath, + width: 1280, + height: 720, + durationMs: 6000, + fps: FPS, + hasAudio: true, + rotation: 0, + }); + const timesMs = Array.from({ length: 13 }, (_, i) => i * 500); + await harness.addAnalysis( + assetId, + "transcript", + await inference.transcribe({ path: sourcePath, language: "en" }), + ); + await harness.addAnalysis( + assetId, + "saliency", + await inference.saliency({ path: sourcePath, timesMs }), + ); + + const brandDir = path.join(harness.mediaRoot, "fixtures"); + await writeLogo(path.join(brandDir, "logo.png")); + await writeMusicBed(path.join(brandDir, "bed.wav"), 6); + const logoArtifactId = await harness.addArtifact( + path.join(brandDir, "logo.png"), + "logo.png", + "image/png", + ); + const musicArtifactId = await harness.addArtifact( + path.join(brandDir, "bed.wav"), + "bed.wav", + "audio/wav", + ); + const kit = renderTestKit({ logoArtifactId, musicArtifactId }); + + const privacy = emulatorPrivacy(harness, inference); + const service = createMediaRenderService(harness.deps); + const handler = createMediaRenderHandler(harness.deps, privacy); + const actor = { spaceId: harness.spaceId, userId: harness.userId }; + fx = { + harness, + inference, + renderer, + assetId, + sourcePath, + musicArtifactId, + async render(edl, quality) { + const projectId = await harness.createProject({ + assetIds: [assetId], + kit, + profileId: (edl as Edl).output.profileId, + }); + const plan = await harness.createPlan(projectId, edl); + const started = await service.start(actor, { planId: plan.id, quality }); + await handler({ renderId: started.id }); + const render = await harness.deps.repos.getRender(harness.spaceId, started.id); + expect(render.error).toBeNull(); + expect(render.status).toBe("done"); + return { + id: render.id, + dir: path.join(harness.mediaRoot, harness.spaceId, "renders", render.id), + }; + }, + }; + }); + + afterAll(async () => { + await fx?.harness.cleanup(); + }); + + it("renders 16:9 to 9:16 with auto-reframe, captions, brand overlay and loudness", async () => { + const edl = { + edlVersion: 1, + output: { + profileId: vertical.id, + width: vertical.video.width, + height: vertical.video.height, + fps: FPS, + durationTargetMs: 4000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [ + { id: "c1", assetId: fx.assetId, inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: fx.assetId, inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: fx.musicArtifactId, + inMs: 0, + outMs: 4000, + atMs: 0, + gainDb: -6, + fadeOutMs: 500, + duckUnderSpeech: true, + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "cut", durationMs: 0 }], + captions: { mode: "both", presetId: "clean" }, + overlays: [ + { id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 }, + { id: "lt", kind: "lower_third", atMs: 500, durationMs: 1500, text: "Example: 50% off" }, + ], + metadata: { title: "Example teaser" }, + }; + const render = await fx.render(edl, "final"); + const video = path.join(render.dir, "video.mp4"); + const probe = await fx.renderer.probe(video, { measureLoudness: true }); + + expect(Math.abs(probe.durationMs - 4000)).toBeLessThanOrEqual(FRAME_MS + 1); + expect(probe.width / probe.height).toBeCloseTo(9 / 16, 3); + expect(probe.width).toBe(vertical.video.width); + expect(probe.fps).toBe(FPS); + expect(probe.videoCodec).toBe("h264"); + expect(Math.abs(probe.integratedLufs! - vertical.audio.targetLufs)).toBeLessThanOrEqual(1); + expect(probe.truePeakDb!).toBeLessThanOrEqual(vertical.audio.truePeakDb + 0.5); + + expect(findMetadataLeaks(probe.tags)).toEqual([]); + const tagText = JSON.stringify(probe.tags).toLowerCase(); + for (const leak of ["location", "iso6709", "make", "model", "creation_time", "examplecam"]) { + expect(tagText).not.toContain(leak); + } + expect(probe.tags["format.title"]).toBe("Example teaser"); + + const poster = await fx.renderer.probe(path.join(render.dir, "poster.jpg")); + expect(findMetadataLeaks(poster.tags)).toEqual([]); + + const stored = await fx.harness.deps.repos.getRender(fx.harness.spaceId, render.id); + expect(stored.gateReport?.passed).toBe(true); + expect(stored.outputs.map((output) => output.fileName).sort()).toEqual([ + "captions.srt", + "captions.vtt", + "poster.jpg", + "video.mp4", + ]); + }, 240_000); + + it("tracks the generated face disc with the emulator", async () => { + const tracks = await fx.inference.trackFaces({ path: fx.sourcePath, sampleFps: 2 }); + expect(tracks.tracks).toHaveLength(1); + const disc = { x0: FACE.cx - FACE.radius * (9 / 16), x1: FACE.cx + FACE.radius * (9 / 16) }; + for (const { box } of tracks.tracks[0]!.keyframes) { + expect(box.x).toBeLessThanOrEqual(disc.x0); + expect(box.x + box.w).toBeGreaterThanOrEqual(disc.x1); + expect(box.y).toBeLessThanOrEqual(FACE.cy - FACE.radius); + expect(box.y + box.h).toBeGreaterThanOrEqual(FACE.cy + FACE.radius); + } + }); + + it("blurs a redaction region below the variance threshold and leaves the rest unchanged", async () => { + // A static region over testsrc2's fine checkerboard, held for the whole clip. + const box = { x: 0.68, y: 0.64, w: 0.14, h: 0.2 }; + const keyframes: BoxKeyframe[] = [ + { tMs: 0, box }, + { tMs: 3000, box }, + ]; + const edl = (regions: unknown[]) => ({ + edlVersion: 1, + output: { + profileId: landscape.id, + width: landscape.video.width, + height: landscape.video.height, + fps: FPS, + durationTargetMs: 3000, + durationHardMinMs: 1000, + durationHardMaxMs: 120_000, + }, + tracks: { + video: [{ id: "c1", assetId: fx.assetId, inMs: 0, outMs: 3000, atMs: 0, reframe: "none" }], + }, + redactions: { regions }, + metadata: {}, + }); + const plain = await fx.render(edl([]), "draft"); + const redacted = await fx.render( + edl([{ id: "region", clipId: "c1", keyframes, treatment: "blur" }]), + "draft", + ); + + const W = landscape.video.width; + const H = landscape.video.height; + const inner: PixelBox = { + x0: Math.ceil((box.x + box.w * 0.15) * W), + y0: Math.ceil((box.y + box.h * 0.15) * H), + x1: Math.floor((box.x + box.w * 0.85) * W), + y1: Math.floor((box.y + box.h * 0.85) * H), + }; + // Padding (10 %) and blur bleed at the edge stay out of the comparison. + const margin = 0.04; + const outer: PixelBox = { + x0: Math.floor((box.x - margin) * W), + y0: Math.floor((box.y - margin) * H), + x1: Math.ceil((box.x + box.w + margin) * W), + y1: Math.ceil((box.y + box.h + margin) * H), + }; + for (const tMs of [500, 1500, 2500]) { + const before = await grayFrame(path.join(plain.dir, "video.mp4"), tMs, W, H); + const after = await grayFrame(path.join(redacted.dir, "video.mp4"), tMs, W, H); + const original = variance(before, W, inner); + const blurred = variance(after, W, inner); + const outside = meanDifferenceOutside(before, after, W, H, outer); + expect(original).toBeGreaterThan(VARIANCE_THRESHOLD * 4); + expect(blurred).toBeLessThan(VARIANCE_THRESHOLD); + expect(outside).toBeLessThan(1.5); + } + }, 240_000); +});