import path from "node:path"; import type { BoxKeyframe, Edl } from "@rakazo/contracts"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import type { MediaPrivacyService } from "../../adapters/src/media-context.js"; import { EmulatorMediaInferenceProvider } from "../../adapters/src/media-emulator.js"; import { FfmpegMediaRenderer } from "../../adapters/src/media-ffmpeg-renderer.js"; import type { RenderHarness } from "../../adapters/src/media-render.fixtures.js"; import { createFakePrivacy, createRenderHarness, renderTestKit, renderTestProfile, } from "../../adapters/src/media-render.fixtures.js"; import { createMediaRenderHandler, createMediaRenderService, } from "../../adapters/src/media-render.js"; import { findMetadataLeaks } from "../../media/src/gate.js"; import type { PixelBox } from "./fixtures/media/synthetic-media.js"; import { grayFrame, hasFfmpeg, meanDifferenceOutside, variance, writeLogo, writeMusicBed, writeSyntheticClip, } from "./fixtures/media/synthetic-media.js"; /* * Real render integration (spec 13, Docker job). Synthetic fixtures are made * with ffmpeg lavfi; inference runs on the deterministic emulator; the render * goes through the real service, job handler, compiler, and ffmpeg renderer, * and every result is checked with ffprobe. Runs wherever ffmpeg is installed * (the CI test job and the media Docker job) and skips cleanly without it. * * Redactions belong to the privacy slice. Its stand-in here applies EDL region * redactions through the emulator's `redact` (the same MediaInferenceProvider * call the privacy service uses) and refuses to return an unredacted source. */ const FPS = 30; const FRAME_MS = 1000 / FPS; /** Grayscale variance a blurred region must stay under. */ const VARIANCE_THRESHOLD = 15; /** * The generated "face": a flat disc inside every box the media emulator can * report as the first face of a `__faces-1__` file (boxes are seeded by the * file, but always contain this point and radius). */ const FACE = { cx: 0.19, cy: 0.28, radius: 0.05 }; const vertical = renderTestProfile(); const landscape = renderTestProfile({ id: "example-landscape-test", name: "Example landscape test", video: { aspectRatios: ["16:9"], width: 640, height: 360, fps: FPS, codec: "h264", maxBitrateKbps: 6000, maxFileSizeMb: 500, }, safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 }, }); const motion = { id: "none", templates: () => [], render: async (): Promise => { throw new Error("no motion templates in this plan"); }, }; /** Privacy stand-in: region redactions through the inference provider, fail closed. */ function emulatorPrivacy( harness: RenderHarness, inference: EmulatorMediaInferenceProvider, ): MediaPrivacyService { const base = createFakePrivacy(harness.sources); return { ...base, async prepareSources(input) { const sources = new Map(); for (const clip of input.edl.tracks.video) { const source = harness.sources.get(clip.assetId); if (!source) throw new Error(`no source for ${clip.assetId}`); const regions = input.edl.redactions.regions.filter((r) => r.clipId === clip.id); if (regions.length === 0) { sources.set(clip.assetId, source); continue; } const outPath = path.join(input.workDir, `redacted-${clip.id}.mp4`); const result = await inference.redact({ path: source.path, outPath, redactions: regions.map((region) => ({ keyframes: region.keyframes, mode: region.treatment === "cover" ? "cover" : "blur", paddingPct: 10, featherPct: 0, })), signal: input.signal, }); if (result.framesProcessed <= 0) throw new Error("redaction processed no frames"); sources.set(clip.assetId, { ...source, path: result.outPath }); } return { sources, aiAlterations: [], fallbacks: [] }; }, }; } interface Fixture { harness: RenderHarness; inference: EmulatorMediaInferenceProvider; renderer: FfmpegMediaRenderer; assetId: string; sourcePath: string; musicArtifactId: string; render: (edl: unknown, quality: "draft" | "final") => Promise<{ id: string; dir: string }>; } describe.skipIf(!hasFfmpeg())("media render integration with real ffmpeg", () => { let fx: Fixture; beforeAll(async () => { const renderer = new FfmpegMediaRenderer(); const harness = await createRenderHarness({ profiles: [vertical, landscape], renderer, motion, }); const inference = new EmulatorMediaInferenceProvider({ root: harness.mediaRoot }); harness.deps.inference = inference; const name = "lake__faces-1__speech__.mp4"; const assetId = await harness.createAsset(name); const sourcePath = harness.assetPath(assetId, name); await writeSyntheticClip(sourcePath, { seconds: 6, width: 1280, height: 720, fps: FPS, volumeDb: -32, face: FACE, }); harness.setSource(assetId, { path: sourcePath, width: 1280, height: 720, durationMs: 6000, fps: FPS, hasAudio: true, rotation: 0, }); const timesMs = Array.from({ length: 13 }, (_, i) => i * 500); await harness.addAnalysis( assetId, "transcript", await inference.transcribe({ path: sourcePath, language: "en" }), ); await harness.addAnalysis( assetId, "saliency", await inference.saliency({ path: sourcePath, timesMs }), ); const brandDir = path.join(harness.mediaRoot, "fixtures"); await writeLogo(path.join(brandDir, "logo.png")); await writeMusicBed(path.join(brandDir, "bed.wav"), 6); const logoArtifactId = await harness.addArtifact( path.join(brandDir, "logo.png"), "logo.png", "image/png", ); const musicArtifactId = await harness.addArtifact( path.join(brandDir, "bed.wav"), "bed.wav", "audio/wav", ); const kit = renderTestKit({ logoArtifactId, musicArtifactId }); const privacy = emulatorPrivacy(harness, inference); const service = createMediaRenderService(harness.deps); const handler = createMediaRenderHandler(harness.deps, privacy); const actor = { spaceId: harness.spaceId, userId: harness.userId }; fx = { harness, inference, renderer, assetId, sourcePath, musicArtifactId, async render(edl, quality) { const projectId = await harness.createProject({ assetIds: [assetId], kit, profileId: (edl as Edl).output.profileId, }); const plan = await harness.createPlan(projectId, edl); const started = await service.start(actor, { planId: plan.id, quality }); await handler({ renderId: started.id }); const render = await harness.deps.repos.getRender(harness.spaceId, started.id); expect(render.error).toBeNull(); expect(render.status).toBe("done"); return { id: render.id, dir: path.join(harness.mediaRoot, harness.spaceId, "renders", render.id), }; }, }; }); afterAll(async () => { await fx?.harness.cleanup(); }); it("renders 16:9 to 9:16 with auto-reframe, captions, brand overlay and loudness", async () => { const edl = { edlVersion: 1, output: { profileId: vertical.id, width: vertical.video.width, height: vertical.video.height, fps: FPS, durationTargetMs: 4000, durationHardMinMs: 1000, durationHardMaxMs: 120_000, }, tracks: { video: [ { id: "c1", assetId: fx.assetId, inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, { id: "c2", assetId: fx.assetId, inMs: 3000, outMs: 5000, atMs: 2000, reframe: "auto" }, ], audio: [ { id: "bed", kind: "music", artifactId: fx.musicArtifactId, inMs: 0, outMs: 4000, atMs: 0, gainDb: -6, fadeOutMs: 500, duckUnderSpeech: true, }, ], }, transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "cut", durationMs: 0 }], captions: { mode: "both", presetId: "clean" }, overlays: [ { id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 }, { id: "lt", kind: "lower_third", atMs: 500, durationMs: 1500, text: "Example: 50% off" }, ], metadata: { title: "Example teaser" }, }; const render = await fx.render(edl, "final"); const video = path.join(render.dir, "video.mp4"); const probe = await fx.renderer.probe(video, { measureLoudness: true }); expect(Math.abs(probe.durationMs - 4000)).toBeLessThanOrEqual(FRAME_MS + 1); expect(probe.width / probe.height).toBeCloseTo(9 / 16, 3); expect(probe.width).toBe(vertical.video.width); expect(probe.fps).toBe(FPS); expect(probe.videoCodec).toBe("h264"); expect(Math.abs(probe.integratedLufs! - vertical.audio.targetLufs)).toBeLessThanOrEqual(1); expect(probe.truePeakDb!).toBeLessThanOrEqual(vertical.audio.truePeakDb + 0.5); expect(findMetadataLeaks(probe.tags)).toEqual([]); const tagText = JSON.stringify(probe.tags).toLowerCase(); for (const leak of ["location", "iso6709", "make", "model", "creation_time", "examplecam"]) { expect(tagText).not.toContain(leak); } expect(probe.tags["format.title"]).toBe("Example teaser"); const poster = await fx.renderer.probe(path.join(render.dir, "poster.jpg")); expect(findMetadataLeaks(poster.tags)).toEqual([]); const stored = await fx.harness.deps.repos.getRender(fx.harness.spaceId, render.id); expect(stored.gateReport?.passed).toBe(true); expect(stored.outputs.map((output) => output.fileName).sort()).toEqual([ "captions.srt", "captions.vtt", "poster.jpg", "video.mp4", ]); }, 240_000); it("tracks the generated face disc with the emulator", async () => { const tracks = await fx.inference.trackFaces({ path: fx.sourcePath, sampleFps: 2 }); expect(tracks.tracks).toHaveLength(1); const disc = { x0: FACE.cx - FACE.radius * (9 / 16), x1: FACE.cx + FACE.radius * (9 / 16) }; for (const { box } of tracks.tracks[0]!.keyframes) { expect(box.x).toBeLessThanOrEqual(disc.x0); expect(box.x + box.w).toBeGreaterThanOrEqual(disc.x1); expect(box.y).toBeLessThanOrEqual(FACE.cy - FACE.radius); expect(box.y + box.h).toBeGreaterThanOrEqual(FACE.cy + FACE.radius); } }); it("blurs a redaction region below the variance threshold and leaves the rest unchanged", async () => { // A static region over testsrc2's fine checkerboard, held for the whole clip. const box = { x: 0.68, y: 0.64, w: 0.14, h: 0.2 }; const keyframes: BoxKeyframe[] = [ { tMs: 0, box }, { tMs: 3000, box }, ]; const edl = (regions: unknown[]) => ({ edlVersion: 1, output: { profileId: landscape.id, width: landscape.video.width, height: landscape.video.height, fps: FPS, durationTargetMs: 3000, durationHardMinMs: 1000, durationHardMaxMs: 120_000, }, tracks: { video: [{ id: "c1", assetId: fx.assetId, inMs: 0, outMs: 3000, atMs: 0, reframe: "none" }], }, redactions: { regions }, metadata: {}, }); const plain = await fx.render(edl([]), "draft"); const redacted = await fx.render( edl([{ id: "region", clipId: "c1", keyframes, treatment: "blur" }]), "draft", ); const W = landscape.video.width; const H = landscape.video.height; const inner: PixelBox = { x0: Math.ceil((box.x + box.w * 0.15) * W), y0: Math.ceil((box.y + box.h * 0.15) * H), x1: Math.floor((box.x + box.w * 0.85) * W), y1: Math.floor((box.y + box.h * 0.85) * H), }; // Padding (10 %) and blur bleed at the edge stay out of the comparison. const margin = 0.04; const outer: PixelBox = { x0: Math.floor((box.x - margin) * W), y0: Math.floor((box.y - margin) * H), x1: Math.ceil((box.x + box.w + margin) * W), y1: Math.ceil((box.y + box.h + margin) * H), }; for (const tMs of [500, 1500, 2500]) { const before = await grayFrame(path.join(plain.dir, "video.mp4"), tMs, W, H); const after = await grayFrame(path.join(redacted.dir, "video.mp4"), tMs, W, H); const original = variance(before, W, inner); const blurred = variance(after, W, inner); const outside = meanDifferenceOutside(before, after, W, H, outer); expect(original).toBeGreaterThan(VARIANCE_THRESHOLD * 4); expect(blurred).toBeLessThan(VARIANCE_THRESHOLD); expect(outside).toBeLessThan(1.5); } }, 240_000); });