import { copyFile, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; import { cosineSimilarity, haversineKm } from "@rakazo/media"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { buildRedactionGraph, EmulatorMediaInferenceProvider, parseEmulatorMarkers, redactionRegions, } from "./media-emulator.js"; import { ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js"; import { EmulatorMediaSourceProvider, emulatorAsset } from "./media-emulator-source.js"; import { probeMedia } from "./media-folder-source-probe.js"; const hasFfmpeg = ffmpegAvailable(); let root: string; let provider: EmulatorMediaInferenceProvider; beforeAll(async () => { root = await mkdtemp(path.join(tmpdir(), "media-emulator-")); provider = new EmulatorMediaInferenceProvider({ root }); }); afterAll(async () => { await rm(root, { recursive: true, force: true }); }); /** A non-media file: the emulator falls back to a 10 s clip, so no ffmpeg is needed. */ async function fixture(name: string, content = "synthetic bytes") { const file = path.join(root, name); await writeFile(file, content); return file; } describe("parseEmulatorMarkers", () => { it("reads every documented marker", () => { expect( parseEmulatorMarkers( "/x/clip__faces-2____person-a____noface-gap-1000-2500____nsfw-explicit_nudity-0.9____nsfw-violence-0.4____cut-1500____cut-3000____speech____theme-b____inpaintq-0.3__.mp4", ), ).toEqual({ faces: 2, person: "a", noFaceGaps: [{ startMs: 1000, endMs: 2500 }], nsfw: { explicit_nudity: 0.9, violence: 0.4 }, cutsMs: [1500, 3000], speech: true, theme: "b", inpaintQuality: 0.3, }); expect(parseEmulatorMarkers("__nsfw-unknown-0.5____nsfw-violence-7__.mp4").nsfw).toEqual({}); }); }); describe("redaction regions", () => { it("holds the union of neighbouring keyframes and bounds the graph", () => { const regions = redactionRegions( { keyframes: [ { tMs: 1000, box: { x: 0.5, y: 0.5, w: 0.1, h: 0.1 } }, { tMs: 0, box: { x: 0.1, y: 0.1, w: 0.1, h: 0.1 } }, ], mode: "blur", paddingPct: 0, featherPct: 0, }, 1000, 1000, ); expect(regions).toEqual([ { startMs: 0, endMs: 1000, box: { x: 100, y: 100, w: 500, h: 500 } }, { startMs: 1000, endMs: 1100, box: { x: 500, y: 500, w: 100, h: 100 } }, ]); const keyframes = Array.from({ length: 5000 }, (_, i) => ({ tMs: i * 40, box: { x: (i % 50) / 100, y: 0.2, w: 0.1, h: 0.1 }, })); const many = redactionRegions( { keyframes, mode: "pixelate", paddingPct: 25, featherPct: 0 }, 1920, 1080, ); expect(many.length).toBeLessThanOrEqual(400); expect(many[0]?.startMs).toBe(0); expect(many.at(-1)?.endMs).toBe(4999 * 40 + 100); const graph = buildRedactionGraph( [{ keyframes, mode: "solid", paddingPct: 0, featherPct: 0, color: "#ff0000" }], 1920, 1080, ); expect(graph.filter).toContain("color=0xff0000@1:t=fill"); }); }); describe("EmulatorMediaInferenceProvider", () => { it("is quiet without markers", async () => { const file = await fixture("plain.mp4"); const scenes = await provider.detectScenes({ path: file }); expect(scenes.scenes).toEqual([{ startMs: 0, endMs: 10_000 }]); expect(scenes.shots).toHaveLength(1); expect((await provider.trackFaces({ path: file, sampleFps: 2 })).tracks).toEqual([]); const faces = await provider.detectFaces({ path: file, sampleFps: 1 }); expect(faces.frames).toHaveLength(10); expect(faces.frames.every((frame) => frame.faces.length === 0)).toBe(true); const classified = await provider.classifyContent({ path: file, sampleFps: 2 }); expect(classified.frames).toHaveLength(20); for (const frame of classified.frames) expect(Object.values(frame.scores).every((score) => score === 0.01)).toBe(true); expect(await provider.transcribe({ path: file })).toEqual({ language: null, words: [] }); }); it("forces faces, gaps, cuts, classifier scores and speech from the file name", async () => { const file = await fixture( "rich__faces-2____noface-gap-2000-3000____cut-4000____nsfw-explicit_nudity-0.9____speech__.mp4", ); const tracks = await provider.trackFaces({ path: file, sampleFps: 2 }); expect(tracks.tracks.map((track) => track.trackId)).toEqual(["track-1", "track-2"]); const times = tracks.tracks[0]?.keyframes.map((keyframe) => keyframe.tMs) ?? []; expect(times).toContain(1500); expect(times).not.toContain(2000); expect(times).not.toContain(2500); expect(times).toContain(3000); const gapFrame = await provider.detectFaces({ path: file, sampleFps: 1, timesMs: [2500, 3500], }); expect(gapFrame.frames.map((frame) => frame.faces.length)).toEqual([0, 2]); expect((await provider.detectScenes({ path: file })).scenes).toEqual([ { startMs: 0, endMs: 4000 }, { startMs: 4000, endMs: 10_000 }, ]); const classified = await provider.classifyContent({ path: file, sampleFps: 0.5, extraTimesMs: [4000, 4001], }); expect(classified.frames.map((frame) => frame.tMs)).toEqual([0, 2000, 4000, 4001, 6000, 8000]); for (const frame of classified.frames) { expect(frame.scores.explicit_nudity).toBe(0.9); expect(frame.scores.violence).toBe(0.01); } const english = await provider.transcribe({ path: file }); const german = await provider.transcribe({ path: file, language: "de-DE" }); expect(english.language).toBe("en"); expect(english.words.map((word) => word.text).slice(0, 2)).toEqual(["welcome", "back"]); expect(german.language).toBe("de"); expect(german.words[0]).toEqual({ startMs: 400, endMs: 720, text: "willkommen", confidence: 0.95, }); }); it("is deterministic per file content and keeps boxes stable", async () => { const file = await fixture("same__faces-1__.mp4"); const copy = path.join(root, "copy__faces-1__.mp4"); await copyFile(file, copy); const a = await provider.detectFaces({ path: file, sampleFps: 4 }); const b = await provider.detectFaces({ path: file, sampleFps: 4 }); const c = await provider.detectFaces({ path: copy, sampleFps: 4 }); expect(a).toEqual(b); expect(a).toEqual(c); const boxes = a.frames.map((frame) => frame.faces[0]?.box); expect(Math.abs((boxes.at(-1)?.x ?? 0) - (boxes[0]?.x ?? 0))).toBeLessThan(0.05); }); it("embeds people and themes consistently", async () => { const personA1 = await fixture("one__faces-1____person-a__.mp4", "first"); const personA2 = await fixture("two__faces-1____person-a__.mp4", "second"); const personB = await fixture("three__faces-1____person-b__.mp4", "third"); const embed = async (file: string) => (await provider.trackFaces({ path: file, sampleFps: 1 })).tracks[0]?.embedding ?? []; expect(cosineSimilarity(await embed(personA1), await embed(personA2))).toBeCloseTo(1, 5); expect(cosineSimilarity(await embed(personA1), await embed(personB))).toBeLessThan(0.8); const themeA1 = await fixture("x__theme-a__.mp4", "x"); const themeA2 = await fixture("y__theme-a__.mp4", "y"); const themeB = await fixture("z__theme-b__.mp4", "z"); const mean = async (file: string) => (await provider.embedFrames({ path: file, timesMs: [0, 1000] })).vectors[0] ?? []; expect(cosineSimilarity(await mean(themeA1), await mean(themeA2))).toBeGreaterThan(0.95); expect(cosineSimilarity(await mean(themeA1), await mean(themeB))).toBeLessThan(0.85); }); it("never synthesizes identities", async () => { const file = await fixture("synth.mp4"); const capabilities = await provider.capabilities(); expect(capabilities.operations.synthesizeIdentity).toMatchObject({ available: false, license: null, }); expect(capabilities.operations.inpaintVideo).toMatchObject({ available: true, experimental: true, }); await expect( provider.synthesizeIdentity({ path: file, track: [], seed: 1, outPath: path.join(root, "s.mp4"), }), ).rejects.toBeInstanceOf(UnsupportedMediaOperationError); }); it("rejects paths outside its root", async () => { await expect(provider.detectScenes({ path: "/etc/hostname" })).rejects.toThrow( "outside the allowed root", ); await expect(provider.detectScenes({ path: "relative.mp4" })).rejects.toThrow(); }); it.skipIf(!hasFfmpeg)("writes masks and inpaints with the reported quality", async () => { const clip = await renderTestClip(path.join(root, "mark__inpaintq-0.4__.mp4"), { seconds: 1, audio: false, }); const masks = await provider.segmentPrompted({ path: clip, prompt: { tMs: 0, point: { x: 0.5, y: 0.5 } }, propagate: true, outDir: path.join(root, "marks/m1"), }); expect((await readdir(masks.maskDir)).sort()).toEqual(["0.png", "500.png"]); expect(masks.keyframes[0]?.box).toEqual({ x: 0.44, y: 0.44, w: 0.12, h: 0.12 }); const inpainted = await provider.inpaintVideo({ path: clip, maskDir: masks.maskDir, outPath: path.join(root, "marks/m1.mp4"), }); expect(inpainted.quality.score).toBe(0.4); expect((await probeMedia(inpainted.outPath)).durationMs).toBeGreaterThan(900); }); it.skipIf(!hasFfmpeg)("renders very large redaction graphs through a script file", async () => { const clip = await renderTestClip(path.join(root, "busy.mp4"), { seconds: 1, audio: false }); const instruction = (offset: number) => ({ keyframes: Array.from({ length: 2000 }, (_, i) => ({ tMs: i, box: { x: offset + (i % 40) / 400, y: 0.1, w: 0.1, h: 0.1 }, })), mode: "blur" as const, paddingPct: 0, featherPct: 0, }); const instructions = [instruction(0), instruction(0.3), instruction(0.6)]; expect(buildRedactionGraph(instructions, 320, 180).filter.length).toBeGreaterThan(100_000); const result = await provider.redact({ path: clip, redactions: instructions, outPath: path.join(root, "busy-redacted.mp4"), }); expect(result.framesProcessed).toBe(25); }); it.skipIf(!hasFfmpeg)("measures loudness with ebur128", async () => { const clip = await renderTestClip(path.join(root, "tone.mp4"), { seconds: 2 }); const features = await provider.audioFeatures({ path: clip }); expect(features.integratedLufs).toBeLessThan(-10); expect(features.integratedLufs).toBeGreaterThan(-40); expect(features.energy.length).toBeGreaterThan(0); expect(features.beatsMs.length).toBeGreaterThan(0); const silent = await renderTestClip(path.join(root, "mute.mp4"), { seconds: 1, audio: false }); expect(await provider.audioFeatures({ path: silent })).toMatchObject({ integratedLufs: null, silences: [{ startMs: 0, endMs: 1000 }], }); }); }); describe("EmulatorMediaSourceProvider", () => { const config = { count: 30, seed: 4 }; it("describes three sessions, two themes and the marker assets", async () => { const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") }); const first = await source.list({ config, cursor: null, limit: 12 }); const second = await source.list({ config, cursor: first.nextCursor, limit: 100 }); expect(second.nextCursor).toBeNull(); const items = [...first.items, ...second.items]; expect(items).toHaveLength(30); expect(new Set(items.map((item) => item.externalId)).size).toBe(30); expect(await source.list({ config, cursor: null, limit: 12 })).toEqual(first); const assets = items.map((_, index) => emulatorAsset(config.seed, config.count, index)); expect(new Set(assets.map((asset) => asset.session))).toEqual(new Set([0, 1, 2])); expect(new Set(assets.map((asset) => asset.theme))).toEqual(new Set(["a", "b"])); const sessionStarts = [0, 1, 2].map( (session) => assets.find((asset) => asset.session === session)!, ); expect( haversineKm( { lat: sessionStarts[0]!.gpsLat, lon: sessionStarts[0]!.gpsLon }, { lat: sessionStarts[1]!.gpsLat, lon: sessionStarts[1]!.gpsLon }, ), ).toBeGreaterThan(2); const names = items.map((item) => item.externalId).join(" "); expect(names).toContain("__faces-1____person-a__"); expect(names).toContain("__faces-1____person-b__"); expect(names).toContain("__nsfw-explicit_nudity-0.9__"); expect(names).toContain("__speech__"); expect(items.some((item) => item.kind === "photo")).toBe(true); }); it("rejects unknown external ids", async () => { const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") }); for (const bad of [ "../etc/passwd", "emu-4-00099__theme-b__.mp4", "emu-5-00000__theme-a__.mp4", "emu-4-00000.mp4", ]) await expect(source.resolve(config, bad), bad).rejects.toThrow(); }); it.skipIf(!hasFfmpeg)("renders and caches clips on demand", async () => { const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") }); const asset = emulatorAsset(config.seed, config.count, 0); const file = await source.resolve(config, asset.fileName); expect(path.basename(file)).toBe(asset.fileName); const probed = await probeMedia(file); expect(probed).toMatchObject({ durationMs: 4000, width: 320, height: 180, hasAudio: true }); expect(probed.capturedAt?.toISOString()).toBe(asset.capturedAt.toISOString()); expect(await source.resolve(config, asset.fileName)).toBe(file); const tracks = await provider.trackFaces({ path: file, sampleFps: 1 }); expect(tracks.tracks).toHaveLength(1); const photo = emulatorAsset(config.seed, config.count, 4); expect(photo.kind).toBe("photo"); expect((await probeMedia(await source.resolve(config, photo.fileName))).width).toBe(320); }); });