270ea8068a
Add the Media Studio adapters behind the frozen provider contracts: a deterministic offline inference emulator driven by file-name markers that renders real redactions with ffmpeg, a synthetic emulator library, the bearer-auth HTTP client for the local media sidecar with validated responses, typed errors, busy retries and timeouts, a lazily paginated folder source that never follows symlinks, and the configuration factory. A shared conformance suite runs against the emulator and, in the Docker integration job, against the sidecar. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj
336 lines
14 KiB
TypeScript
336 lines
14 KiB
TypeScript
import { copyFile, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
|
import { cosineSimilarity, haversineKm } from "@rakazo/media";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import {
|
|
buildRedactionGraph,
|
|
EmulatorMediaInferenceProvider,
|
|
parseEmulatorMarkers,
|
|
redactionRegions,
|
|
} from "./media-emulator.js";
|
|
import { ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js";
|
|
import { EmulatorMediaSourceProvider, emulatorAsset } from "./media-emulator-source.js";
|
|
import { probeMedia } from "./media-folder-source-probe.js";
|
|
|
|
const hasFfmpeg = ffmpegAvailable();
|
|
let root: string;
|
|
let provider: EmulatorMediaInferenceProvider;
|
|
|
|
beforeAll(async () => {
|
|
root = await mkdtemp(path.join(tmpdir(), "media-emulator-"));
|
|
provider = new EmulatorMediaInferenceProvider({ root });
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await rm(root, { recursive: true, force: true });
|
|
});
|
|
|
|
/** A non-media file: the emulator falls back to a 10 s clip, so no ffmpeg is needed. */
|
|
async function fixture(name: string, content = "synthetic bytes") {
|
|
const file = path.join(root, name);
|
|
await writeFile(file, content);
|
|
return file;
|
|
}
|
|
|
|
describe("parseEmulatorMarkers", () => {
|
|
it("reads every documented marker", () => {
|
|
expect(
|
|
parseEmulatorMarkers(
|
|
"/x/clip__faces-2____person-a____noface-gap-1000-2500____nsfw-explicit_nudity-0.9____nsfw-violence-0.4____cut-1500____cut-3000____speech____theme-b____inpaintq-0.3__.mp4",
|
|
),
|
|
).toEqual({
|
|
faces: 2,
|
|
person: "a",
|
|
noFaceGaps: [{ startMs: 1000, endMs: 2500 }],
|
|
nsfw: { explicit_nudity: 0.9, violence: 0.4 },
|
|
cutsMs: [1500, 3000],
|
|
speech: true,
|
|
theme: "b",
|
|
inpaintQuality: 0.3,
|
|
});
|
|
expect(parseEmulatorMarkers("__nsfw-unknown-0.5____nsfw-violence-7__.mp4").nsfw).toEqual({});
|
|
});
|
|
});
|
|
|
|
describe("redaction regions", () => {
|
|
it("holds the union of neighbouring keyframes and bounds the graph", () => {
|
|
const regions = redactionRegions(
|
|
{
|
|
keyframes: [
|
|
{ tMs: 1000, box: { x: 0.5, y: 0.5, w: 0.1, h: 0.1 } },
|
|
{ tMs: 0, box: { x: 0.1, y: 0.1, w: 0.1, h: 0.1 } },
|
|
],
|
|
mode: "blur",
|
|
paddingPct: 0,
|
|
featherPct: 0,
|
|
},
|
|
1000,
|
|
1000,
|
|
);
|
|
expect(regions).toEqual([
|
|
{ startMs: 0, endMs: 1000, box: { x: 100, y: 100, w: 500, h: 500 } },
|
|
{ startMs: 1000, endMs: 1100, box: { x: 500, y: 500, w: 100, h: 100 } },
|
|
]);
|
|
const keyframes = Array.from({ length: 5000 }, (_, i) => ({
|
|
tMs: i * 40,
|
|
box: { x: (i % 50) / 100, y: 0.2, w: 0.1, h: 0.1 },
|
|
}));
|
|
const many = redactionRegions(
|
|
{ keyframes, mode: "pixelate", paddingPct: 25, featherPct: 0 },
|
|
1920,
|
|
1080,
|
|
);
|
|
expect(many.length).toBeLessThanOrEqual(400);
|
|
expect(many[0]?.startMs).toBe(0);
|
|
expect(many.at(-1)?.endMs).toBe(4999 * 40 + 100);
|
|
const graph = buildRedactionGraph(
|
|
[{ keyframes, mode: "solid", paddingPct: 0, featherPct: 0, color: "#ff0000" }],
|
|
1920,
|
|
1080,
|
|
);
|
|
expect(graph.filter).toContain("color=0xff0000@1:t=fill");
|
|
});
|
|
});
|
|
|
|
describe("EmulatorMediaInferenceProvider", () => {
|
|
it("is quiet without markers", async () => {
|
|
const file = await fixture("plain.mp4");
|
|
const scenes = await provider.detectScenes({ path: file });
|
|
expect(scenes.scenes).toEqual([{ startMs: 0, endMs: 10_000 }]);
|
|
expect(scenes.shots).toHaveLength(1);
|
|
expect((await provider.trackFaces({ path: file, sampleFps: 2 })).tracks).toEqual([]);
|
|
const faces = await provider.detectFaces({ path: file, sampleFps: 1 });
|
|
expect(faces.frames).toHaveLength(10);
|
|
expect(faces.frames.every((frame) => frame.faces.length === 0)).toBe(true);
|
|
const classified = await provider.classifyContent({ path: file, sampleFps: 2 });
|
|
expect(classified.frames).toHaveLength(20);
|
|
for (const frame of classified.frames)
|
|
expect(Object.values(frame.scores).every((score) => score === 0.01)).toBe(true);
|
|
expect(await provider.transcribe({ path: file })).toEqual({ language: null, words: [] });
|
|
});
|
|
|
|
it("forces faces, gaps, cuts, classifier scores and speech from the file name", async () => {
|
|
const file = await fixture(
|
|
"rich__faces-2____noface-gap-2000-3000____cut-4000____nsfw-explicit_nudity-0.9____speech__.mp4",
|
|
);
|
|
const tracks = await provider.trackFaces({ path: file, sampleFps: 2 });
|
|
expect(tracks.tracks.map((track) => track.trackId)).toEqual(["track-1", "track-2"]);
|
|
const times = tracks.tracks[0]?.keyframes.map((keyframe) => keyframe.tMs) ?? [];
|
|
expect(times).toContain(1500);
|
|
expect(times).not.toContain(2000);
|
|
expect(times).not.toContain(2500);
|
|
expect(times).toContain(3000);
|
|
const gapFrame = await provider.detectFaces({
|
|
path: file,
|
|
sampleFps: 1,
|
|
timesMs: [2500, 3500],
|
|
});
|
|
expect(gapFrame.frames.map((frame) => frame.faces.length)).toEqual([0, 2]);
|
|
|
|
expect((await provider.detectScenes({ path: file })).scenes).toEqual([
|
|
{ startMs: 0, endMs: 4000 },
|
|
{ startMs: 4000, endMs: 10_000 },
|
|
]);
|
|
const classified = await provider.classifyContent({
|
|
path: file,
|
|
sampleFps: 0.5,
|
|
extraTimesMs: [4000, 4001],
|
|
});
|
|
expect(classified.frames.map((frame) => frame.tMs)).toEqual([0, 2000, 4000, 4001, 6000, 8000]);
|
|
for (const frame of classified.frames) {
|
|
expect(frame.scores.explicit_nudity).toBe(0.9);
|
|
expect(frame.scores.violence).toBe(0.01);
|
|
}
|
|
const english = await provider.transcribe({ path: file });
|
|
const german = await provider.transcribe({ path: file, language: "de-DE" });
|
|
expect(english.language).toBe("en");
|
|
expect(english.words.map((word) => word.text).slice(0, 2)).toEqual(["welcome", "back"]);
|
|
expect(german.language).toBe("de");
|
|
expect(german.words[0]).toEqual({
|
|
startMs: 400,
|
|
endMs: 720,
|
|
text: "willkommen",
|
|
confidence: 0.95,
|
|
});
|
|
});
|
|
|
|
it("is deterministic per file content and keeps boxes stable", async () => {
|
|
const file = await fixture("same__faces-1__.mp4");
|
|
const copy = path.join(root, "copy__faces-1__.mp4");
|
|
await copyFile(file, copy);
|
|
const a = await provider.detectFaces({ path: file, sampleFps: 4 });
|
|
const b = await provider.detectFaces({ path: file, sampleFps: 4 });
|
|
const c = await provider.detectFaces({ path: copy, sampleFps: 4 });
|
|
expect(a).toEqual(b);
|
|
expect(a).toEqual(c);
|
|
const boxes = a.frames.map((frame) => frame.faces[0]?.box);
|
|
expect(Math.abs((boxes.at(-1)?.x ?? 0) - (boxes[0]?.x ?? 0))).toBeLessThan(0.05);
|
|
});
|
|
|
|
it("embeds people and themes consistently", async () => {
|
|
const personA1 = await fixture("one__faces-1____person-a__.mp4", "first");
|
|
const personA2 = await fixture("two__faces-1____person-a__.mp4", "second");
|
|
const personB = await fixture("three__faces-1____person-b__.mp4", "third");
|
|
const embed = async (file: string) =>
|
|
(await provider.trackFaces({ path: file, sampleFps: 1 })).tracks[0]?.embedding ?? [];
|
|
expect(cosineSimilarity(await embed(personA1), await embed(personA2))).toBeCloseTo(1, 5);
|
|
expect(cosineSimilarity(await embed(personA1), await embed(personB))).toBeLessThan(0.8);
|
|
|
|
const themeA1 = await fixture("x__theme-a__.mp4", "x");
|
|
const themeA2 = await fixture("y__theme-a__.mp4", "y");
|
|
const themeB = await fixture("z__theme-b__.mp4", "z");
|
|
const mean = async (file: string) =>
|
|
(await provider.embedFrames({ path: file, timesMs: [0, 1000] })).vectors[0] ?? [];
|
|
expect(cosineSimilarity(await mean(themeA1), await mean(themeA2))).toBeGreaterThan(0.95);
|
|
expect(cosineSimilarity(await mean(themeA1), await mean(themeB))).toBeLessThan(0.85);
|
|
});
|
|
|
|
it("never synthesizes identities", async () => {
|
|
const file = await fixture("synth.mp4");
|
|
const capabilities = await provider.capabilities();
|
|
expect(capabilities.operations.synthesizeIdentity).toMatchObject({
|
|
available: false,
|
|
license: null,
|
|
});
|
|
expect(capabilities.operations.inpaintVideo).toMatchObject({
|
|
available: true,
|
|
experimental: true,
|
|
});
|
|
await expect(
|
|
provider.synthesizeIdentity({
|
|
path: file,
|
|
track: [],
|
|
seed: 1,
|
|
outPath: path.join(root, "s.mp4"),
|
|
}),
|
|
).rejects.toBeInstanceOf(UnsupportedMediaOperationError);
|
|
});
|
|
|
|
it("rejects paths outside its root", async () => {
|
|
await expect(provider.detectScenes({ path: "/etc/hostname" })).rejects.toThrow(
|
|
"outside the allowed root",
|
|
);
|
|
await expect(provider.detectScenes({ path: "relative.mp4" })).rejects.toThrow();
|
|
});
|
|
|
|
it.skipIf(!hasFfmpeg)("writes masks and inpaints with the reported quality", async () => {
|
|
const clip = await renderTestClip(path.join(root, "mark__inpaintq-0.4__.mp4"), {
|
|
seconds: 1,
|
|
audio: false,
|
|
});
|
|
const masks = await provider.segmentPrompted({
|
|
path: clip,
|
|
prompt: { tMs: 0, point: { x: 0.5, y: 0.5 } },
|
|
propagate: true,
|
|
outDir: path.join(root, "marks/m1"),
|
|
});
|
|
expect((await readdir(masks.maskDir)).sort()).toEqual(["0.png", "500.png"]);
|
|
expect(masks.keyframes[0]?.box).toEqual({ x: 0.44, y: 0.44, w: 0.12, h: 0.12 });
|
|
const inpainted = await provider.inpaintVideo({
|
|
path: clip,
|
|
maskDir: masks.maskDir,
|
|
outPath: path.join(root, "marks/m1.mp4"),
|
|
});
|
|
expect(inpainted.quality.score).toBe(0.4);
|
|
expect((await probeMedia(inpainted.outPath)).durationMs).toBeGreaterThan(900);
|
|
});
|
|
|
|
it.skipIf(!hasFfmpeg)("renders very large redaction graphs through a script file", async () => {
|
|
const clip = await renderTestClip(path.join(root, "busy.mp4"), { seconds: 1, audio: false });
|
|
const instruction = (offset: number) => ({
|
|
keyframes: Array.from({ length: 2000 }, (_, i) => ({
|
|
tMs: i,
|
|
box: { x: offset + (i % 40) / 400, y: 0.1, w: 0.1, h: 0.1 },
|
|
})),
|
|
mode: "blur" as const,
|
|
paddingPct: 0,
|
|
featherPct: 0,
|
|
});
|
|
const instructions = [instruction(0), instruction(0.3), instruction(0.6)];
|
|
expect(buildRedactionGraph(instructions, 320, 180).filter.length).toBeGreaterThan(100_000);
|
|
const result = await provider.redact({
|
|
path: clip,
|
|
redactions: instructions,
|
|
outPath: path.join(root, "busy-redacted.mp4"),
|
|
});
|
|
expect(result.framesProcessed).toBe(25);
|
|
});
|
|
|
|
it.skipIf(!hasFfmpeg)("measures loudness with ebur128", async () => {
|
|
const clip = await renderTestClip(path.join(root, "tone.mp4"), { seconds: 2 });
|
|
const features = await provider.audioFeatures({ path: clip });
|
|
expect(features.integratedLufs).toBeLessThan(-10);
|
|
expect(features.integratedLufs).toBeGreaterThan(-40);
|
|
expect(features.energy.length).toBeGreaterThan(0);
|
|
expect(features.beatsMs.length).toBeGreaterThan(0);
|
|
const silent = await renderTestClip(path.join(root, "mute.mp4"), { seconds: 1, audio: false });
|
|
expect(await provider.audioFeatures({ path: silent })).toMatchObject({
|
|
integratedLufs: null,
|
|
silences: [{ startMs: 0, endMs: 1000 }],
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("EmulatorMediaSourceProvider", () => {
|
|
const config = { count: 30, seed: 4 };
|
|
|
|
it("describes three sessions, two themes and the marker assets", async () => {
|
|
const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") });
|
|
const first = await source.list({ config, cursor: null, limit: 12 });
|
|
const second = await source.list({ config, cursor: first.nextCursor, limit: 100 });
|
|
expect(second.nextCursor).toBeNull();
|
|
const items = [...first.items, ...second.items];
|
|
expect(items).toHaveLength(30);
|
|
expect(new Set(items.map((item) => item.externalId)).size).toBe(30);
|
|
expect(await source.list({ config, cursor: null, limit: 12 })).toEqual(first);
|
|
|
|
const assets = items.map((_, index) => emulatorAsset(config.seed, config.count, index));
|
|
expect(new Set(assets.map((asset) => asset.session))).toEqual(new Set([0, 1, 2]));
|
|
expect(new Set(assets.map((asset) => asset.theme))).toEqual(new Set(["a", "b"]));
|
|
const sessionStarts = [0, 1, 2].map(
|
|
(session) => assets.find((asset) => asset.session === session)!,
|
|
);
|
|
expect(
|
|
haversineKm(
|
|
{ lat: sessionStarts[0]!.gpsLat, lon: sessionStarts[0]!.gpsLon },
|
|
{ lat: sessionStarts[1]!.gpsLat, lon: sessionStarts[1]!.gpsLon },
|
|
),
|
|
).toBeGreaterThan(2);
|
|
const names = items.map((item) => item.externalId).join(" ");
|
|
expect(names).toContain("__faces-1____person-a__");
|
|
expect(names).toContain("__faces-1____person-b__");
|
|
expect(names).toContain("__nsfw-explicit_nudity-0.9__");
|
|
expect(names).toContain("__speech__");
|
|
expect(items.some((item) => item.kind === "photo")).toBe(true);
|
|
});
|
|
|
|
it("rejects unknown external ids", async () => {
|
|
const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") });
|
|
for (const bad of [
|
|
"../etc/passwd",
|
|
"emu-4-00099__theme-b__.mp4",
|
|
"emu-5-00000__theme-a__.mp4",
|
|
"emu-4-00000.mp4",
|
|
])
|
|
await expect(source.resolve(config, bad), bad).rejects.toThrow();
|
|
});
|
|
|
|
it.skipIf(!hasFfmpeg)("renders and caches clips on demand", async () => {
|
|
const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") });
|
|
const asset = emulatorAsset(config.seed, config.count, 0);
|
|
const file = await source.resolve(config, asset.fileName);
|
|
expect(path.basename(file)).toBe(asset.fileName);
|
|
const probed = await probeMedia(file);
|
|
expect(probed).toMatchObject({ durationMs: 4000, width: 320, height: 180, hasAudio: true });
|
|
expect(probed.capturedAt?.toISOString()).toBe(asset.capturedAt.toISOString());
|
|
expect(await source.resolve(config, asset.fileName)).toBe(file);
|
|
const tracks = await provider.trackFaces({ path: file, sampleFps: 1 });
|
|
expect(tracks.tracks).toHaveLength(1);
|
|
const photo = emulatorAsset(config.seed, config.count, 4);
|
|
expect(photo.kind).toBe("photo");
|
|
expect((await probeMedia(await source.resolve(config, photo.fileName))).width).toBe(320);
|
|
});
|
|
});
|