cynarAI/evOo-local
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat(media): anonymization, marks, consent, and the render gate

Browse Source 
prepareSources applies every face, mark, and region protection before the
compiler runs and fails closed when one cannot be honored: all face modes,
tracking with interpolation and grown holds over lost detections, reframe_out
crops with blur fallback, synthetic identity only through a licensed provider
(blur fallback otherwise), inpainting with a quality floor that falls back to
a cover, and person-bound marks re-found next to the face. The gate runs on
the render output: classifier at the profile rate plus cuts, face tracking
matched to people locally, pixel verification of every recorded protection,
consent and age proof (always for paid video), text and technical checks.
Auto-fix proposes only compliance repairs and stops after three rounds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w
This commit is contained in:
Claude 2026-10-03 18:57:04 +00:00
parent d6957c4670
commit 8d07ab17d4
No known key found for this signature in database
17 changed files with 4681 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

424
packages/adapters/src/media-anonymize.test.ts Normal file
View File

@ -0,0 +1,424 @@
import { copyFile } from "node:fs/promises";
import path from "node:path";
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
import type { AnonymizeMode, Edl } from "@rakazo/contracts";
import { afterEach, describe, expect, it, vi } from "vitest";
import { prepareSources } from "./media-anonymize.js";
import { ffmpegAvailable } from "./media-emulator-clips.js";
import { runGate } from "./media-gate.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import {
ACTOR,
cleanProbe,
createPrivacyHarness,
testEdl,
testProfile,
} from "./media-privacy-fixtures.js";
import { PrivacyUnavailableError } from "./media-privacy-io.js";
import { extractLuma, residualVisibility } from "./media-privacy-pixels.js";
import { readPrivacyRecord } from "./media-privacy-plan.js";
const hasFfmpeg = ffmpegAvailable();
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
/** One asset with one face of person "alice" (stored track), one clip over it. */
async function setup(
mode: AnonymizeMode,
options: {
fileName?: string;
edl?: (assetId: string) => Edl;
inference?: (emulator: PrivacyHarness["inference"]) => MediaInferenceProvider | null;
} = {},
) {
harness = await createPrivacyHarness(options.inference);
const asset = await harness.addAsset(options.fileName ?? "clip__faces-1____person-alice__.mp4");
harness.addPerson("alice", mode);
await harness.analyseFaces(asset.id, ["alice"]);
await harness.learnPerson("alice", asset.id);
const edl =
options.edl?.(asset.id) ?? testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
const workDir = await harness.workDir(`render-${mode}`);
return { h: harness, asset, edl, workDir };
}
async function gateOn(
h: PrivacyHarness,
input: { edl: Edl; workDir: string; prepared: string; outputName?: string },
profile = testProfile(),
) {
// The compiler is not under test: the prepared source stands in for the render output.
const outputPath = path.join(input.workDir, input.outputName ?? "output.mp4");
await copyFile(input.prepared, outputPath);
return runGate(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl: input.edl,
profile,
outputPath,
ffprobe: cleanProbe(input.edl),
copy: { title: null, caption: null, tags: [] },
bundleHasDisclosure: false,
autoFixRound: 0,
});
}
describe.skipIf(!hasFfmpeg)("prepareSources", () => {
it.each([
"blur",
"pixelate",
"solid",
"sticker",
"partial_eyes",
"partial_upper",
"partial_lower",
] as const)("applies %s to a protected person and the gate verifies it", async (mode) => {
const { h, asset, edl, workDir } = await setup(mode);
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const source = prepared.sources.get(asset.id);
expect(source?.path).toContain(`${path.sep}privacy${path.sep}`);
expect(source).toMatchObject({ width: 320, height: 180, rotation: 0, still: false });
const record = await readPrivacyRecord(workDir);
expect(record?.protectedPeople).toEqual({ alice: mode });
const check = record?.assets[asset.id]?.checks[0];
expect(check).toMatchObject({ kind: "face", personId: "alice" });
// The protected box changed, the rest of the frame did not.
const base = await extractLuma(path.join(h.root, asset.storagePath!), {
tMs: 1000,
width: 320,
height: 180,
});
const treated = await extractLuma(source!.path, { tMs: 1000, width: 320, height: 180 });
const box = check!.keyframes.find((k) => k.tMs >= 1000)!.box;
expect(residualVisibility({ source: base, treated, box })).toBeLessThan(0.05);
expect(
residualVisibility({ source: base, treated, box: { x: 0.6, y: 0.6, w: 0.35, h: 0.35 } }),
).toBeGreaterThan(0.95);
const report = await gateOn(h, { edl, workDir, prepared: source!.path });
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
expect(report.passed).toBe(true);
});
it("crops a reframe_out face out of frame", async () => {
const { h, asset, workDir } = await setup("reframe_out");
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
width: 180,
height: 320,
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const source = prepared.sources.get(asset.id)!;
expect(prepared.fallbacks).toEqual([]);
expect(source.width / source.height).toBeCloseTo(180 / 320, 1);
const record = await readPrivacyRecord(workDir);
expect(record?.assets[asset.id]?.crop).not.toBeNull();
const report = await gateOn(h, { edl, workDir, prepared: source.path });
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
});
it("falls back from reframe_out to blur and says so when no crop avoids the face", async () => {
const { h, asset, edl, workDir } = await setup("none", {
edl: (assetId) =>
testEdl([{ id: "c1", assetId, inMs: 0, outMs: 2000 }], {
redactions: {
faces: [
{
id: "big",
target: { personId: "alice" },
mode: "reframe_out",
paddingPct: 100,
featherPct: 0,
minSizePct: 50,
},
],
marks: [],
regions: [],
},
}),
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.fallbacks).toEqual([
"person alice: no crop keeps the face out of frame; blurred instead",
]);
const source = prepared.sources.get(asset.id)!;
expect(source.width).toBe(320);
const report = await gateOn(h, { edl, workDir, prepared: source.path });
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
});
it("blocks the export when tracking loses the face for more than 500 ms", async () => {
const { h, asset, edl, workDir } = await setup("blur", {
fileName: "clip__faces-1____person-alice____noface-gap-600-1400__.mp4",
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const record = await readPrivacyRecord(workDir);
expect(record?.trackingGaps).toEqual([
expect.objectContaining({ assetId: asset.id, personId: "alice" }),
]);
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "tracking_gap", severity: "block", personId: "alice" }),
);
});
it("falls back to blur when synthetic identity is unavailable and never fakes it", async () => {
const { h, asset, edl, workDir } = await setup("synthetic_identity");
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.fallbacks).toEqual([
"person alice: synthetic identity is not available; blurred instead",
]);
expect(prepared.aiAlterations).not.toContain("synthetic_identity");
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.passed).toBe(true);
});
it("uses a licensed synthetic identity with a stable seed and no target image", async () => {
const synthesize = vi.fn();
const { h, asset, edl, workDir } = await setup("synthetic_identity", {
inference: (emulator) => {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.capabilities = async () => {
const caps = await emulator.capabilities();
caps.operations.synthesizeIdentity = {
available: true,
experimental: true,
model: "test-synth",
license: "Apache-2.0",
commercialUse: true,
};
return caps;
};
provider.synthesizeIdentity = async (request) => {
synthesize(request);
// Stand-in synthesis: a solid fill over the track (real models paint a face).
return emulator.redact({
path: request.path,
outPath: request.outPath,
redactions: [
{ keyframes: request.track, mode: "solid", paddingPct: 20, featherPct: 0 },
],
});
};
return provider;
},
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.aiAlterations).toContain("synthetic_identity");
expect(synthesize).toHaveBeenCalledTimes(1);
const request = synthesize.mock.calls[0]![0] as Record<string, unknown>;
expect(Object.keys(request).sort()).toEqual(["outPath", "path", "seed", "signal", "track"]);
const seed = h.store.people[0]!.syntheticSeed;
expect(request.seed).toBe(seed);
expect(seed).not.toBeNull();
const record = await readPrivacyRecord(workDir);
expect(record?.expectedFaces).toEqual([
expect.objectContaining({ personId: "alice", mode: "synthetic_identity" }),
]);
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.passed).toBe(true);
});
it("fails closed without a media worker", async () => {
const { h, edl, workDir } = await setup("blur", { inference: () => null });
await expect(
prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
}),
).rejects.toBeInstanceOf(PrivacyUnavailableError);
});
it("returns the untouched source when nothing needs protection", async () => {
const { h, asset, edl, workDir } = await setup("none");
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.sources.get(asset.id)?.path).toBe(path.join(h.root, asset.storagePath!));
expect(prepared.aiAlterations).toEqual([]);
});
it("blurs unassigned faces next to a protected person and records voice disguise", async () => {
const { h, asset, workDir } = await setup("blur", {
fileName: "clip__faces-2____person-alice__.mp4",
});
// Track 2 stays unassigned (analysis stored it without a person).
h.store.tracks[1]!.personId = null;
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
redactions: { faces: [], marks: [], regions: [], voice: { method: "pitch", semitones: 3 } },
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.aiAlterations).toEqual(["voice_disguise"]);
const record = await readPrivacyRecord(workDir);
expect(record?.protectUnassigned).toBe(true);
expect(record?.assets[asset.id]?.checks.map((c) => c.personId)).toEqual(["alice", null]);
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
});
it("protects a photo and renders it from a redacted still", async () => {
const { h, asset, edl, workDir } = await setup("pixelate", {
fileName: "photo__faces-1____person-alice__.png",
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const source = prepared.sources.get(asset.id)!;
expect(source).toMatchObject({ still: true, durationMs: 0, width: 320, height: 180 });
expect(source.path.endsWith(".png")).toBe(true);
const record = await readPrivacyRecord(workDir);
const box = record!.assets[asset.id]!.checks[0]!.keyframes[0]!.box;
const base = await extractLuma(path.join(h.root, asset.storagePath!), {
tMs: 0,
width: 320,
height: 180,
still: true,
});
const treated = await extractLuma(source.path, {
tMs: 0,
width: 320,
height: 180,
still: true,
});
expect(residualVisibility({ source: base, treated, box })).toBeLessThan(0.05);
});
it("applies EDL regions and covers one that no crop can avoid", async () => {
const { h, asset, workDir } = await setup("none");
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
redactions: {
faces: [],
marks: [],
regions: [
{
id: "plate",
clipId: "c1",
keyframes: [{ tMs: 0, box: { x: 0.7, y: 0.7, w: 0.2, h: 0.2 } }],
treatment: "blur",
},
{
id: "everything",
clipId: "c1",
keyframes: [{ tMs: 0, box: { x: 0, y: 0, w: 1, h: 1 } }],
treatment: "reframe_out",
},
],
},
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.fallbacks).toEqual([
"region everything: no crop keeps it out of frame; covered instead",
]);
const record = await readPrivacyRecord(workDir);
expect(record?.assets[asset.id]?.checks.map((c) => c.id)).toEqual(["plate", "everything"]);
});
it("blocks when a provider claims to redact but leaves the face visible", async () => {
const { h, asset, edl, workDir } = await setup("blur", {
inference: (emulator) => {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.redact = async (request) => {
await copyFile(request.path, request.outPath);
return { outPath: request.outPath, framesProcessed: 50 };
};
return provider;
},
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "face_not_covered", personId: "alice" }),
);
});
});

928
packages/adapters/src/media-anonymize.ts Normal file
View File

@ -0,0 +1,928 @@
import { createHash } from "node:crypto";
import { mkdir, rm } from "node:fs/promises";
import path from "node:path";
import type { RedactionInstruction } from "@rakazo/adapter-kit";
import type {
AiAlteration,
AnonymizeMode,
BoxKeyframe,
Edl,
EdlVideoClip,
MarkTreatment,
MediaCapabilities,
MediaRenderQuality,
NormalizedBox,
NormalizedPoint,
} from "@rakazo/contracts";
import type { MediaRepos } from "@rakazo/db";
import type { CompileSource } from "@rakazo/media";
import { MAX_TRACKING_GAP_MS, reframeExcluding } from "@rakazo/media";
import type { MediaActor, MediaPreparedSources, MediaServiceDeps } from "./media-context.js";
import {
anchorFromFace,
faceAt,
firstConfident,
placeAnchor,
promptBox,
segmentMark,
} from "./media-marks.js";
import type { PrivacyToolOptions, ProbedSource } from "./media-privacy-io.js";
import {
analysisFile,
copyInto,
cropFile,
firstFrame,
inferenceCapabilities,
isInsideRoot,
maskKeyframes,
operationAvailable,
PrivacyUnavailableError,
probeSource,
readTrackKeyframes,
resolveAssetFile,
resolveStored,
} from "./media-privacy-io.js";
import type {
CandidateTrack,
PixelBox,
PrivacyCheck,
PrivacyRecord,
Protection,
TimeWindow,
} from "./media-privacy-plan.js";
import {
boxAt,
clipWindows,
cropPixels,
effectiveModes,
FACE_TRACK_FPS,
iou,
isPartial,
MARK_PADDING_PCT,
MARK_QUALITY_MIN,
MARK_REVIEW_MIN_SCORE,
matchPerson,
outsideCrop,
overlapsWindows,
PRIVACY_RECORD_FILE,
padBox,
partialCheckBand,
protectKeyframes,
resolveProtection,
TRACK_MATCH_MIN_IOU,
trackOverlap,
unionBox,
writePrivacyRecord,
} from "./media-privacy-plan.js";
/*
* prepareSources (spec 7): resolve every clip's file and apply all face, mark,
* and region protection before the compiler sees it. The compiler applies no
* redactions, so anything this function cannot honor throws
* (PrivacyUnavailableError) instead of returning an unredacted source.
*
* Per asset, in this order: synthetic identities, inpainted marks, one
* redaction pass (blur, pixelate, solid, sticker, partial bands, covers) by the
* inference provider, then a static crop for reframe_out. Faces are found from
* the stored tracks plus a fresh tracking pass; tracking gaps over 500 ms are
* covered by the grown last box and recorded so the gate blocks them. Voice
* disguise is applied by the compiler on the program audio and only recorded
* here. Everything the gate must verify goes into privacy.json in workDir.
*/
export type AnonymizeDeps = Pick<
MediaServiceDeps,
"repos" | "inference" | "mediaRoot" | "sources"
> &
PrivacyToolOptions;
type AssetRow = Awaited<ReturnType<MediaRepos["getAssetRecord"]>>;
type PersonRow = Awaited<ReturnType<MediaRepos["listPersonRecords"]>>[number];
/** reframe_out boxes are unioned per bucket of this length so the crop search stays small. */
const REFRAME_BUCKET_MS = 1000;
/** Landmark samples per partial-mode track. */
const MAX_LANDMARK_SAMPLES = 120;
interface AssetPlan {
asset: AssetRow;
clips: EdlVideoClip[];
windows: TimeWindow[];
base: string;
probe: ProbedSource;
dir: string;
candidates: CandidateTrack[];
instructions: RedactionInstruction[];
checks: PrivacyCheck[];
synthetic: Array<{ personId: string | null; keyframes: BoxKeyframe[]; seed: number }>;
inpaint: Array<{ markId: string; maskDir: string; keyframes: BoxKeyframe[] }>;
reframe: Array<{
check: PrivacyCheck;
/** Boxes the crop must exclude. */
boxes: Array<{ tMs: number; box: NormalizedBox }>;
/** Applied when no crop can exclude the boxes. */
fallback: () => void;
}>;
}
export interface PrepareInput {
actor: MediaActor;
projectId: string;
edl: Edl;
quality: MediaRenderQuality;
workDir: string;
signal?: AbortSignal;
}
export async function prepareSources(
deps: AnonymizeDeps,
input: PrepareInput,
): Promise<MediaPreparedSources> {
const { actor, edl, signal } = input;
const spaceId = actor.spaceId;
const workDir = await resolveStored(deps.mediaRoot, input.workDir);
await mkdir(workDir, { recursive: true });
const privacyDir = path.join(workDir, "privacy");
await rm(privacyDir, { recursive: true, force: true });
// A record from an earlier attempt must never vouch for this one.
await rm(path.join(workDir, PRIVACY_RECORD_FILE), { force: true });
const clipsByAsset = new Map<string, EdlVideoClip[]>();
for (const clip of edl.tracks.video) {
clipsByAsset.set(clip.assetId, [...(clipsByAsset.get(clip.assetId) ?? []), clip]);
}
const assetIds = [...clipsByAsset.keys()].sort();
const assets = await deps.repos.listAssetRecords(spaceId, assetIds);
if (assets.length !== assetIds.length)
throw new PrivacyUnavailableError("a clip asset is missing");
const project = await deps.repos.getProjectRecord(spaceId, input.projectId);
const kit = project.brandKitId
? (await deps.repos.getBrandKit(spaceId, project.brandKitId)).kit
: null;
const people = await deps.repos.listPersonRecords(spaceId);
const peopleById = new Map(people.map((person) => [person.id, person]));
const defaults = new Map(
people.map((person) => [person.id, person.anonymizeByDefault as AnonymizeMode]),
);
const modes = effectiveModes(edl.redactions.faces, defaults);
const faceWork =
edl.redactions.faces.length > 0 || [...modes.values()].some((mode) => mode !== "none");
const capabilities = await inferenceCapabilities(deps.inference, signal);
const stored = await deps.repos.faceTrackRecords(spaceId, assetIds);
const centroids = faceWork ? await deps.repos.personCentroids(spaceId) : [];
const fallbacks: string[] = [];
const alterations = new Set<AiAlteration>();
if (edl.redactions.voice) alterations.add("voice_disguise");
const record: PrivacyRecord = {
version: 1,
assets: {},
trackingGaps: [],
reviews: [],
expectedFaces: [],
unassignedFaces: [],
aiAlterations: [],
protectedPeople: {},
visiblePeople: [],
protectUnassigned: false,
};
// 1. Files and face candidates per asset.
const plans: AssetPlan[] = [];
for (const asset of assets) {
signal?.throwIfAborted();
const clips = clipsByAsset.get(asset.id) ?? [];
const base = await resolveAssetFile(deps, spaceId, asset, input.quality);
const probe = await probeSource(base, asset.kind, { ...deps, signal });
const windows = probe.still ? [{ startMs: 0, endMs: 0 }] : clipWindows(clips);
const plan: AssetPlan = {
asset,
clips,
windows,
base,
probe,
dir: path.join(privacyDir, asset.id),
candidates: [],
instructions: [],
checks: [],
synthetic: [],
inpaint: [],
reframe: [],
};
const storedTracks = stored.filter((track) => track.assetId === asset.id);
const fromStore: CandidateTrack[] = [];
for (const track of storedTracks) {
fromStore.push({
id: track.id,
storedId: track.id,
personId: track.personId,
keyframes: await readTrackKeyframes(deps.mediaRoot, track.keyframesPath),
});
}
if (faceWork) {
if (deps.inference && operationAvailable(capabilities, "trackFaces")) {
const file = (await analysisFile(deps, spaceId, asset)) ?? (await workingCopy(deps, plan));
const fresh = await deps.inference.trackFaces({
path: file,
sampleFps: FACE_TRACK_FPS,
signal,
});
plan.candidates = mergeTracks(asset.id, fromStore, fresh.tracks, centroids);
} else {
const analysed = await deps.repos.latestAnalysis(spaceId, asset.id, "faces");
if (!analysed) {
throw new PrivacyUnavailableError(
`faces of asset ${asset.id} were never detected and the media worker is unavailable`,
);
}
plan.candidates = fromStore;
}
} else {
plan.candidates = fromStore;
}
plan.candidates = plan.candidates.filter((track) => visibleIn(track, plan));
plans.push(plan);
}
// 2. Who is visible and who is protected.
const visible = new Set<string>();
for (const plan of plans) {
for (const track of plan.candidates) {
if (track.personId) visible.add(track.personId);
else {
const span = spanOf(track);
if (span) record.unassignedFaces.push({ assetId: plan.asset.id, ...span });
}
}
}
record.visiblePeople = [...visible].sort();
for (const [personId, mode] of modes) {
if (mode !== "none" && visible.has(personId)) record.protectedPeople[personId] = mode;
}
record.protectUnassigned = Object.keys(record.protectedPeople).length > 0;
const colors = {
fill: kit?.colors.primary,
sticker: kit?.colors.accent ?? kit?.colors.primary,
};
// 3. Faces.
const syntheticReady = operationAvailable(capabilities, "synthesizeIdentity");
for (const plan of plans) {
for (const track of plan.candidates) {
const protection = resolveProtection(track, {
faces: edl.redactions.faces,
defaults,
protectUnassigned: record.protectUnassigned,
});
if (!protection) continue;
const protectedTrack = protectKeyframes(track.keyframes, plan.windows, protection.minSizePct);
const keyframes = plan.probe.still
? stillKeyframes(protectedTrack.keyframes)
: protectedTrack.keyframes;
if (keyframes.length === 0) continue;
if (!plan.probe.still) {
for (const gap of protectedTrack.gaps) {
record.trackingGaps.push({ assetId: plan.asset.id, personId: track.personId, ...gap });
}
}
await planFace(deps, plan, {
track,
protection,
keyframes,
person: track.personId ? (peopleById.get(track.personId) ?? null) : null,
syntheticReady,
capabilities,
colors,
fallbacks,
record,
signal,
});
}
}
// 4. Marks (asset marks and the standing marks of people seen in the asset).
const edlMarks = new Map(edl.redactions.marks.map((mark) => [mark.markId, mark.treatment]));
for (const plan of plans) {
const marks = await deps.repos.marksForAsset(spaceId, plan.asset.id);
for (const mark of marks) {
const treatment: MarkTreatment = edlMarks.get(mark.id) ?? mark.treatment;
if (treatment === "none") continue;
const located = await locateMark(deps, plan, mark, signal);
if (!located) continue;
if (located.review) {
record.reviews.push({ markId: mark.id, assetId: plan.asset.id, score: located.score });
}
const keyframes = plan.probe.still
? stillKeyframes(located.keyframes)
: located.keyframes.filter((k) => nearWindows(k.tMs, plan.windows));
if (keyframes.length === 0) continue;
const check: PrivacyCheck = {
kind: "mark",
id: mark.id,
personId: mark.personId,
markId: mark.id,
keyframes,
maskDir: located.maskDir,
};
const cover = () => {
plan.instructions.push({
keyframes,
mode: "cover",
paddingPct: MARK_PADDING_PCT,
featherPct: 0,
maskDir: located.maskDir,
...(colors.fill ? { color: colors.fill } : {}),
});
plan.checks.push(check);
};
if (treatment === "blur") {
plan.instructions.push({
keyframes,
mode: "blur",
paddingPct: MARK_PADDING_PCT,
featherPct: 0,
maskDir: located.maskDir,
});
plan.checks.push(check);
} else if (treatment === "cover") {
cover();
} else if (treatment === "reframe_out") {
plan.reframe.push({
check,
boxes: keyframes.map((k) => ({ tMs: k.tMs, box: padBox(k.box, MARK_PADDING_PCT) })),
fallback: () => {
fallbacks.push(`mark ${mark.id}: no crop keeps it out of frame; covered instead`);
cover();
},
});
} else if (operationAvailable(capabilities, "inpaintVideo")) {
plan.inpaint.push({ markId: mark.id, maskDir: located.maskDir, keyframes });
plan.checks.push(check);
} else {
fallbacks.push(`mark ${mark.id}: inpainting is unavailable; covered instead`);
cover();
}
}
}
// 5. Regions drawn in the EDL.
for (const region of edl.redactions.regions) {
const clip = edl.tracks.video.find((candidate) => candidate.id === region.clipId);
const plan = plans.find((candidate) => candidate.asset.id === clip?.assetId);
if (!clip || !plan) throw new PrivacyUnavailableError(`region ${region.id} has no clip`);
const keyframes = plan.probe.still ? stillKeyframes(region.keyframes) : region.keyframes;
const check: PrivacyCheck = {
kind: "region",
id: region.id,
personId: null,
markId: null,
keyframes,
maskDir: null,
};
const apply = (mode: "blur" | "cover") => {
plan.instructions.push({
keyframes,
mode,
paddingPct: 0,
featherPct: 0,
...(mode === "cover" && colors.fill ? { color: colors.fill } : {}),
});
plan.checks.push(check);
};
if (region.treatment === "reframe_out") {
plan.reframe.push({
check,
boxes: keyframes,
fallback: () => {
fallbacks.push(`region ${region.id}: no crop keeps it out of frame; covered instead`);
apply("cover");
},
});
} else {
apply(region.treatment);
}
}
// 6. Apply per asset.
const sources = new Map<string, CompileSource>();
for (const plan of plans) {
signal?.throwIfAborted();
const crop = chooseCrop(plan, edl);
const hasWork =
plan.synthetic.length > 0 ||
plan.inpaint.length > 0 ||
plan.instructions.length > 0 ||
crop !== null;
if (!hasWork) {
sources.set(plan.asset.id, compileSource(plan.base, plan.probe, plan.probe.rotation));
continue;
}
const prepared = await applyPlan(deps, plan, crop, {
capabilities,
alterations,
fallbacks,
signal,
});
const probed = await probeSource(prepared, plan.asset.kind, { ...deps, signal });
verifyPrepared(plan, crop, probed);
sources.set(plan.asset.id, compileSource(prepared, probed, 0));
record.assets[plan.asset.id] = {
basePath: plan.base,
preparedPath: prepared,
width: plan.probe.width,
height: plan.probe.height,
still: plan.probe.still,
crop,
checks: plan.checks,
};
}
record.aiAlterations = [...alterations].sort();
await writePrivacyRecord(workDir, record);
return { sources, aiAlterations: record.aiAlterations, fallbacks };
}
// ---------------------------------------------------------------------------
// Faces
// ---------------------------------------------------------------------------
/**
* Fresh tracks take over stored ones they overlap (keeping the stored id and
* person); unmatched fresh tracks are matched to people by embedding; stored
* tracks the fresh pass missed stay (fail closed).
*/
export function mergeTracks(
assetId: string,
stored: CandidateTrack[],
fresh: Array<{
keyframes: Array<BoxKeyframe & { confidence: number }>;
embedding: number[] | null;
}>,
centroids: Array<{ personId: string; centroid: Uint8Array }>,
): CandidateTrack[] {
const taken = new Set<string>();
const merged: CandidateTrack[] = [];
fresh.forEach((track, index) => {
let best: { candidate: CandidateTrack; score: number } | null = null;
for (const candidate of stored) {
if (taken.has(candidate.id)) continue;
const score = trackOverlap(track.keyframes, candidate.keyframes);
if (score >= TRACK_MATCH_MIN_IOU && (!best || score > best.score))
best = { candidate, score };
}
if (best) {
taken.add(best.candidate.id);
merged.push({
...best.candidate,
personId: best.candidate.personId ?? matchPerson(track.embedding, centroids),
keyframes: [...track.keyframes, ...best.candidate.keyframes].sort((a, b) => a.tMs - b.tMs),
});
return;
}
merged.push({
id: `fresh-${assetId}-${index + 1}`,
storedId: null,
personId: matchPerson(track.embedding, centroids),
keyframes: track.keyframes,
});
});
for (const candidate of stored) if (!taken.has(candidate.id)) merged.push(candidate);
return merged;
}
function nearWindows(tMs: number, windows: TimeWindow[]): boolean {
return windows.some(
(w) => tMs >= w.startMs - MAX_TRACKING_GAP_MS && tMs <= w.endMs + MAX_TRACKING_GAP_MS,
);
}
function spanOf(track: CandidateTrack): TimeWindow | null {
if (track.keyframes.length === 0) return null;
const times = track.keyframes.map((k) => k.tMs);
return { startMs: Math.min(...times), endMs: Math.max(...times) };
}
function visibleIn(track: CandidateTrack, plan: AssetPlan): boolean {
if (plan.probe.still) return track.keyframes.length > 0;
const span = spanOf(track);
return span !== null && overlapsWindows(span, plan.windows);
}
/** A still shows one moment: every box at once, held over the whole image. */
function stillKeyframes(keyframes: BoxKeyframe[]): BoxKeyframe[] {
const first = keyframes[0];
if (!first) return [];
const box = keyframes.reduce((acc, k) => unionBox(acc, k.box), first.box);
return [
{ tMs: 0, box },
{ tMs: 1000, box },
];
}
/** Stable per-person seed for a synthetic identity (stored on first use). */
export function syntheticSeedFor(id: string): number {
return createHash("sha256").update(`synthetic:${id}`).digest().readInt32LE(0);
}
async function planFace(
deps: AnonymizeDeps,
plan: AssetPlan,
input: {
track: CandidateTrack;
protection: Protection;
keyframes: BoxKeyframe[];
person: PersonRow | null;
syntheticReady: boolean;
capabilities: MediaCapabilities | null;
colors: { fill?: string; sticker?: string };
fallbacks: string[];
record: PrivacyRecord;
signal?: AbortSignal;
},
): Promise<void> {
const { track, protection, keyframes } = input;
const who = track.personId ? `person ${track.personId}` : `face ${track.id}`;
let mode = protection.mode;
const padded = keyframes.map((k) => ({ tMs: k.tMs, box: padBox(k.box, protection.paddingPct) }));
const check: PrivacyCheck = {
kind: "face",
id: track.id,
personId: track.personId,
markId: null,
keyframes: padded,
maskDir: null,
};
const span = spanOf({ ...track, keyframes });
if (mode === "synthetic_identity") {
if (input.syntheticReady) {
let seed: number;
if (input.person) {
seed = input.person.syntheticSeed ?? syntheticSeedFor(input.person.id);
if (input.person.syntheticSeed === null) {
await deps.repos.updatePerson(plan.asset.spaceId, input.person.id, {
syntheticSeed: seed,
});
input.person.syntheticSeed = seed;
}
} else {
seed = syntheticSeedFor(track.id);
}
plan.synthetic.push({ personId: track.personId, keyframes, seed });
plan.checks.push(check);
if (span) {
input.record.expectedFaces.push({
assetId: plan.asset.id,
personId: track.personId,
mode,
...span,
});
}
return;
}
input.fallbacks.push(`${who}: synthetic identity is not available; blurred instead`);
mode = "blur";
}
if (mode === "reframe_out") {
plan.reframe.push({
check,
boxes: padded,
fallback: () => {
input.fallbacks.push(`${who}: no crop keeps the face out of frame; blurred instead`);
plan.instructions.push({
keyframes,
mode: "blur",
paddingPct: protection.paddingPct,
featherPct: protection.featherPct,
});
plan.checks.push(check);
},
});
return;
}
const instruction: RedactionInstruction = {
keyframes,
mode,
paddingPct: protection.paddingPct,
featherPct: protection.featherPct,
};
if (mode === "solid" && input.colors.fill) instruction.color = input.colors.fill;
if (mode === "sticker" && input.colors.sticker) instruction.color = input.colors.sticker;
if (isPartial(mode)) {
const partial = mode;
const landmarks = await landmarksFor(deps, plan, keyframes, input.capabilities, input.signal);
if (landmarks.length > 0) instruction.landmarks = landmarks;
check.keyframes = padded.map((k) => ({ tMs: k.tMs, box: partialCheckBand(k.box, partial) }));
if (span) {
input.record.expectedFaces.push({
assetId: plan.asset.id,
personId: track.personId,
mode,
...span,
});
}
}
plan.instructions.push(instruction);
plan.checks.push(check);
}
/** Five-point landmarks per sampled keyframe for partial modes, when the provider has them. */
async function landmarksFor(
deps: AnonymizeDeps,
plan: AssetPlan,
keyframes: BoxKeyframe[],
capabilities: MediaCapabilities | null,
signal?: AbortSignal,
): Promise<Array<{ tMs: number; points: NormalizedPoint[] }>> {
if (!deps.inference || !operationAvailable(capabilities, "faceLandmarks")) return [];
const step = Math.max(1, Math.ceil(keyframes.length / MAX_LANDMARK_SAMPLES));
const timesMs = keyframes.filter((_, i) => i % step === 0).map((k) => k.tMs);
const file =
(await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan));
const result = await deps.inference.faceLandmarks({ path: file, timesMs, signal });
const out: Array<{ tMs: number; points: NormalizedPoint[] }> = [];
for (const frame of result.frames) {
const expected = boxAt(keyframes, frame.tMs);
if (!expected) continue;
const best = frame.faces
.map((face) => ({ face, score: iou(face.box, expected) }))
.sort((a, b) => b.score - a.score)[0];
if (best && best.score > 0.2) out.push({ tMs: frame.tMs, points: best.face.landmarks });
}
return out;
}
// ---------------------------------------------------------------------------
// Marks
// ---------------------------------------------------------------------------
type MarkRow = Awaited<ReturnType<MediaRepos["marksForAsset"]>>[number];
/**
* Mask and keyframes of a mark in this asset. Marks drawn on the asset use
* their stored mask (segmenting now when none exists); standing person marks
* are re-prompted at the same place relative to the person's face. Low
* segmentation confidence sends the mark to review.
*/
async function locateMark(
deps: AnonymizeDeps,
plan: AssetPlan,
mark: MarkRow,
signal?: AbortSignal,
): Promise<{ maskDir: string; keyframes: BoxKeyframe[]; score: number; review: boolean } | null> {
const outDir = path.join(plan.dir, `mark-${mark.id}`);
if (mark.assetId === plan.asset.id) {
if (mark.maskDir) {
const maskDir = await resolveStored(deps.mediaRoot, mark.maskDir);
return { maskDir, keyframes: await maskKeyframes(maskDir), score: 1, review: false };
}
if (!mark.prompt) throw new PrivacyUnavailableError(`mark ${mark.id} has no prompt`);
const file =
(await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan));
const segmented = await segmentMark(deps, { file, prompt: mark.prompt, outDir, signal });
return {
maskDir: segmented.maskDir,
keyframes: segmented.keyframes,
score: segmented.score,
review: false,
};
}
// A standing mark of a person seen in this asset.
if (!mark.personId) return null;
const tracks = plan.candidates.filter((track) => track.personId === mark.personId);
if (tracks.length === 0) return null;
const anchorFace = await originFace(deps, plan.asset.spaceId, mark);
const target = tracks
.map((track) =>
firstConfident(track.keyframes, Math.min(...plan.windows.map((w) => w.startMs))),
)
.filter((k) => k !== null)
.sort((a, b) => a.tMs - b.tMs)[0];
const placed =
anchorFace && target && mark.prompt
? placeAnchor(anchorFromFace(promptBox(mark.prompt), anchorFace), target.box)
: null;
if (!placed || !target) {
// Cannot relate the mark to the face here: cover the person's face region and review.
const keyframes = tracks.flatMap((track) =>
track.keyframes.map((k) => ({ tMs: k.tMs, box: k.box })),
);
return { maskDir: await emptyMaskDir(outDir), keyframes, score: 0, review: true };
}
const file =
(await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan));
const segmented = await segmentMark(deps, {
file,
prompt: { tMs: target.tMs, box: placed },
outDir,
signal,
});
return {
maskDir: segmented.maskDir,
keyframes: segmented.keyframes,
score: segmented.score,
review: segmented.score < MARK_REVIEW_MIN_SCORE,
};
}
/** The person's face in the frame a standing mark was drawn on. */
async function originFace(
deps: AnonymizeDeps,
spaceId: string,
mark: MarkRow,
): Promise<NormalizedBox | null> {
if (!mark.assetId || !mark.prompt || !mark.personId) return null;
const tracks = (await deps.repos.faceTrackRecords(spaceId, [mark.assetId])).filter(
(t) => t.personId === mark.personId,
);
for (const track of tracks) {
const face = faceAt(
await readTrackKeyframes(deps.mediaRoot, track.keyframesPath),
mark.prompt.tMs,
);
if (face) return face;
}
return null;
}
async function emptyMaskDir(dir: string): Promise<string> {
await mkdir(dir, { recursive: true });
return dir;
}
// ---------------------------------------------------------------------------
// Applying
// ---------------------------------------------------------------------------
/** In-root copy of the base file for providers (once per asset). */
async function workingCopy(deps: AnonymizeDeps, plan: AssetPlan): Promise<string> {
if (!(await isInsideRoot(deps.mediaRoot, plan.base)))
plan.base = await copyInto(plan.base, plan.dir);
return plan.base;
}
/** Static crop that keeps every reframe_out box out of frame; applies fallbacks otherwise. */
function chooseCrop(plan: AssetPlan, edl: Edl): PixelBox | null {
if (plan.reframe.length === 0) return null;
const boxes = bucketBoxes(plan.reframe.flatMap((item) => item.boxes));
const geometry = {
sourceWidth: plan.probe.width,
sourceHeight: plan.probe.height,
outWidth: edl.output.width,
outHeight: edl.output.height,
durationMs: plan.probe.durationMs,
};
const keyframe = reframeExcluding(boxes, geometry)?.[0];
const crop = keyframe ? cropPixels(geometry, keyframe) : null;
if (
crop &&
boxes.every(({ box }) => outsideCrop(box, crop, plan.probe.width, plan.probe.height))
) {
for (const item of plan.reframe) plan.checks.push(item.check);
return crop;
}
for (const item of plan.reframe) item.fallback();
return null;
}
/** Union boxes per second so the crop search stays small; unions only widen the excluded area. */
function bucketBoxes(boxes: Array<{ tMs: number; box: NormalizedBox }>) {
const buckets = new Map<number, NormalizedBox>();
for (const { tMs, box } of boxes) {
const key = Math.floor(tMs / REFRAME_BUCKET_MS);
const current = buckets.get(key);
buckets.set(key, current ? unionBox(current, box) : box);
}
return [...buckets.entries()].map(([key, box]) => ({ tMs: key * REFRAME_BUCKET_MS, box }));
}
function stepPath(plan: AssetPlan, file: string, step: string, ext = ".mp4"): string {
const stem = path
.basename(file, path.extname(file))
.replace(/\.(?:synth|inpaint|redacted|cropped)[-\w]*$/, "");
return path.join(plan.dir, `${stem}.${step}${ext}`);
}
async function applyPlan(
deps: AnonymizeDeps,
plan: AssetPlan,
crop: PixelBox | null,
context: {
capabilities: MediaCapabilities | null;
alterations: Set<AiAlteration>;
fallbacks: string[];
signal?: AbortSignal;
},
): Promise<string> {
const { signal } = context;
await mkdir(plan.dir, { recursive: true });
const needsProvider = plan.synthetic.length + plan.inpaint.length + plan.instructions.length > 0;
if (needsProvider && !deps.inference) {
throw new PrivacyUnavailableError("redaction needs the media worker, which is not configured");
}
let current = needsProvider ? await workingCopy(deps, plan) : plan.base;
const inference = deps.inference;
if (inference) {
for (const [index, synthetic] of plan.synthetic.entries()) {
const outPath = stepPath(plan, current, `synth-${index + 1}`);
current = (
await inference.synthesizeIdentity({
path: current,
track: synthetic.keyframes,
seed: synthetic.seed,
outPath,
signal,
})
).outPath;
context.alterations.add("synthetic_identity");
}
for (const mark of plan.inpaint) {
const outPath = stepPath(plan, current, `inpaint-${mark.markId}`);
const result = await inference.inpaintVideo({
path: current,
maskDir: mark.maskDir,
outPath,
signal,
});
if (result.quality.score >= MARK_QUALITY_MIN) {
current = result.outPath;
context.alterations.add("inpaint");
continue;
}
// Never ship artifacts silently: low quality falls back to a cover.
context.fallbacks.push(
`mark ${mark.markId}: inpainting scored ${result.quality.score.toFixed(2)}; covered instead`,
);
await rm(result.outPath, { force: true });
plan.instructions.push({
keyframes: mark.keyframes,
mode: "cover",
paddingPct: MARK_PADDING_PCT,
featherPct: 0,
maskDir: mark.maskDir,
});
}
if (plan.instructions.length > 0) {
if (!operationAvailable(context.capabilities, "redact")) {
throw new PrivacyUnavailableError("the media worker cannot redact");
}
const outPath = stepPath(plan, current, "redacted");
const result = await inference.redact({
path: current,
redactions: plan.instructions,
outPath,
signal,
});
if (result.framesProcessed <= 0)
throw new PrivacyUnavailableError("redaction processed no frames");
current = result.outPath;
}
}
if (crop) {
const outPath = stepPath(plan, current, "cropped", plan.probe.still ? ".png" : ".mp4");
await cropFile(current, outPath, crop, plan.probe.still, { ...deps, signal });
current = outPath;
}
if (plan.probe.still && path.extname(current).toLowerCase() !== ".png" && current !== plan.base) {
const outPath = stepPath(plan, current, "still", ".png");
await firstFrame(current, outPath, { ...deps, signal });
current = outPath;
}
return current;
}
/** The prepared file must show the whole clip range at the expected size. */
function verifyPrepared(plan: AssetPlan, crop: PixelBox | null, probed: ProbedSource): void {
const width = crop ? crop.w : plan.probe.width;
const height = crop ? crop.h : plan.probe.height;
if (probed.width !== width || probed.height !== height) {
throw new PrivacyUnavailableError(
`redacted asset ${plan.asset.id} is ${probed.width}x${probed.height}, expected ${width}x${height}`,
);
}
if (plan.probe.still) return;
const neededMs = Math.max(...plan.clips.map((clip) => clip.outMs));
const frameMs = 1000 / Math.max(1, plan.probe.fps);
if (probed.durationMs + 2 * frameMs < Math.min(neededMs, plan.probe.durationMs)) {
throw new PrivacyUnavailableError(`redacted asset ${plan.asset.id} is shorter than its clips`);
}
}
function compileSource(
file: string,
probe: ProbedSource,
rotation: CompileSource["rotation"],
): CompileSource {
const quarter = rotation === 90 || rotation === 270;
return {
path: file,
// CompileSource wants the coded size; probe reports the display size.
width: quarter ? probe.height : probe.width,
height: quarter ? probe.width : probe.height,
durationMs: probe.durationMs,
fps: probe.fps,
hasAudio: probe.hasAudio,
rotation,
still: probe.still,
};
}

97
packages/adapters/src/media-consent.test.ts Normal file
View File

@ -0,0 +1,97 @@
import { afterEach, describe, expect, it } from "vitest";
import {
assertExportConsent,
ConsentRequiredError,
consentIssues,
consentRequirements,
enforceConsentFlags,
participantsOf,
} from "./media-consent.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import { ACTOR, createPrivacyHarness, testEdl, testProfile } from "./media-privacy-fixtures.js";
const now = new Date("2026-06-01T12:00:00Z");
const past = new Date("2026-01-01T00:00:00Z");
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
describe("consent rules", () => {
it("always requires consent and age proof for paid video", () => {
expect(consentRequirements(testProfile())).toEqual({ consent: false, age: false });
expect(consentRequirements(testProfile({ kind: "paid_video" }))).toEqual({
consent: true,
age: true,
});
expect(enforceConsentFlags(testProfile({ kind: "paid_video" })).consent).toEqual({
requirePerformerConsent: true,
requireAgeVerification: true,
});
});
it("accepts only recorded, past proof with a verification method", () => {
const paid = testProfile({ kind: "paid_video" });
const ok = {
id: "a",
consentRecordedAt: past,
ageVerifiedAt: past,
ageVerificationMethod: "platform_verified",
};
expect(consentIssues(paid, [ok], [], now)).toEqual([]);
const future = { ...ok, consentRecordedAt: new Date("2027-01-01T00:00:00Z") };
const noMethod = { ...ok, id: "b", ageVerificationMethod: "none" };
expect(consentIssues(paid, [future, noMethod], [{ atMs: 10 }], now).map((i) => i.code)).toEqual(
["consent_missing", "age_unverified", "unknown_performer"],
);
expect(consentIssues(testProfile(), [noMethod], [{}], now)).toEqual([]);
});
it("collects the people and unassigned faces inside the clip ranges", () => {
const edl = testEdl([{ id: "c1", assetId: "a1", inMs: 1000, outMs: 2000 }]);
expect(
participantsOf(edl, [
{ assetId: "a1", personId: "alice", startMs: 0, endMs: 1200 },
{ assetId: "a1", personId: "bob", startMs: 2500, endMs: 3000 },
{ assetId: "a1", personId: null, startMs: 1500, endMs: 1600 },
{ assetId: "a2", personId: "carol", startMs: 0, endMs: 5000 },
]),
).toEqual({
personIds: ["alice"],
unassigned: [{ assetId: "a1", startMs: 1500, endMs: 1600 }],
});
});
});
describe("export consent", () => {
it("blocks a paid export until every participant has consent and age proof", async () => {
harness = await createPrivacyHarness(() => null);
const alice = harness.addPerson("alice", "none");
harness.store.tracks.push({
id: "t1",
assetId: "a1",
personId: "alice",
startMs: 0,
endMs: 1000,
confidenceMin: 0.9,
keyframesPath: "x.json",
modelVersion: "v1",
});
const edl = testEdl([{ id: "c1", assetId: "a1", inMs: 0, outMs: 1000 }]);
const paid = testProfile({ kind: "paid_video" });
await expect(assertExportConsent(harness.deps, ACTOR, { edl, profile: paid })).rejects.toThrow(
ConsentRequiredError,
);
await expect(
assertExportConsent(harness.deps, ACTOR, { edl, profile: testProfile() }),
).resolves.toBeUndefined();
alice.consentRecordedAt = past;
alice.ageVerifiedAt = past;
alice.ageVerificationMethod = "document_on_file";
await expect(
assertExportConsent(harness.deps, ACTOR, { edl, profile: paid }),
).resolves.toBeUndefined();
});
});

161
packages/adapters/src/media-consent.ts Normal file
View File

@ -0,0 +1,161 @@
import type { Edl, GateIssue, PlatformProfile } from "@rakazo/contracts";
import type { MediaActor, MediaServiceDeps } from "./media-context.js";
import { clipWindows, overlapsWindows } from "./media-privacy-plan.js";
/*
* Consent and age proof (spec 0 and 6.4). A paid-video profile always needs
* documented consent and age verification of every recognised participant,
* whatever its own flags say; other profiles follow their consent flags.
* Faces that belong to no person can never show consent and block.
*/
export interface ConsentRequirements {
consent: boolean;
age: boolean;
}
export function consentRequirements(profile: PlatformProfile): ConsentRequirements {
const paid = profile.kind === "paid_video";
return {
consent: paid || profile.consent.requirePerformerConsent,
age: paid || profile.consent.requireAgeVerification,
};
}
/** The profile with its consent flags raised to what the hard rules demand. */
export function enforceConsentFlags(profile: PlatformProfile): PlatformProfile {
const required = consentRequirements(profile);
return {
...profile,
consent: { requirePerformerConsent: required.consent, requireAgeVerification: required.age },
};
}
export interface PersonConsentState {
id: string;
consentRecordedAt: Date | string | null;
ageVerifiedAt: Date | string | null;
ageVerificationMethod: string;
}
/** Valid proof: a recorded date that is not in the future. */
function proven(value: Date | string | null, now: Date): boolean {
if (!value) return false;
const at = value instanceof Date ? value : new Date(value);
return !Number.isNaN(at.getTime()) && at.getTime() <= now.getTime();
}
export function hasConsent(person: PersonConsentState, now: Date): boolean {
return proven(person.consentRecordedAt, now);
}
export function hasAgeProof(person: PersonConsentState, now: Date): boolean {
return person.ageVerificationMethod !== "none" && proven(person.ageVerifiedAt, now);
}
/**
* Consent issues for the participants of an export. `unassigned` counts faces
* no person could be matched to; each blocks because nobody can consent for it.
*/
export function consentIssues(
profile: PlatformProfile,
people: PersonConsentState[],
unassigned: Array<{ atMs?: number; endMs?: number }>,
now: Date,
): GateIssue[] {
const required = consentRequirements(profile);
if (!required.consent && !required.age) return [];
const issues: GateIssue[] = [];
for (const person of [...people].sort((a, b) => a.id.localeCompare(b.id))) {
if (required.consent && !hasConsent(person, now)) {
issues.push({
check: "consent",
severity: "block",
code: "consent_missing",
message: "no recorded consent",
personId: person.id,
});
}
if (required.age && !hasAgeProof(person, now)) {
issues.push({
check: "consent",
severity: "block",
code: "age_unverified",
message: "age is not verified",
personId: person.id,
});
}
}
for (const face of unassigned) {
issues.push({
check: "consent",
severity: "block",
code: "unknown_performer",
message: "a visible face is not assigned to a person with consent",
...(face.atMs !== undefined ? { atMs: face.atMs } : {}),
...(face.endMs !== undefined ? { endMs: face.endMs } : {}),
});
}
return issues;
}
/**
* People and unassigned faces in the clip ranges of an EDL, from the stored
* face tracks of its assets.
*/
export function participantsOf(
edl: Edl,
tracks: Array<{ assetId: string; personId: string | null; startMs: number; endMs: number }>,
): {
personIds: string[];
unassigned: Array<{ assetId: string; startMs: number; endMs: number }>;
} {
const personIds = new Set<string>();
const unassigned: Array<{ assetId: string; startMs: number; endMs: number }> = [];
for (const track of tracks) {
const windows = clipWindows(edl.tracks.video.filter((clip) => clip.assetId === track.assetId));
if (!overlapsWindows(track, windows)) continue;
if (track.personId) personIds.add(track.personId);
else unassigned.push({ assetId: track.assetId, startMs: track.startMs, endMs: track.endMs });
}
return { personIds: [...personIds].sort(), unassigned };
}
export class ConsentRequiredError extends Error {
constructor(readonly issues: GateIssue[]) {
super(
`export blocked: ${issues.length} participant${issues.length === 1 ? "" : "s"} without documented consent or age proof`,
);
this.name = "ConsentRequiredError";
}
}
/**
* Consent issues for exporting a render to a profile, re-read at export time
* so a withdrawn consent stops a bundle that passed the gate earlier.
*/
export async function exportConsentIssues(
deps: Pick<MediaServiceDeps, "repos" | "now">,
actor: MediaActor,
input: { edl: Edl; profile: PlatformProfile },
): Promise<GateIssue[]> {
const required = consentRequirements(input.profile);
if (!required.consent && !required.age) return [];
const assetIds = [...new Set(input.edl.tracks.video.map((clip) => clip.assetId))];
const tracks = await deps.repos.faceTrackRecords(actor.spaceId, assetIds);
const { personIds, unassigned } = participantsOf(input.edl, tracks);
const people = personIds.length
? await deps.repos.listPersonRecords(actor.spaceId, personIds)
: [];
return consentIssues(input.profile, people, unassigned, deps.now());
}
/** Throws ConsentRequiredError when the export may not happen. */
export async function assertExportConsent(
deps: Pick<MediaServiceDeps, "repos" | "now">,
actor: MediaActor,
input: { edl: Edl; profile: PlatformProfile },
): Promise<void> {
const issues = await exportConsentIssues(deps, actor, input);
if (issues.length > 0) throw new ConsentRequiredError(issues);
}

188
packages/adapters/src/media-gate-autofix.test.ts Normal file
View File

@ -0,0 +1,188 @@
import type { Edl, GateReport } from "@rakazo/contracts";
import { MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
import { afterEach, describe, expect, it, vi } from "vitest";
import { autoFixRender, nonCompliantChange, proposeAutoFix } from "./media-gate-autofix.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import { ACTOR, createPrivacyHarness, testEdl } from "./media-privacy-fixtures.js";
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
function edlWithFace(): Edl {
return testEdl(
[
{ id: "c1", assetId: "a1", inMs: 0, outMs: 4000 },
{ id: "c2", assetId: "a2", inMs: 0, outMs: 4000 },
],
{
redactions: {
faces: [
{
id: "f1",
target: { personId: "alice" },
mode: "blur",
paddingPct: 20,
featherPct: 10,
minSizePct: 2,
},
],
marks: [{ markId: "m1", treatment: "inpaint" }],
regions: [],
},
},
);
}
function blocked(issues: GateReport["issues"], round = 0): GateReport {
return {
profileId: "test-profile",
passed: false,
checkedAt: "2026-06-01T12:00:00.000Z",
sampledFrames: 10,
issues,
autoFixRound: round,
};
}
const contentHit = {
check: "content" as const,
severity: "block" as const,
code: "content_threshold",
message: "explicit_nudity 0.90 above 0.5",
atMs: 7000,
endMs: 7500,
category: "explicit_nudity" as const,
score: 0.9,
};
describe("proposeAutoFix", () => {
it("only widens protection, covers marks, and cuts offending footage", () => {
const edl = edlWithFace();
const proposal = proposeAutoFix({
edl,
report: blocked([
contentHit,
{
check: "anonymization",
severity: "block",
code: "face_not_covered",
message: "face covered 80 %",
atMs: 100,
personId: "alice",
},
{
check: "anonymization",
severity: "block",
code: "mark_visible",
message: "visible",
atMs: 200,
markId: "m1",
},
]),
round: 0,
alternatives: [],
});
expect("edl" in proposal).toBe(true);
if (!("edl" in proposal)) return;
expect(proposal.edl.redactions.faces[0]!.paddingPct).toBe(45);
expect(proposal.edl.redactions.marks[0]!.treatment).toBe("cover");
expect(proposal.edl.tracks.video.find((c) => c.id === "c2")!.outMs).toBe(2500);
expect(proposal.changes.length).toBeGreaterThanOrEqual(3);
expect(nonCompliantChange(edl, proposal.edl)).toBeNull();
});
it("stops after the round limit", () => {
const proposal = proposeAutoFix({
edl: edlWithFace(),
report: blocked([contentHit], MAX_AUTO_FIX_ROUNDS),
round: MAX_AUTO_FIX_ROUNDS,
alternatives: [],
});
expect(proposal).toEqual({ reason: "round_limit", changes: [] });
});
it("rejects any change that weakens protection or adds content", () => {
const edl = edlWithFace();
const weaker = structuredClone(edl);
weaker.redactions.faces[0]!.paddingPct = 5;
expect(nonCompliantChange(edl, weaker)).toMatch(/shrank/);
const removed = structuredClone(edl);
removed.redactions.faces = [];
expect(nonCompliantChange(edl, removed)).toMatch(/removed/);
const unmasked = structuredClone(edl);
unmasked.redactions.marks[0]!.treatment = "blur";
expect(nonCompliantChange(edl, unmasked)).toMatch(/changed to blur/);
const grown = structuredClone(edl);
grown.tracks.video[0]!.outMs = 9000;
expect(nonCompliantChange(edl, grown)).toMatch(/grew/);
const overlay = structuredClone(edl);
overlay.overlays.push({ id: "o1", kind: "text", atMs: 0, durationMs: 100, text: "x" });
expect(nonCompliantChange(edl, overlay)).toMatch(/overlays/);
const swapped = structuredClone(edl);
swapped.redactions.faces[0]!.mode = "partial_eyes";
expect(nonCompliantChange(edl, swapped)).toMatch(/mode/);
});
});
describe("autoFixRender", () => {
async function blockedRender(round: number) {
harness = await createPrivacyHarness(() => null);
const edl = edlWithFace();
harness.store.plans.push({ id: "plan-1", projectId: "project-1", edl });
harness.store.renders.push({
id: "render-1",
planId: "plan-1",
projectId: "project-1",
quality: "draft",
status: "blocked",
autoFixRound: round,
gateReport: blocked([contentHit], round),
});
harness.store.renders.push({ id: "render-2", status: "queued", autoFixRound: 0 });
const submitPlan = vi.fn(async () => ({
ok: true as const,
plan: { id: "plan-2" } as never,
}));
const start = vi.fn(async () => ({ id: "render-2" }) as never);
return {
services: { deps: harness.deps, marketing: { submitPlan }, render: { start } },
submitPlan,
start,
};
}
it("submits a new plan with the reasons and renders it as the next round", async () => {
const { services, submitPlan, start } = await blockedRender(1);
const result = await autoFixRender(services, ACTOR, "render-1");
expect(result.status).toBe("rerendering");
expect(submitPlan).toHaveBeenCalledWith(
ACTOR,
expect.objectContaining({
projectId: "project-1",
createdBy: "agent",
rationale: expect.stringMatching(/^Auto-fix round 2 of 3:\n- clip c2: /),
}),
);
expect(start).toHaveBeenCalledWith(ACTOR, { planId: "plan-2", quality: "draft" });
expect(harness!.store.renders[1]!.autoFixRound).toBe(2);
});
it("hands over to the user after three rounds", async () => {
const { services, submitPlan } = await blockedRender(MAX_AUTO_FIX_ROUNDS);
const result = await autoFixRender(services, ACTOR, "render-1");
expect(result).toEqual({ status: "user_required", reason: "round_limit", changes: [] });
expect(submitPlan).not.toHaveBeenCalled();
});
it("does nothing for a render that is not blocked", async () => {
const { services } = await blockedRender(0);
harness!.store.renders[0]!.status = "done";
expect(await autoFixRender(services, ACTOR, "render-1")).toMatchObject({
status: "user_required",
reason: "not_blocked",
});
});
});

193
packages/adapters/src/media-gate-autofix.ts Normal file
View File

@ -0,0 +1,193 @@
import type { Edl, EdlError, GateReport, MediaRender, MediaRenderQuality } from "@rakazo/contracts";
import { EdlSchema, GateReportSchema, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
import { ACTIVE_RENDER_STATUSES } from "@rakazo/db";
import { planAutoFix } from "@rakazo/media";
import type {
MediaActor,
MediaMarketingService,
MediaRenderService,
MediaServiceDeps,
} from "./media-context.js";
/*
* Auto-fix for blocked renders (spec 6): only compliance-preserving repairs
* (trim or drop a clip, take another segment, change the crop, widen a
* redaction, cover a mark) through planAutoFix, re-checked here so no round
* can weaken protection or add anything new. Each round submits a new plan
* version with the reasons as rationale and starts a new render, which runs
* the gate again. After MAX_AUTO_FIX_ROUNDS the user takes over.
*/
export type AutoFixResult =
| { status: "rerendering"; round: number; render: MediaRender; changes: string[] }
| {
status: "user_required";
reason: "not_blocked" | "round_limit" | "no_safe_fix" | "plan_rejected";
changes: string[];
errors?: EdlError[];
};
export interface AutoFixDeps {
deps: Pick<MediaServiceDeps, "repos">;
marketing: Pick<MediaMarketingService, "submitPlan">;
render: Pick<MediaRenderService, "start">;
}
/**
* Why a proposed fix is not a pure compliance repair, or null when it is.
* Allowed: fewer or shorter clips, other segments, other crops, wider padding
* on the same face redactions, added blur redactions, marks switched to cover.
* Forbidden: anything that removes or weakens protection or adds content.
*/
export function nonCompliantChange(before: Edl, after: Edl): string | null {
const facesBefore = new Map(before.redactions.faces.map((f) => [f.id, f]));
for (const face of before.redactions.faces) {
const next = after.redactions.faces.find((f) => f.id === face.id);
if (!next) return `face redaction ${face.id} was removed`;
if (JSON.stringify(next.target) !== JSON.stringify(face.target))
return `face redaction ${face.id} changed its target`;
if (next.mode !== face.mode) return `face redaction ${face.id} changed its mode`;
if (next.paddingPct < face.paddingPct || next.minSizePct < face.minSizePct)
return `face redaction ${face.id} shrank`;
}
for (const face of after.redactions.faces) {
if (!facesBefore.has(face.id) && face.mode !== "blur")
return `added face redaction ${face.id} is not a blur`;
}
for (const mark of before.redactions.marks) {
const next = after.redactions.marks.find((m) => m.markId === mark.markId);
if (!next) return `mark ${mark.markId} lost its treatment`;
if (next.treatment !== mark.treatment && next.treatment !== "cover")
return `mark ${mark.markId} changed to ${next.treatment}`;
}
if (after.redactions.marks.length !== before.redactions.marks.length)
return "mark treatments were added";
const clipIds = new Set(after.tracks.video.map((clip) => clip.id));
for (const region of before.redactions.regions) {
if (!clipIds.has(region.clipId)) continue;
const next = after.redactions.regions.find((r) => r.id === region.id);
if (!next || JSON.stringify(next) !== JSON.stringify(region))
return `region ${region.id} changed`;
}
if (after.redactions.regions.length > before.redactions.regions.length)
return "regions were added";
if (JSON.stringify(after.redactions.voice) !== JSON.stringify(before.redactions.voice))
return "voice disguise changed";
const beforeClips = new Map(before.tracks.video.map((clip) => [clip.id, clip]));
for (const clip of after.tracks.video) {
const old = beforeClips.get(clip.id);
if (!old) return `clip ${clip.id} was added`;
if (clip.speed !== old.speed || clip.muted !== old.muted)
return `clip ${clip.id} changed speed or sound`;
if (JSON.stringify(clip.color) !== JSON.stringify(old.color))
return `clip ${clip.id} changed its colors`;
if (clip.assetId === old.assetId && (clip.inMs < old.inMs || clip.outMs > old.outMs)) {
if (clip.segmentId === old.segmentId) return `clip ${clip.id} grew`;
}
}
if (after.overlays.length > before.overlays.length) return "overlays were added";
for (const overlay of after.overlays) {
const old = before.overlays.find((o) => o.id === overlay.id);
if (!old || old.text !== overlay.text || old.kind !== overlay.kind)
return `overlay ${overlay.id} changed`;
}
if (
JSON.stringify(after.tracks.audio) !== JSON.stringify(before.tracks.audio) ||
JSON.stringify(after.captions) !== JSON.stringify(before.captions) ||
JSON.stringify(after.intro) !== JSON.stringify(before.intro) ||
JSON.stringify(after.outro) !== JSON.stringify(before.outro) ||
JSON.stringify(after.output) !== JSON.stringify(before.output)
) {
return "audio, captions, bookends, or output settings changed";
}
for (const alteration of after.metadata.aiAlterations) {
if (!before.metadata.aiAlterations.includes(alteration))
return `AI alteration ${alteration} was added`;
}
return null;
}
/** Plan one auto-fix round for a blocked render, or say why the user must take over. */
export function proposeAutoFix(input: {
edl: Edl;
report: GateReport;
round: number;
alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>;
}): { edl: Edl; changes: string[] } | { reason: "round_limit" | "no_safe_fix"; changes: string[] } {
if (input.round >= MAX_AUTO_FIX_ROUNDS) return { reason: "round_limit", changes: [] };
const fix = planAutoFix(
input.edl,
{ ...input.report, autoFixRound: input.round },
input.alternatives,
);
if (!fix) return { reason: "no_safe_fix", changes: [] };
const parsed = EdlSchema.safeParse(fix.edl);
if (!parsed.success) return { reason: "no_safe_fix", changes: fix.changes };
const violation = nonCompliantChange(input.edl, parsed.data);
if (violation)
return { reason: "no_safe_fix", changes: [...fix.changes, `rejected: ${violation}`] };
return { edl: parsed.data, changes: fix.changes };
}
/**
* Run one auto-fix round for a blocked render: new plan version (createdBy
* agent, rationale lists every change) and a new render carrying the next
* round number, whose gate decides again.
*/
export async function autoFixRender(
services: AutoFixDeps,
actor: MediaActor,
renderId: string,
): Promise<AutoFixResult> {
const { repos } = services.deps;
const row = await repos.getRenderRecord(actor.spaceId, renderId);
const report = GateReportSchema.nullable().parse(row.gateReport ?? null);
if (row.status !== "blocked" || !report || report.passed) {
return { status: "user_required", reason: "not_blocked", changes: [] };
}
const plan = await repos.getPlan(actor.spaceId, row.planId);
const project = await repos.getProjectRecord(actor.spaceId, row.projectId);
const assetIds = [
...new Set([...project.assetIds, ...plan.edl.tracks.video.map((clip) => clip.assetId)]),
];
const segments = await repos.listSegments(actor.spaceId, assetIds);
const alternatives = segments
.filter((segment) => !segment.paidOnly)
.map((segment) => ({
segmentId: segment.id,
assetId: segment.assetId,
startMs: segment.startMs,
endMs: segment.endMs,
}));
const proposal = proposeAutoFix({ edl: plan.edl, report, round: row.autoFixRound, alternatives });
if ("reason" in proposal) {
return { status: "user_required", reason: proposal.reason, changes: proposal.changes };
}
const round = row.autoFixRound + 1;
const rationale = [
`Auto-fix round ${round} of ${MAX_AUTO_FIX_ROUNDS}:`,
...proposal.changes,
].join("\n- ");
const submitted = await services.marketing.submitPlan(actor, {
projectId: row.projectId,
edl: proposal.edl,
rationale,
createdBy: "agent",
});
if (!submitted.ok) {
return {
status: "user_required",
reason: "plan_rejected",
changes: proposal.changes,
errors: submitted.errors,
};
}
const started = await services.render.start(actor, {
planId: submitted.plan.id,
quality: row.quality as MediaRenderQuality,
});
const updated = await repos.updateRender(actor.spaceId, started.id, { autoFixRound: round }, [
...ACTIVE_RENDER_STATUSES,
]);
return { status: "rerendering", round, render: updated ?? started, changes: proposal.changes };
}

250
packages/adapters/src/media-gate.test.ts Normal file
View File

@ -0,0 +1,250 @@
import { copyFile, rm } from "node:fs/promises";
import path from "node:path";
import type { MediaInferenceProvider, TrackFacesResult } from "@rakazo/adapter-kit";
import type { AnonymizeMode, Edl, PlatformProfile } from "@rakazo/contracts";
import { decodeVectors } from "@rakazo/media";
import { afterEach, describe, expect, it } from "vitest";
import { prepareSources } from "./media-anonymize.js";
import { ffmpegAvailable } from "./media-emulator-clips.js";
import { cutTimes, GATE_PIXEL_STEP_MS, pixelSampleTimes, runGate } from "./media-gate.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import {
ACTOR,
cleanProbe,
createPrivacyHarness,
testEdl,
testProfile,
} from "./media-privacy-fixtures.js";
import { PRIVACY_RECORD_FILE } from "./media-privacy-plan.js";
const hasFfmpeg = ffmpegAvailable();
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
/** Faces the fake detector reports on the render output. */
let outputFaces: TrackFacesResult = { sampleFps: 4, tracks: [] };
let classifierAvailable = true;
function withOutputDetector(emulator: PrivacyHarness["inference"]): MediaInferenceProvider {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.trackFaces = async (request) =>
path.basename(request.path).startsWith("output") ? outputFaces : emulator.trackFaces(request);
provider.capabilities = async () => {
const caps = await emulator.capabilities();
caps.operations.classifyContent.available = classifierAvailable;
return caps;
};
return provider;
}
async function prepared(
mode: AnonymizeMode,
options: { file?: string; unassigned?: boolean } = {},
) {
outputFaces = { sampleFps: 4, tracks: [] };
classifierAvailable = true;
harness = await createPrivacyHarness(withOutputDetector);
const asset = await harness.addAsset(options.file ?? "clip__faces-2____person-alice__.mp4");
harness.addPerson("alice", mode);
await harness.analyseFaces(asset.id, ["alice", options.unassigned ? null : "alice"]);
await harness.learnPerson("alice", asset.id);
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
const workDir = await harness.workDir(`gate-${mode}`);
const result = await prepareSources(harness.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const output = path.join(workDir, "output.mp4");
await copyFile(result.sources.get(asset.id)!.path, output);
return { h: harness, asset, edl, workDir, output };
}
function gate(
h: PrivacyHarness,
input: { edl: Edl; output: string },
options: {
profile?: PlatformProfile;
copy?: { title: string | null; caption: string | null; tags: string[] };
probe?: Partial<ReturnType<typeof cleanProbe>>;
} = {},
) {
return runGate(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl: input.edl,
profile: options.profile ?? testProfile(),
outputPath: input.output,
ffprobe: { ...cleanProbe(input.edl), ...options.probe },
copy: options.copy ?? { title: null, caption: null, tags: [] },
bundleHasDisclosure: false,
autoFixRound: 0,
});
}
const box = { x: 0.4, y: 0.4, w: 0.2, h: 0.2 };
const strangerEmbedding = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
describe("gate sampling", () => {
it("samples each clip range and spreads long ones", () => {
const edl = testEdl([
{ id: "a", assetId: "x", inMs: 0, outMs: 1000 },
{ id: "b", assetId: "x", inMs: 5000, outMs: 6000 },
]);
expect(cutTimes(edl)).toEqual([0, 999, 1000, 1999]);
expect(pixelSampleTimes(edl.tracks.video, false)).toEqual([0, 500, 960, 5000, 5500, 5960]);
expect(pixelSampleTimes(edl.tracks.video, true)).toEqual([0]);
const long = pixelSampleTimes([{ ...edl.tracks.video[0]!, outMs: 600_000 }], false);
expect(long).toHaveLength(40);
expect(long[1]! - long[0]!).toBeGreaterThanOrEqual(GATE_PIXEL_STEP_MS);
});
});
describe.skipIf(!hasFfmpeg)("runGate", () => {
it("passes a protected render with a clean classifier", async () => {
const { h, edl, output } = await prepared("blur");
const report = await gate(h, { edl, output });
expect(report.issues).toEqual([]);
expect(report.passed).toBe(true);
expect(report.sampledFrames).toBeGreaterThan(0);
});
it("classifies the rendered output and blocks above the threshold", async () => {
const { h, edl, workDir, output } = await prepared("blur");
const flagged = path.join(workDir, "output__nsfw-explicit_nudity-0.9__.mp4");
await copyFile(output, flagged);
const report = await gate(h, { edl, output: flagged });
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({
check: "content",
code: "content_threshold",
category: "explicit_nudity",
score: 0.9,
atMs: 0,
}),
);
});
it("blocks when the classifier is unavailable", async () => {
const { h, edl, output } = await prepared("blur");
classifierAvailable = false;
const report = await gate(h, { edl, output });
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "classifier_unavailable", severity: "block" }),
);
});
it("blocks a recognisable protected face and an unknown face in the output", async () => {
const { h, edl, output } = await prepared("blur");
const alice = h.store.people[0]!;
const embedding = [...decodeVectors(alice.centroid!, 16)[0]!];
outputFaces = {
sampleFps: 4,
tracks: [
{ trackId: "o1", keyframes: [{ tMs: 250, box, confidence: 0.9 }], embedding },
{
trackId: "o2",
keyframes: [{ tMs: 750, box, confidence: 0.9 }],
embedding: strangerEmbedding,
},
],
};
const report = await gate(h, { edl, output });
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "face_not_covered", personId: "alice", atMs: 250 }),
);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "unknown_face", atMs: 750 }),
);
});
it("lets a partially covered face stay detectable without counting it as unknown", async () => {
const { h, edl, workDir, output } = await prepared("partial_eyes");
expect(path.basename(workDir)).toBe("gate-partial_eyes");
outputFaces = {
sampleFps: 4,
tracks: [
{
trackId: "o1",
keyframes: [{ tMs: 250, box, confidence: 0.9 }],
embedding: strangerEmbedding,
},
],
};
const report = await gate(h, { edl, output });
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
});
it("requires consent and age proof of every participant for paid video", async () => {
const { h, edl, output } = await prepared("none");
// The profile's own flags are off; paid video turns them on regardless.
const paid = testProfile({ kind: "paid_video" });
const blocked = await gate(h, { edl, output }, { profile: paid });
expect(blocked.passed).toBe(false);
expect(blocked.issues.filter((i) => i.check === "consent").map((i) => i.code)).toEqual([
"consent_missing",
"age_unverified",
]);
const alice = h.store.people[0]!;
alice.consentRecordedAt = new Date("2026-01-01T00:00:00Z");
alice.ageVerifiedAt = new Date("2026-01-01T00:00:00Z");
alice.ageVerificationMethod = "document_on_file";
const allowed = await gate(h, { edl, output }, { profile: paid });
expect(allowed.issues.filter((i) => i.check === "consent")).toEqual([]);
expect(allowed.passed).toBe(true);
});
it("blocks consent profiles when a face belongs to nobody", async () => {
const { h, edl, output } = await prepared("none", { unassigned: true });
const profile = testProfile({
consent: { requirePerformerConsent: true, requireAgeVerification: true },
});
const alice = h.store.people[0]!;
alice.consentRecordedAt = new Date("2026-01-01T00:00:00Z");
alice.ageVerifiedAt = new Date("2026-01-01T00:00:00Z");
alice.ageVerificationMethod = "platform_verified";
const report = await gate(h, { edl, output }, { profile });
expect(report.issues).toEqual([
expect.objectContaining({ check: "consent", code: "unknown_performer", atMs: 0 }),
]);
});
it("blocks banned text and a duration outside the profile", async () => {
const { h, edl, output } = await prepared("blur");
const profile = testProfile({
duration: { hardMinMs: 0, hardMaxMs: 1500 },
text: {
captionMaxChars: 100,
hashtagLimit: 3,
linkPolicy: "forbidden",
bannedPatterns: [{ pattern: "free\\s+money", reason: "spam" }],
},
});
const report = await gate(
h,
{ edl, output },
{ profile, copy: { title: "Free money", caption: "see example.com", tags: [] } },
);
const codes = report.issues.map((issue) => issue.code);
expect(codes).toEqual(
expect.arrayContaining(["duration_too_long", "banned_pattern", "link_forbidden"]),
);
expect(report.passed).toBe(false);
});
it("blocks when the sources were not prepared for this render", async () => {
const { h, edl, workDir, output } = await prepared("blur");
await rm(path.join(workDir, PRIVACY_RECORD_FILE));
const report = await gate(h, { edl, output });
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "privacy_unverified", severity: "block" }),
);
});
});

396
packages/adapters/src/media-gate.ts Normal file
View File

@ -0,0 +1,396 @@
import path from "node:path";
import type {
ContentCategory,
Edl,
EdlVideoClip,
FfprobeReport,
GateIssue,
GateReport,
NormalizedBox,
PlatformProfile,
} from "@rakazo/contracts";
import type { GateInput } from "@rakazo/media";
import { clipLengthMs, evaluateGate, introMs, MAX_MARK_RESIDUAL } from "@rakazo/media";
import { assertInside } from "@rakazo/media/node";
import { consentIssues, enforceConsentFlags, hasAgeProof, hasConsent } from "./media-consent.js";
import type { MediaActor, MediaServiceDeps } from "./media-context.js";
import type { PrivacyToolOptions } from "./media-privacy-io.js";
import { inferenceCapabilities, maskTimes, operationAvailable } from "./media-privacy-io.js";
import type { LumaFrame } from "./media-privacy-pixels.js";
import {
controlAgreement,
extractLuma,
readPngLuma,
residualVisibility,
} from "./media-privacy-pixels.js";
import type { PrivacyCheck, PrivacyRecord, TimeWindow } from "./media-privacy-plan.js";
import {
boxAt,
isPartial,
matchPerson,
readPrivacyRecord,
sourceToOutput,
} from "./media-privacy-plan.js";
/*
* Gate execution on the rendered output (spec 6). Content is classified on the
* output file at the profile's sample rate plus every cut; faces are tracked on
* the output and matched to people locally. Protection is verified pixel by
* pixel: for every protected region prepareSources recorded, the prepared
* frame is compared with the unredacted frame at the same moment. The
* compiler only crops, scales, and composites prepared frames, so a region
* altered there stays altered in the output. Anything the gate cannot check
* blocks (fail closed).
*/
export type GateDeps = Pick<MediaServiceDeps, "repos" | "inference" | "mediaRoot" | "now"> &
PrivacyToolOptions;
/** Face tracking rate on the output. */
export const GATE_FACE_FPS = 4;
/** Pixel verification step inside each clip range. */
export const GATE_PIXEL_STEP_MS = 500;
/** Pixel samples per asset at most (spread evenly). */
export const MAX_PIXEL_SAMPLES_PER_ASSET = 40;
/** Below this agreement outside the treated regions, two frames are not comparable. */
const MIN_CONTROL_AGREEMENT = 0.8;
export interface GateRunInput {
actor: MediaActor;
projectId: string;
edl: Edl;
profile: PlatformProfile;
outputPath: string;
ffprobe: FfprobeReport;
copy: { title: string | null; caption: string | null; tags: string[] };
bundleHasDisclosure: boolean;
autoFixRound: number;
signal?: AbortSignal;
}
type Faces = GateInput["faces"];
type Marks = GateInput["marks"];
function block(issue: Omit<GateIssue, "severity">): GateIssue {
return { severity: "block", ...issue };
}
/** Output times of every cut: clip starts and ends, intro end. */
export function cutTimes(edl: Edl): number[] {
const intro = introMs(edl);
const times = new Set<number>();
if (intro > 0) times.add(intro);
for (const clip of edl.tracks.video) {
times.add(intro + clip.atMs);
times.add(Math.max(0, intro + clip.atMs + clipLengthMs(clip) - 1));
}
return [...times].sort((a, b) => a - b);
}
function clipsOf(edl: Edl, assetId: string): EdlVideoClip[] {
return edl.tracks.video.filter((clip) => clip.assetId === assetId);
}
/** Output windows of a source span across every clip of its asset. */
function toOutput(edl: Edl, assetId: string, span: TimeWindow): TimeWindow[] {
return clipsOf(edl, assetId).flatMap((clip) => {
const mapped = sourceToOutput(edl, clip, span);
return mapped ? [mapped] : [];
});
}
/** Evaluate the full gate for a render output. */
export async function runGate(deps: GateDeps, input: GateRunInput): Promise<GateReport> {
const { actor, edl, signal } = input;
const profile = enforceConsentFlags(input.profile);
const outputPath = await assertInside(deps.mediaRoot, input.outputPath);
const record = await readPrivacyRecord(path.dirname(outputPath));
const capabilities = await inferenceCapabilities(deps.inference, signal);
const extra: GateIssue[] = [];
const now = deps.now();
if (!record) {
extra.push(
block({
check: "anonymization",
code: "privacy_unverified",
message: "sources were not prepared for this render, so protection cannot be shown",
}),
);
}
// Content on the rendered output.
let content: GateInput["content"] = [];
let sampledFrames = 0;
if (Object.keys(profile.content.thresholds).length > 0) {
if (deps.inference && operationAvailable(capabilities, "classifyContent")) {
const classified = await deps.inference.classifyContent({
path: outputPath,
sampleFps: profile.content.sampleFps,
extraTimesMs: cutTimes(edl),
signal,
});
content = classified.frames.map((frame) => ({
tMs: frame.tMs,
scores: frame.scores as Partial<Record<ContentCategory, number>>,
}));
sampledFrames = content.length;
} else {
extra.push(
block({
check: "content",
code: "classifier_unavailable",
message: "the content classifier is unavailable, so the output cannot be checked",
}),
);
}
}
// Faces re-detected on the output.
const faces: Faces = [];
const matchedPeople = new Set<string>();
const protectedPeople = record?.protectedPeople ?? {};
const consentRequired =
profile.consent.requirePerformerConsent || profile.consent.requireAgeVerification;
const needFaces = Object.keys(protectedPeople).length > 0 || consentRequired;
if (needFaces) {
if (deps.inference && operationAvailable(capabilities, "trackFaces")) {
const tracked = await deps.inference.trackFaces({
path: outputPath,
sampleFps: GATE_FACE_FPS,
signal,
});
const centroids = await deps.repos.personCentroids(actor.spaceId);
const expected = expectedOutputFaces(edl, record);
const unmatchedAt = new Map<number, number>();
const slot = (tMs: number) => Math.round(tMs / (1000 / GATE_FACE_FPS));
for (const track of tracked.tracks) {
const personId = matchPerson(track.embedding, centroids);
if (personId) matchedPeople.add(personId);
const mode = personId ? protectedPeople[personId] : undefined;
for (const keyframe of track.keyframes) {
if (personId) {
// Partial modes and synthetic identities stay detectable by design;
// their regions are verified pixel by pixel below.
if (mode && (isPartial(mode) || mode === "synthetic_identity")) continue;
faces.push({ tMs: keyframe.tMs, box: keyframe.box, personId, coverage: mode ? 0 : 1 });
continue;
}
const key = slot(keyframe.tMs);
const seen = unmatchedAt.get(key) ?? 0;
unmatchedAt.set(key, seen + 1);
const allowance = expected.filter(
(w) => keyframe.tMs >= w.startMs && keyframe.tMs <= w.endMs,
).length;
if (seen < allowance) continue;
faces.push({ tMs: keyframe.tMs, box: keyframe.box, personId: null, coverage: 0 });
}
}
} else {
extra.push(
block({
check: "anonymization",
code: "face_check_unavailable",
message: "face detection is unavailable, so faces in the output cannot be checked",
}),
);
}
}
// Pixel verification of every recorded protection.
const marks: Marks = [];
if (record) {
const verified = await verifyPixels(deps, edl, record, signal);
for (const sample of verified) {
const covered = sample.residual <= MAX_MARK_RESIDUAL;
const { check } = sample;
if (check.kind === "face") {
if (check.personId && protectedPeople[check.personId]) {
faces.push({
tMs: sample.outputMs,
box: sample.box,
personId: check.personId,
coverage: covered ? 1 : 1 - sample.residual,
});
} else if (!covered) {
extra.push(
block({
check: "anonymization",
code: "face_not_covered",
message: `face covered ${((1 - sample.residual) * 100).toFixed(0)} %`,
atMs: sample.outputMs,
endMs: sample.outputMs,
score: 1 - sample.residual,
...(check.personId ? { personId: check.personId } : {}),
}),
);
}
} else {
marks.push({
markId: check.markId ?? check.id,
tMs: sample.outputMs,
residualVisibility: sample.residual,
});
}
}
for (const review of record.reviews) {
const window = toOutput(edl, review.assetId, {
startMs: 0,
endMs: Number.MAX_SAFE_INTEGER,
})[0];
extra.push(
block({
check: "anonymization",
code: "mark_needs_review",
message: `mark found with low confidence (${review.score.toFixed(2)}); review it`,
markId: review.markId,
...(window ? { atMs: window.startMs, endMs: window.endMs } : {}),
}),
);
}
}
const trackingGaps: GateInput["trackingGaps"] = (record?.trackingGaps ?? []).flatMap((gap) =>
toOutput(edl, gap.assetId, gap).map((window) => ({ ...window, personId: gap.personId })),
);
// People in the output and their consent state.
const ids = [...new Set([...(record?.visiblePeople ?? []), ...matchedPeople])].sort();
const persons = ids.length ? await deps.repos.listPersonRecords(actor.spaceId, ids) : [];
const people: GateInput["people"] = persons.map((person) => ({
id: person.id,
anonymizeByDefault: protectedPeople[person.id] ?? "none",
consentRecordedAt: hasConsent(person, now)
? new Date(person.consentRecordedAt!).toISOString()
: null,
ageVerifiedAt: hasAgeProof(person, now) ? new Date(person.ageVerifiedAt!).toISOString() : null,
}));
if (record && consentRequired) {
const unassigned = record.unassignedFaces.flatMap((face) =>
toOutput(edl, face.assetId, face).map((w) => ({ atMs: w.startMs, endMs: w.endMs })),
);
extra.push(...consentIssues(profile, [], unassigned, now));
}
const report = evaluateGate({
profile,
edl,
ffprobe: input.ffprobe,
content,
faces,
trackingGaps,
marks,
people,
copy: input.copy,
aiAlterations: record?.aiAlterations ?? [],
bundleHasDisclosure: input.bundleHasDisclosure,
watermarkBox: null,
sampledFrames,
autoFixRound: input.autoFixRound,
now,
});
const issues = [...report.issues, ...extra];
return { ...report, issues, passed: !issues.some((issue) => issue.severity === "block") };
}
/** Output windows where a face stays detectable by design (synthetic identity, partial modes). */
function expectedOutputFaces(edl: Edl, record: PrivacyRecord | null): TimeWindow[] {
return (record?.expectedFaces ?? []).flatMap((face) => toOutput(edl, face.assetId, face));
}
interface PixelSample {
check: PrivacyCheck;
outputMs: number;
box: NormalizedBox;
residual: number;
}
/** Source times to verify per asset: every GATE_PIXEL_STEP_MS inside its clips, spread evenly. */
export function pixelSampleTimes(clips: EdlVideoClip[], still: boolean): number[] {
if (still) return [0];
const times = new Set<number>();
for (const clip of clips) {
for (let t = clip.inMs; t < clip.outMs; t += GATE_PIXEL_STEP_MS) times.add(t);
times.add(Math.max(clip.inMs, clip.outMs - 40));
}
const sorted = [...times].sort((a, b) => a - b);
if (sorted.length <= MAX_PIXEL_SAMPLES_PER_ASSET) return sorted;
const step = (sorted.length - 1) / (MAX_PIXEL_SAMPLES_PER_ASSET - 1);
return Array.from(
{ length: MAX_PIXEL_SAMPLES_PER_ASSET },
(_, i) => sorted[Math.round(i * step)]!,
);
}
async function verifyPixels(
deps: GateDeps,
edl: Edl,
record: PrivacyRecord,
signal?: AbortSignal,
): Promise<PixelSample[]> {
const samples: PixelSample[] = [];
const maskCache = new Map<string, number[]>();
for (const [assetId, entry] of Object.entries(record.assets).sort((a, b) =>
a[0].localeCompare(b[0]),
)) {
if (entry.checks.length === 0) continue;
const clips = clipsOf(edl, assetId);
const treatedSize = entry.crop
? { width: entry.crop.w, height: entry.crop.h }
: { width: entry.width, height: entry.height };
const offset = entry.crop ? { x: entry.crop.x, y: entry.crop.y } : undefined;
for (const tMs of pixelSampleTimes(clips, entry.still)) {
signal?.throwIfAborted();
const active = entry.checks.flatMap((check) => {
const box = entry.still ? (check.keyframes[0]?.box ?? null) : boxAt(check.keyframes, tMs);
return box ? [{ check, box }] : [];
});
if (active.length === 0) continue;
const tool = { ffmpegPath: deps.ffmpegPath, signal };
const source = await extractLuma(
entry.basePath,
{ tMs, width: entry.width, height: entry.height, still: entry.still },
tool,
);
const treated = await extractLuma(
entry.preparedPath,
{ tMs, ...treatedSize, still: entry.still },
tool,
);
const aligned =
controlAgreement({ source, treated, boxes: active.map((a) => a.box), offset }) >=
MIN_CONTROL_AGREEMENT;
const outputTimes = entry.still
? clips.map((clip) => introMs(edl) + clip.atMs)
: clips.flatMap((clip) => {
const mapped = sourceToOutput(edl, clip, { startMs: tMs, endMs: tMs });
return mapped ? [mapped.startMs] : [];
});
for (const { check, box } of active) {
let residual = 1;
if (aligned) {
const mask = check.maskDir ? await maskAt(check.maskDir, tMs, maskCache) : undefined;
residual = residualVisibility({ source, treated, box, offset, mask });
}
for (const outputMs of outputTimes) samples.push({ check, outputMs, box, residual });
}
}
}
return samples;
}
/** The mask frame nearest at or before `tMs` (or the first one), if the directory has any. */
async function maskAt(
maskDir: string,
tMs: number,
cache: Map<string, number[]>,
): Promise<LumaFrame | undefined> {
let times = cache.get(maskDir);
if (!times) {
times = await maskTimes(maskDir).catch(() => []);
cache.set(maskDir, times);
}
if (times.length === 0) return undefined;
const before = times.filter((t) => t <= tMs).at(-1) ?? times[0]!;
return readPngLuma(path.join(maskDir, `${before}.png`));
}

300
packages/adapters/src/media-marks.test.ts Normal file
View File

@ -0,0 +1,300 @@
import { copyFile, stat } from "node:fs/promises";
import path from "node:path";
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
import type { Edl } from "@rakazo/contracts";
import { afterEach, describe, expect, it, vi } from "vitest";
import { prepareSources } from "./media-anonymize.js";
import { ffmpegAvailable } from "./media-emulator-clips.js";
import { runGate } from "./media-gate.js";
import { anchorFromFace, createMark, placeAnchor, promptBox, updateMark } from "./media-marks.js";
import { createMediaPrivacyService } from "./media-privacy.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import {
ACTOR,
cleanProbe,
createPrivacyHarness,
testEdl,
testProfile,
} from "./media-privacy-fixtures.js";
import { PrivacyUnavailableError } from "./media-privacy-io.js";
import { readPrivacyRecord } from "./media-privacy-plan.js";
const hasFfmpeg = ffmpegAvailable();
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
const tattoo = { tMs: 500, box: { x: 0.6, y: 0.55, w: 0.2, h: 0.25 } };
async function render(h: PrivacyHarness, edl: Edl, name: string) {
const workDir = await h.workDir(name);
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
return { workDir, prepared };
}
async function gate(h: PrivacyHarness, edl: Edl, workDir: string, prepared: string) {
const outputPath = path.join(workDir, "output.mp4");
await copyFile(prepared, outputPath);
return runGate(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
profile: testProfile(),
outputPath,
ffprobe: cleanProbe(edl),
copy: { title: null, caption: null, tags: [] },
bundleHasDisclosure: false,
autoFixRound: 0,
});
}
describe("mark geometry", () => {
it("turns a point prompt into a small box inside the frame", () => {
expect(promptBox({ tMs: 0, point: { x: 0.99, y: 0.01 } })).toEqual({
x: 0.88,
y: 0,
w: 0.12,
h: 0.12,
});
expect(() => promptBox({ tMs: 0 })).toThrow(PrivacyUnavailableError);
});
it("keeps a mark's place relative to the face", () => {
const anchor = anchorFromFace(
{ x: 0.5, y: 0.5, w: 0.1, h: 0.1 },
{ x: 0.4, y: 0.3, w: 0.2, h: 0.2 },
);
const placed = placeAnchor(anchor, { x: 0.1, y: 0.1, w: 0.4, h: 0.4 })!;
expect(placed.x).toBeCloseTo(0.3);
expect(placed.y).toBeCloseTo(0.5);
expect(placed.w).toBeCloseTo(0.2);
expect(placeAnchor(anchor, { x: 0.95, y: 0.95, w: 0.4, h: 0.4 })).toBeNull();
});
});
describe.skipIf(!hasFfmpeg)("marks", () => {
it("segments, propagates, scores, and stores the mask under the media root", async () => {
harness = await createPrivacyHarness();
const asset = await harness.addAsset("clip.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "blur",
});
expect(mark).toMatchObject({ treatment: "blur", qualityScore: 0.9, suggestedFallback: null });
const stored = harness.store.marks[0]!;
expect(path.isAbsolute(stored.maskDir!)).toBe(false);
expect((await stat(path.join(harness.root, stored.maskDir!, "0.png"))).isFile()).toBe(true);
});
it("suggests a fallback when inpainting scores low and rescoring follows the treatment", async () => {
harness = await createPrivacyHarness();
const asset = await harness.addAsset("clip__inpaintq-0.3__.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "inpaint",
});
expect(mark.qualityScore).toBeLessThanOrEqual(0.3);
expect(mark.suggestedFallback).toBe("cover");
const blurred = await updateMark(harness.deps, ACTOR, { markId: mark.id, treatment: "blur" });
expect(blurred).toMatchObject({
treatment: "blur",
qualityScore: 0.9,
suggestedFallback: null,
});
});
it("keeps a mark without a worker and rendering it fails closed", async () => {
harness = await createPrivacyHarness(() => null);
const asset = await harness.addAsset("clip.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "license_plate",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "cover",
});
expect(mark.qualityScore).toBeNull();
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
await expect(render(harness, edl, "no-worker")).rejects.toBeInstanceOf(PrivacyUnavailableError);
});
it("treats a marked region through the video and the gate checks what is left", async () => {
harness = await createPrivacyHarness();
const asset = await harness.addAsset("clip.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "none",
});
// The plan asks for a cover even though the mark itself says none.
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
redactions: { faces: [], marks: [{ markId: mark.id, treatment: "cover" }], regions: [] },
});
const { workDir, prepared } = await render(harness, edl, "cover");
const record = await readPrivacyRecord(workDir);
expect(record?.assets[asset.id]?.checks).toEqual([
expect.objectContaining({ kind: "mark", markId: mark.id }),
]);
const report = await gate(harness, edl, workDir, prepared.sources.get(asset.id)!.path);
expect(report.passed).toBe(true);
});
it("blocks when the treatment left the mark visible", async () => {
harness = await createPrivacyHarness((emulator) => {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.redact = async (request) => {
await copyFile(request.path, request.outPath);
return { outPath: request.outPath, framesProcessed: 50 };
};
return provider;
});
const asset = await harness.addAsset("clip.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "background_detail",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "blur",
});
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
const { workDir, prepared } = await render(harness, edl, "leak");
const report = await gate(harness, edl, workDir, prepared.sources.get(asset.id)!.path);
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "mark_visible", markId: mark.id }),
);
});
it("inpaints when quality holds and covers instead when it does not", async () => {
harness = await createPrivacyHarness();
const good = await harness.addAsset("good.mp4");
const poor = await harness.addAsset("poor__inpaintq-0.3__.mp4");
for (const asset of [good, poor]) {
await createMark(harness.deps, ACTOR, {
kind: "scar",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "inpaint",
});
}
const edl = testEdl([
{ id: "c1", assetId: good.id, inMs: 0, outMs: 1000 },
{ id: "c2", assetId: poor.id, inMs: 0, outMs: 1000 },
]);
const { prepared } = await render(harness, edl, "inpaint");
expect(prepared.aiAlterations).toEqual(["inpaint"]);
expect(prepared.fallbacks).toEqual([
expect.stringMatching(/inpainting scored 0\.30; covered instead$/),
]);
});
it("re-finds a person's standing mark in another video relative to the face", async () => {
harness = await createPrivacyHarness();
const origin = await harness.addAsset("a__faces-1____person-alice__.mp4");
const other = await harness.addAsset("b__faces-1____person-alice__.mp4");
harness.addPerson("alice", "none");
await harness.analyseFaces(origin.id, ["alice"]);
await harness.analyseFaces(other.id, ["alice"]);
const segment = vi.spyOn(harness.inference, "segmentPrompted");
const mark = await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: "left forearm",
personId: "alice",
assetId: origin.id,
prompt: tattoo,
treatment: "blur",
});
const edl = testEdl([{ id: "c1", assetId: other.id, inMs: 0, outMs: 2000 }]);
const { workDir, prepared } = await render(harness, edl, "standing");
expect(segment).toHaveBeenCalledTimes(2);
const reprompt = segment.mock.calls[1]![0];
expect(reprompt.path).toContain("b__faces-1");
expect(reprompt.prompt.box).toBeDefined();
const record = await readPrivacyRecord(workDir);
expect(record?.reviews).toEqual([]);
expect(record?.assets[other.id]?.checks).toEqual([
expect.objectContaining({ kind: "mark", markId: mark.id }),
]);
const report = await gate(harness, edl, workDir, prepared.sources.get(other.id)!.path);
expect(report.passed).toBe(true);
});
it("sends a low-confidence re-find to review, which blocks the export", async () => {
harness = await createPrivacyHarness((emulator) => {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.segmentPrompted = async (request) => {
const result = await emulator.segmentPrompted(request);
return request.path.includes("b__") ? { ...result, score: 0.3 } : result;
};
return provider;
});
const origin = await harness.addAsset("a__faces-1____person-alice__.mp4");
const other = await harness.addAsset("b__faces-1____person-alice__.mp4");
harness.addPerson("alice", "none");
await harness.analyseFaces(origin.id, ["alice"]);
await harness.analyseFaces(other.id, ["alice"]);
const mark = await createMark(harness.deps, ACTOR, {
kind: "jewelry",
label: null,
personId: "alice",
assetId: origin.id,
prompt: tattoo,
treatment: "blur",
});
const edl = testEdl([{ id: "c1", assetId: other.id, inMs: 0, outMs: 2000 }]);
const { workDir, prepared } = await render(harness, edl, "review");
const report = await gate(harness, edl, workDir, prepared.sources.get(other.id)!.path);
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "mark_needs_review", markId: mark.id }),
);
});
it("removes a person with their tracks, marks, and files", async () => {
harness = await createPrivacyHarness();
const asset = await harness.addAsset("clip__faces-1____person-alice__.mp4");
harness.addPerson("alice", "blur");
await harness.analyseFaces(asset.id, ["alice"]);
await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: null,
personId: "alice",
assetId: asset.id,
prompt: tattoo,
treatment: "blur",
});
const files = [
path.join(harness.root, harness.store.tracks[0]!.keyframesPath),
path.join(harness.root, harness.store.marks[0]!.maskDir!),
];
const service = createMediaPrivacyService(harness.deps);
await service.removePerson(ACTOR, "alice");
expect(harness.store.people).toEqual([]);
expect(harness.store.tracks).toEqual([]);
for (const file of files) await expect(stat(file)).rejects.toThrow();
});
});

322
packages/adapters/src/media-marks.ts Normal file
View File

@ -0,0 +1,322 @@
import { mkdir, rm } from "node:fs/promises";
import path from "node:path";
import type { SegmentPromptedResult } from "@rakazo/adapter-kit";
import type {
BoxKeyframe,
MarkTreatment,
MediaMark,
MediaMarkPrompt,
NormalizedBox,
} from "@rakazo/contracts";
import type { CreateMarkInputType, MediaActor, MediaServiceDeps } from "./media-context.js";
import type { PrivacyToolOptions } from "./media-privacy-io.js";
import {
analysisFile,
inferenceCapabilities,
operationAvailable,
PrivacyUnavailableError,
probeSource,
resolveStored,
toStored,
} from "./media-privacy-io.js";
import { extractLuma, temporalStability } from "./media-privacy-pixels.js";
import type { TrackKeyframe } from "./media-privacy-plan.js";
import { boxAt, FACE_MIN_CONFIDENCE, MARK_QUALITY_MIN, unionBox } from "./media-privacy-plan.js";
/*
* Identifying marks (spec 7): the user marks once by click or box, the
* segmenter (SAM2 in the sidecar) propagates the mask through the video, and
* the treatment is scored. A score under MARK_QUALITY_MIN never ships
* silently: the mark carries a suggested fallback and rendering applies it.
*/
/** Side of the box a point prompt stands for, normalized. */
const POINT_PROMPT_SIZE = 0.12;
/** Frames sampled for the local flicker measurement of an inpaint preview. */
const STABILITY_FRAMES = 6;
const STABILITY_STEP_MS = 200;
export type MarkDeps = Pick<MediaServiceDeps, "repos" | "inference" | "mediaRoot" | "sources"> &
PrivacyToolOptions;
/** Directory of a mark's masks and previews, relative to the media root. */
export function markDir(spaceId: string, markId: string): string {
return path.join(spaceId, "marks", markId);
}
/** The prompt as a box: a point prompt stands for a small box around it. */
export function promptBox(prompt: MediaMarkPrompt): NormalizedBox {
if (prompt.box) return prompt.box;
if (!prompt.point) throw new PrivacyUnavailableError("a mark needs a box or a point");
const x = Math.min(Math.max(0, prompt.point.x - POINT_PROMPT_SIZE / 2), 1 - POINT_PROMPT_SIZE);
const y = Math.min(Math.max(0, prompt.point.y - POINT_PROMPT_SIZE / 2), 1 - POINT_PROMPT_SIZE);
return { x, y, w: POINT_PROMPT_SIZE, h: POINT_PROMPT_SIZE };
}
/** Fallback to propose when a treatment scores too low. */
export function fallbackFor(treatment: MarkTreatment): MarkTreatment {
return treatment === "cover" ? "reframe_out" : "cover";
}
/**
* Score a treatment. Segmentation confidence bounds every treatment (a mask
* that does not follow the mark cannot hide it); inpainting also needs the
* provider's temporal and edge quality and a local flicker check of its
* preview. Returns the score and, under MARK_QUALITY_MIN, the fallback.
*/
export async function scoreTreatment(
deps: MarkDeps,
input: {
treatment: MarkTreatment;
segmentationScore: number;
file: string;
maskDir: string;
keyframes: BoxKeyframe[];
previewDir: string;
signal?: AbortSignal;
},
): Promise<{ qualityScore: number; suggestedFallback: MarkTreatment | null }> {
let score = input.segmentationScore;
if (input.treatment === "inpaint") {
const capabilities = await inferenceCapabilities(deps.inference, input.signal);
if (!deps.inference || !operationAvailable(capabilities, "inpaintVideo")) {
return { qualityScore: 0, suggestedFallback: "cover" };
}
await mkdir(input.previewDir, { recursive: true });
const preview = await deps.inference.inpaintVideo({
path: input.file,
maskDir: input.maskDir,
outPath: path.join(input.previewDir, "inpaint-preview.mp4"),
signal: input.signal,
});
const local = await previewStability(deps, preview.outPath, input.keyframes, input.signal);
score = Math.min(score, preview.quality.score, preview.quality.temporalStability, local);
}
const qualityScore = Math.round(score * 1000) / 1000;
return {
qualityScore,
suggestedFallback:
qualityScore < MARK_QUALITY_MIN && input.treatment !== "none"
? fallbackFor(input.treatment)
: null,
};
}
/** Flicker inside the mark region of an inpainted preview around the first keyframe. */
async function previewStability(
deps: MarkDeps,
file: string,
keyframes: BoxKeyframe[],
signal?: AbortSignal,
): Promise<number> {
const first = keyframes[0];
if (!first) return 1;
const { width, height } = await frameSize(file, deps, signal);
const frames = [];
let region: NormalizedBox = first.box;
for (let i = 0; i < STABILITY_FRAMES; i++) {
const tMs = first.tMs + i * STABILITY_STEP_MS;
const box = boxAt(keyframes, tMs, STABILITY_STEP_MS);
if (box) region = unionBox(region, box);
try {
frames.push(
await extractLuma(file, { tMs, width, height }, { ffmpegPath: deps.ffmpegPath, signal }),
);
} catch {
break;
}
}
return temporalStability(frames, region);
}
async function frameSize(file: string, deps: PrivacyToolOptions, signal?: AbortSignal) {
const probed = await probeSource(file, "video", { ffprobePath: deps.ffprobePath, signal });
return { width: probed.width, height: probed.height };
}
/** Segment a prompt and propagate it through the file into `outDir`. */
export async function segmentMark(
deps: MarkDeps,
input: { file: string; prompt: MediaMarkPrompt; outDir: string; signal?: AbortSignal },
): Promise<SegmentPromptedResult> {
const capabilities = await inferenceCapabilities(deps.inference, input.signal);
if (!deps.inference || !operationAvailable(capabilities, "segmentPrompted")) {
throw new PrivacyUnavailableError("mark segmentation needs the media worker");
}
await rm(input.outDir, { recursive: true, force: true });
await mkdir(input.outDir, { recursive: true });
return deps.inference.segmentPrompted({
path: input.file,
prompt: input.prompt,
propagate: true,
outDir: input.outDir,
signal: input.signal,
});
}
/**
* Where a person-bound mark sits relative to the person's face, from the
* frame it was drawn in: offsets and size in face-box units.
*/
export interface MarkAnchor {
dx: number;
dy: number;
w: number;
h: number;
}
export function anchorFromFace(mark: NormalizedBox, face: NormalizedBox): MarkAnchor {
return {
dx: (mark.x - face.x) / face.w,
dy: (mark.y - face.y) / face.h,
w: mark.w / face.w,
h: mark.h / face.h,
};
}
/** Place an anchor at a face, clamped to the frame. Null when it falls outside. */
export function placeAnchor(anchor: MarkAnchor, face: NormalizedBox): NormalizedBox | null {
const x0 = face.x + anchor.dx * face.w;
const y0 = face.y + anchor.dy * face.h;
const x1 = x0 + anchor.w * face.w;
const y1 = y0 + anchor.h * face.h;
const x = Math.max(0, x0);
const y = Math.max(0, y0);
const w = Math.min(1, x1) - x;
const h = Math.min(1, y1) - y;
return w > 0.001 && h > 0.001 ? { x, y, w, h } : null;
}
/** First confident face keyframe of a track at or after `fromMs`. */
export function firstConfident(keyframes: TrackKeyframe[], fromMs = 0): TrackKeyframe | null {
return (
[...keyframes]
.filter((k) => k.tMs >= fromMs && (k.confidence ?? 1) >= FACE_MIN_CONFIDENCE)
.sort((a, b) => a.tMs - b.tMs)[0] ?? null
);
}
/** Face box of a track nearest to `tMs` (within 250 ms). */
export function faceAt(keyframes: TrackKeyframe[], tMs: number): NormalizedBox | null {
let best: TrackKeyframe | null = null;
for (const k of keyframes) {
if (Math.abs(k.tMs - tMs) > 250) continue;
if (!best || Math.abs(k.tMs - tMs) < Math.abs(best.tMs - tMs)) best = k;
}
return best?.box ?? null;
}
// ---------------------------------------------------------------------------
// Service operations
// ---------------------------------------------------------------------------
/**
* Create a mark: store it, segment from the prompt, propagate through the
* asset, score the treatment, and store a fallback suggestion when the score
* is low. Without the media worker the mark is kept unsegmented; rendering a
* clip of that asset then fails closed until segmentation is possible.
*/
export async function createMark(
deps: MarkDeps,
actor: MediaActor,
input: CreateMarkInputType,
signal?: AbortSignal,
): Promise<MediaMark> {
const asset = await deps.repos.getAssetRecord(actor.spaceId, input.assetId);
const mark = await deps.repos.createMark(actor.spaceId, {
kind: input.kind,
label: input.label,
personId: input.personId,
assetId: input.assetId,
prompt: input.prompt,
treatment: input.treatment,
});
const file = await analysisFile(deps, actor.spaceId, asset);
const capabilities = await inferenceCapabilities(deps.inference, signal);
if (!file || !operationAvailable(capabilities, "segmentPrompted")) return mark;
const relative = markDir(actor.spaceId, mark.id);
const outDir = await resolveStored(deps.mediaRoot, path.join(relative, "masks"));
try {
const segmented = await segmentMark(deps, { file, prompt: input.prompt, outDir, signal });
const scored = await scoreTreatment(deps, {
treatment: input.treatment,
segmentationScore: segmented.score,
file,
maskDir: segmented.maskDir,
keyframes: segmented.keyframes,
previewDir: await resolveStored(deps.mediaRoot, relative),
signal,
});
const record = await deps.repos.recordMarkResult(actor.spaceId, mark.id, {
...scored,
maskDir: await toStored(deps.mediaRoot, segmented.maskDir),
});
const { maskDir: _maskDir, ...dto } = record;
return dto;
} catch (error) {
// The mark stays without a mask (rendering fails closed); the caller sees why.
await deps.repos.removeMark(actor.spaceId, mark.id);
await rm(await resolveStored(deps.mediaRoot, relative), { recursive: true, force: true });
throw error;
}
}
/**
* Update treatment, label, or person. A new treatment is segmented and scored
* again so its suggestion reflects that treatment.
*/
export async function updateMark(
deps: MarkDeps,
actor: MediaActor,
input: {
markId: string;
treatment?: MarkTreatment;
label?: string | null;
personId?: string | null;
},
signal?: AbortSignal,
): Promise<MediaMark> {
const before = await deps.repos.getMark(actor.spaceId, input.markId);
const updated = await deps.repos.updateMark(actor.spaceId, input.markId, {
...(input.treatment !== undefined ? { treatment: input.treatment } : {}),
...(input.label !== undefined ? { label: input.label } : {}),
...(input.personId !== undefined ? { personId: input.personId } : {}),
});
if (input.treatment === undefined || input.treatment === before.treatment) return updated;
if (!before.assetId || !before.prompt) return updated;
const asset = await deps.repos.getAssetRecord(actor.spaceId, before.assetId);
const file = await analysisFile(deps, actor.spaceId, asset);
const capabilities = await inferenceCapabilities(deps.inference, signal);
if (!file || !operationAvailable(capabilities, "segmentPrompted")) {
// Without the worker the old score says nothing about the new treatment.
const record = await deps.repos.recordMarkResult(actor.spaceId, input.markId, {
qualityScore: null,
suggestedFallback: null,
maskDir: before.maskDir,
});
const { maskDir: _maskDir, ...dto } = record;
return dto;
}
const relative = markDir(actor.spaceId, input.markId);
const segmented = await segmentMark(deps, {
file,
prompt: before.prompt,
outDir: await resolveStored(deps.mediaRoot, path.join(relative, "masks")),
signal,
});
const scored = await scoreTreatment(deps, {
treatment: input.treatment,
segmentationScore: segmented.score,
file,
maskDir: segmented.maskDir,
keyframes: segmented.keyframes,
previewDir: await resolveStored(deps.mediaRoot, relative),
signal,
});
const record = await deps.repos.recordMarkResult(actor.spaceId, input.markId, {
...scored,
maskDir: await toStored(deps.mediaRoot, segmented.maskDir),
});
const { maskDir: _maskDir, ...dto } = record;
return dto;
}

437
packages/adapters/src/media-privacy-fixtures.ts Normal file
View File

@ -0,0 +1,437 @@
import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
import type {
AnonymizeMode,
Edl,
FfprobeReport,
MarkTreatment,
MediaMarkKind,
MediaMarkPrompt,
PlatformProfile,
} from "@rakazo/contracts";
import { EdlSchema, PlatformProfileSchema } from "@rakazo/contracts";
import type { MediaRepos } from "@rakazo/db";
import { edlDurationMs, encodeVectors } from "@rakazo/media";
import type { MediaServiceDeps } from "./media-context.js";
import { EmulatorMediaInferenceProvider } from "./media-emulator.js";
import { renderTestClip } from "./media-emulator-clips.js";
import { runMediaTool } from "./media-folder-source-probe.js";
/*
* In-memory repositories and synthetic ffmpeg clips for the privacy tests.
* Clip names carry media-emulator markers (faces, person seeds, detection
* gaps, classifier scores); nothing here shows a real person.
*/
export const SPACE = "space-1";
export const USER = "user-1";
export const ACTOR = { spaceId: SPACE, userId: USER };
interface AssetRow {
id: string;
spaceId: string;
sourceId: string;
externalId: string;
kind: string;
storage: string;
storagePath: string | null;
proxyPath: string | null;
}
interface PersonRow {
id: string;
spaceId: string;
displayName: string;
anonymizeByDefault: string;
consentRecordedAt: Date | null;
ageVerifiedAt: Date | null;
ageVerificationMethod: string;
syntheticSeed: number | null;
centroid: Uint8Array | null;
}
interface TrackRow {
id: string;
assetId: string;
personId: string | null;
startMs: number;
endMs: number;
confidenceMin: number;
keyframesPath: string;
modelVersion: string;
}
interface MarkRow {
id: string;
kind: MediaMarkKind;
label: string | null;
personId: string | null;
assetId: string | null;
prompt: MediaMarkPrompt | null;
treatment: MarkTreatment;
qualityScore: number | null;
suggestedFallback: MarkTreatment | null;
maskDir: string | null;
createdAt: string;
}
export interface PrivacyHarness {
root: string;
inference: EmulatorMediaInferenceProvider;
deps: MediaServiceDeps;
store: {
assets: AssetRow[];
people: PersonRow[];
tracks: TrackRow[];
marks: MarkRow[];
renders: Array<Record<string, unknown>>;
plans: Array<Record<string, unknown>>;
segments: Array<Record<string, unknown>>;
};
/** A synthetic clip, or a still (PNG) when the name ends in .png. */
addAsset(fileName: string, options?: { seconds?: number; id?: string }): Promise<AssetRow>;
addPerson(
id: string,
mode: AnonymizeMode,
consent?: { consent?: boolean; age?: boolean },
): PersonRow;
/** Run emulator tracking on the asset and store the tracks (the analysis result). */
analyseFaces(assetId: string, personIds: Array<string | null>, sampleFps?: number): Promise<void>;
/** Store the emulator identity embedding of a person seed as the person's centroid. */
learnPerson(personId: string, assetId: string, trackIndex?: number): Promise<void>;
workDir(name: string): Promise<string>;
cleanup(): Promise<void>;
}
export function testEdl(
clips: Array<{ id: string; assetId: string; inMs: number; outMs: number; atMs?: number }>,
extra: Partial<Pick<Edl, "redactions">> & { width?: number; height?: number } = {},
): Edl {
let cursor = 0;
const video = clips.map((clip) => {
const atMs = clip.atMs ?? cursor;
cursor = atMs + clip.outMs - clip.inMs;
return { ...clip, atMs, reframe: "none" as const };
});
return EdlSchema.parse({
edlVersion: 1,
output: {
profileId: "test-profile",
width: extra.width ?? 320,
height: extra.height ?? 180,
fps: 25,
durationTargetMs: cursor,
durationHardMinMs: 0,
durationHardMaxMs: 600_000,
},
tracks: { video },
redactions: extra.redactions ?? {},
metadata: {},
});
}
export function testProfile(overrides: Record<string, unknown> = {}): PlatformProfile {
return PlatformProfileSchema.parse({
id: "test-profile",
name: "Test profile",
kind: "free_social",
reviewedAt: "2026-01-01",
video: {
aspectRatios: ["16:9", "9:16"],
width: 1920,
height: 1920,
fps: 60,
codec: "h264",
maxBitrateKbps: 20_000,
maxFileSizeMb: 500,
},
duration: { hardMinMs: 0, hardMaxMs: 600_000 },
safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 },
captions: { recommended: false },
audio: {},
text: { captionMaxChars: 2200, hashtagLimit: 30, linkPolicy: "allowed" },
content: { thresholds: { explicit_nudity: 0.5 } },
disclosure: { aiAlteredLabelRequired: false },
consent: { requirePerformerConsent: false, requireAgeVerification: false },
export: {},
...overrides,
});
}
/** An ffprobe report that passes the technical check for an EDL. */
export function cleanProbe(edl: Edl): FfprobeReport {
return {
durationMs: edlDurationMs(edl),
width: edl.output.width,
height: edl.output.height,
fps: edl.output.fps,
videoCodec: "h264",
audioCodec: null,
sizeBytes: 10_000,
integratedLufs: null,
truePeakDb: null,
tags: {},
};
}
function isolation(): Error {
return new Error("not found in space");
}
export async function createPrivacyHarness(
inferenceOverride?: (emulator: EmulatorMediaInferenceProvider) => MediaInferenceProvider | null,
): Promise<PrivacyHarness> {
const root = await realpath(await mkdtemp(path.join(tmpdir(), "media-privacy-")));
const emulator = new EmulatorMediaInferenceProvider({ root });
const store: PrivacyHarness["store"] = {
assets: [],
people: [],
tracks: [],
marks: [],
renders: [],
plans: [],
segments: [],
};
let counter = 0;
const nextId = (prefix: string) => `${prefix}${++counter}`;
const markDto = (row: MarkRow) => {
const { maskDir: _maskDir, ...dto } = row;
return dto;
};
const findMark = (id: string) => {
const row = store.marks.find((mark) => mark.id === id);
if (!row) throw isolation();
return row;
};
const repos = {
async listAssetRecords(_spaceId: string, ids: string[]) {
return store.assets.filter((asset) => ids.includes(asset.id));
},
async getAssetRecord(_spaceId: string, id: string) {
const asset = store.assets.find((row) => row.id === id);
if (!asset) throw isolation();
return asset;
},
async getSource() {
return { kind: "folder", config: {} };
},
async getProjectRecord(_spaceId: string, projectId: string) {
return {
id: projectId,
brandKitId: null,
assetIds: store.assets.map((a) => a.id),
};
},
async getBrandKit() {
throw isolation();
},
async listPersonRecords(_spaceId: string, ids?: string[]) {
return store.people.filter((person) => !ids || ids.includes(person.id));
},
async personCentroids() {
return store.people.flatMap((person) =>
person.centroid ? [{ personId: person.id, centroid: person.centroid }] : [],
);
},
async updatePerson(_spaceId: string, personId: string, patch: { syntheticSeed?: number }) {
const person = store.people.find((row) => row.id === personId);
if (!person) throw isolation();
if (patch.syntheticSeed !== undefined) person.syntheticSeed = patch.syntheticSeed;
return person;
},
async faceTrackRecords(_spaceId: string, assetIds: string[]) {
return store.tracks.filter((track) => assetIds.includes(track.assetId));
},
async listFaceTracks() {
return store.tracks;
},
async latestAnalysis(_spaceId: string, assetId: string, kind: string) {
return kind === "faces" && store.tracks.some((t) => t.assetId === assetId) ? { kind } : null;
},
async marksForAsset(_spaceId: string, assetId: string) {
const peopleHere = new Set(
store.tracks.filter((t) => t.assetId === assetId && t.personId).map((t) => t.personId),
);
return store.marks.filter(
(mark) => mark.assetId === assetId || (mark.personId && peopleHere.has(mark.personId)),
);
},
async createMark(
_spaceId: string,
input: Omit<MarkRow, "id" | "qualityScore" | "suggestedFallback" | "maskDir" | "createdAt">,
) {
const row: MarkRow = {
...input,
id: nextId("mark"),
qualityScore: null,
suggestedFallback: null,
maskDir: null,
createdAt: new Date(0).toISOString(),
};
store.marks.push(row);
return markDto(row);
},
async getMark(_spaceId: string, id: string) {
return { ...findMark(id) };
},
async updateMark(_spaceId: string, id: string, patch: Partial<MarkRow>) {
Object.assign(findMark(id), patch);
return markDto(findMark(id));
},
async recordMarkResult(
_spaceId: string,
id: string,
outcome: Pick<MarkRow, "qualityScore" | "suggestedFallback" | "maskDir">,
) {
Object.assign(findMark(id), outcome);
return { ...findMark(id) };
},
async removeMark(_spaceId: string, id: string) {
const row = findMark(id);
store.marks = store.marks.filter((mark) => mark.id !== id);
return { maskDir: row.maskDir };
},
async removePersonCompletely(_spaceId: string, personId: string) {
const tracks = store.tracks.filter((t) => t.personId === personId);
const marks = store.marks.filter((m) => m.personId === personId);
store.tracks = store.tracks.filter((t) => t.personId !== personId);
store.marks = store.marks.filter((m) => m.personId !== personId);
store.people = store.people.filter((p) => p.id !== personId);
return {
deletedFaceTracks: tracks.length,
deletedMarks: marks.length,
keyframesPaths: tracks.map((t) => t.keyframesPath),
maskDirs: marks.flatMap((m) => (m.maskDir ? [m.maskDir] : [])),
};
},
async getRenderRecord(_spaceId: string, id: string) {
const render = store.renders.find((row) => row.id === id);
if (!render) throw isolation();
return render;
},
async getPlan(_spaceId: string, id: string) {
const plan = store.plans.find((row) => row.id === id);
if (!plan) throw isolation();
return plan;
},
async listSegments() {
return store.segments;
},
async updateRender(_spaceId: string, id: string, patch: Record<string, unknown>) {
const render = store.renders.find((row) => row.id === id);
if (!render) return null;
Object.assign(render, patch);
return render;
},
};
const libraryDir = path.join(root, SPACE, "library");
await mkdir(libraryDir, { recursive: true });
const deps = {
repos: repos as unknown as MediaRepos,
inference: inferenceOverride ? inferenceOverride(emulator) : emulator,
mediaRoot: root,
sources: {},
now: () => new Date("2026-06-01T12:00:00Z"),
} as unknown as MediaServiceDeps;
const harness: PrivacyHarness = {
root,
inference: emulator,
deps,
store,
async addAsset(fileName, options = {}) {
const file = path.join(libraryDir, fileName);
const still = fileName.endsWith(".png");
if (still) {
await runMediaTool("ffmpeg", [
"-v",
"error",
"-y",
"-f",
"lavfi",
"-i",
"testsrc2=s=320x180:r=1:d=1",
"-frames:v",
"1",
file,
]);
} else {
await renderTestClip(file, { seconds: options.seconds ?? 2 });
}
const row: AssetRow = {
id: options.id ?? nextId("asset"),
spaceId: SPACE,
sourceId: "source-1",
externalId: fileName,
kind: still ? "photo" : "video",
storage: "uploaded",
storagePath: path.relative(root, file),
proxyPath: null,
};
store.assets.push(row);
return row;
},
addPerson(id, mode, consent = {}) {
const at = new Date("2026-01-01T00:00:00Z");
const row: PersonRow = {
id,
spaceId: SPACE,
displayName: id,
anonymizeByDefault: mode,
consentRecordedAt: consent.consent ? at : null,
ageVerifiedAt: consent.age ? at : null,
ageVerificationMethod: consent.age ? "document_on_file" : "none",
syntheticSeed: null,
centroid: null,
};
store.people.push(row);
return row;
},
async analyseFaces(assetId, personIds, sampleFps = 10) {
const asset = store.assets.find((row) => row.id === assetId)!;
const tracked = await emulator.trackFaces({
path: path.join(root, asset.storagePath!),
sampleFps,
});
const dir = path.join(root, SPACE, "faces", assetId);
await mkdir(dir, { recursive: true });
for (const [index, track] of tracked.tracks.entries()) {
const file = path.join(dir, `${track.trackId}.json`);
await writeFile(file, JSON.stringify({ keyframes: track.keyframes }));
const times = track.keyframes.map((k) => k.tMs);
store.tracks.push({
id: nextId("track"),
assetId,
personId: personIds[index] ?? null,
startMs: Math.min(...times),
endMs: Math.max(...times),
confidenceMin: 0.9,
keyframesPath: path.relative(root, file),
modelVersion: "emulator",
});
}
},
async learnPerson(personId, assetId, trackIndex = 0) {
const asset = store.assets.find((row) => row.id === assetId)!;
const tracked = await emulator.trackFaces({
path: path.join(root, asset.storagePath!),
sampleFps: 2,
});
const embedding = tracked.tracks[trackIndex]?.embedding;
const person = store.people.find((row) => row.id === personId)!;
person.centroid = embedding ? encodeVectors([embedding]) : null;
},
async workDir(name) {
const dir = path.join(root, SPACE, "renders", name);
await mkdir(dir, { recursive: true });
return dir;
},
async cleanup() {
await rm(root, { recursive: true, force: true });
},
};
return harness;
}

264
packages/adapters/src/media-privacy-io.ts Normal file
View File

@ -0,0 +1,264 @@
import { constants } from "node:fs";
import { copyFile, mkdir, readdir, readFile, realpath } from "node:fs/promises";
import path from "node:path";
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
import type {
BoxKeyframe,
MediaCapabilities,
MediaInferenceOperation,
MediaRenderQuality,
} from "@rakazo/contracts";
import type { MediaRepos } from "@rakazo/db";
import { assertInside, resolveInside } from "@rakazo/media/node";
import type { MediaServiceDeps } from "./media-context.js";
import { probeMedia, runMediaTool } from "./media-folder-source-probe.js";
import { maskBounds, readPngLuma } from "./media-privacy-pixels.js";
import type { PixelBox, TrackKeyframe } from "./media-privacy-plan.js";
import { parseStoredKeyframes } from "./media-privacy-plan.js";
/*
* File and tool helpers shared by anonymization, marks, and the gate. Stored
* media paths are relative to the media root; every resolution goes through
* the root checks of @rakazo/media/node. ffmpeg runs as an argv array only.
*/
/** A redaction or check that cannot be carried out. Rendering stops instead of exporting. */
export class PrivacyUnavailableError extends Error {
constructor(message: string) {
super(message);
this.name = "PrivacyUnavailableError";
}
}
export interface PrivacyToolOptions {
ffmpegPath?: string;
ffprobePath?: string;
}
/** Absolute path of a stored (root-relative) or already absolute path inside the media root. */
export async function resolveStored(root: string, stored: string): Promise<string> {
return path.isAbsolute(stored) ? assertInside(root, stored) : resolveInside(root, stored);
}
export async function isInsideRoot(root: string, file: string): Promise<boolean> {
try {
await assertInside(root, file);
return true;
} catch {
return false;
}
}
/** The path of a file relative to the media root, for storing. */
export async function toStored(root: string, absolute: string): Promise<string> {
const inside = await assertInside(root, absolute);
return path.relative(await realpath(root), inside);
}
type AssetRow = Awaited<ReturnType<MediaRepos["getAssetRecord"]>>;
/**
* The file a render reads for an asset: the 720p proxy for drafts (when one
* exists), otherwise the original. Originals still on a device cannot render.
*/
export async function resolveAssetFile(
deps: Pick<MediaServiceDeps, "repos" | "sources" | "mediaRoot">,
spaceId: string,
asset: AssetRow,
quality: MediaRenderQuality,
): Promise<string> {
if (quality === "draft" && asset.proxyPath) return resolveStored(deps.mediaRoot, asset.proxyPath);
return resolveOriginal(deps, spaceId, asset);
}
export async function resolveOriginal(
deps: Pick<MediaServiceDeps, "repos" | "sources" | "mediaRoot">,
spaceId: string,
asset: AssetRow,
): Promise<string> {
if (asset.storage === "uploaded") {
if (!asset.storagePath) throw new PrivacyUnavailableError(`asset ${asset.id} has no file yet`);
return resolveStored(deps.mediaRoot, asset.storagePath);
}
if (asset.storage === "referenced") {
const source = await deps.repos.getSource(spaceId, asset.sourceId);
const provider = deps.sources[source.kind as "folder" | "emulator"];
if (!provider) throw new PrivacyUnavailableError(`source of asset ${asset.id} is unavailable`);
return provider.resolve(source.config, asset.externalId);
}
throw new PrivacyUnavailableError(`asset ${asset.id} is still on a device; upload it first`);
}
/**
* A file inference can read: the proxy when it exists (normalized coordinates
* and the timeline match the original), else the original when it lies inside
* the media root. Null when neither does.
*/
export async function analysisFile(
deps: Pick<MediaServiceDeps, "repos" | "sources" | "mediaRoot">,
spaceId: string,
asset: AssetRow,
): Promise<string | null> {
if (asset.proxyPath) return resolveStored(deps.mediaRoot, asset.proxyPath);
try {
const original = await resolveOriginal(deps, spaceId, asset);
return (await isInsideRoot(deps.mediaRoot, original)) ? original : null;
} catch {
return null;
}
}
export interface ProbedSource {
width: number;
height: number;
durationMs: number;
fps: number;
hasAudio: boolean;
rotation: 0 | 90 | 180 | 270;
still: boolean;
}
/** Display size (rotation applied), duration, and audio of a source file. */
export async function probeSource(
file: string,
kind: string,
options: PrivacyToolOptions & { signal?: AbortSignal } = {},
): Promise<ProbedSource> {
const probed = await probeMedia(file, {
ffprobePath: options.ffprobePath,
signal: options.signal,
});
if (!probed.hasVideo || !probed.width || !probed.height)
throw new PrivacyUnavailableError("source has no readable video stream");
const still = kind === "photo" || !probed.durationMs;
const rotation = (((Math.round(probed.rotation / 90) * 90) % 360) + 360) % 360;
return {
width: probed.width,
height: probed.height,
durationMs: still ? 0 : (probed.durationMs ?? 0),
fps: probed.fps ?? 30,
hasAudio: probed.hasAudio,
rotation: rotation as ProbedSource["rotation"],
still,
};
}
/** Copy a file into `dir` under its own name (reflink when the filesystem allows). */
export async function copyInto(file: string, dir: string): Promise<string> {
await mkdir(dir, { recursive: true });
const target = path.join(dir, path.basename(file));
await copyFile(file, target, constants.COPYFILE_FICLONE);
return target;
}
/** Keyframes of a stored face track file (relative to the media root). */
export async function readTrackKeyframes(root: string, stored: string): Promise<TrackKeyframe[]> {
const file = await resolveStored(root, stored);
return parseStoredKeyframes(JSON.parse(await readFile(file, "utf8")));
}
/** Mask frame times of a mask directory, ascending. */
export async function maskTimes(maskDir: string): Promise<number[]> {
const names = await readdir(maskDir);
return names
.filter((name) => /^\d+\.png$/.test(name))
.map((name) => Number.parseInt(name, 10))
.sort((a, b) => a - b);
}
/** Bounding box per mask frame. Empty masks are skipped. */
export async function maskKeyframes(maskDir: string): Promise<BoxKeyframe[]> {
const keyframes: BoxKeyframe[] = [];
for (const tMs of await maskTimes(maskDir)) {
const box = maskBounds(await readPngLuma(path.join(maskDir, `${tMs}.png`)));
if (box) keyframes.push({ tMs, box });
}
return keyframes;
}
/** Whether an inference operation is available; synthetic identity also needs a commercial license. */
export function operationAvailable(
capabilities: MediaCapabilities | null,
operation: MediaInferenceOperation,
): boolean {
const entry = capabilities?.operations[operation];
if (!entry?.available) return false;
if (operation === "synthesizeIdentity") return entry.commercialUse === true;
return true;
}
export async function inferenceCapabilities(
inference: MediaInferenceProvider | null,
signal?: AbortSignal,
): Promise<MediaCapabilities | null> {
return inference ? inference.capabilities(signal) : null;
}
/**
* Lossless static crop of a video (or a still to PNG). Audio is copied; all
* container metadata is dropped.
*/
export async function cropFile(
input: string,
output: string,
crop: PixelBox,
still: boolean,
options: PrivacyToolOptions & { signal?: AbortSignal } = {},
): Promise<void> {
const filter = `crop=${crop.w}:${crop.h}:${crop.x}:${crop.y}`;
const args = still
? [
"-v",
"error",
"-y",
"-i",
input,
"-vf",
filter,
"-frames:v",
"1",
"-map_metadata",
"-1",
output,
]
: [
"-v",
"error",
"-y",
"-i",
input,
"-map",
"0:v:0",
"-map",
"0:a?",
"-vf",
filter,
"-c:v",
"libx264",
"-qp",
"0",
"-preset",
"ultrafast",
"-pix_fmt",
"yuv420p",
"-c:a",
"copy",
"-map_metadata",
"-1",
output,
];
await runMediaTool(options.ffmpegPath ?? "ffmpeg", args, { signal: options.signal });
}
/** First frame of a video as PNG (redaction providers write videos, stills render from images). */
export async function firstFrame(
input: string,
output: string,
options: PrivacyToolOptions & { signal?: AbortSignal } = {},
): Promise<void> {
await runMediaTool(
options.ffmpegPath ?? "ffmpeg",
["-v", "error", "-y", "-i", input, "-frames:v", "1", "-map_metadata", "-1", output],
{ signal: options.signal },
);
}

104
packages/adapters/src/media-privacy-pixels.test.ts Normal file
View File

@ -0,0 +1,104 @@
import { describe, expect, it } from "vitest";
import { encodeMaskPng } from "./media-emulator.js";
import type { LumaFrame } from "./media-privacy-pixels.js";
import {
controlAgreement,
decodePngLuma,
maskBounds,
residualVisibility,
temporalStability,
} from "./media-privacy-pixels.js";
/** A 64x64 checkerboard of 4 px squares: detail everywhere. */
function checker(width = 64, height = 64, shift = 0): LumaFrame {
const luma = new Uint8Array(width * height);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
luma[y * width + x] = (Math.floor((x + shift) / 4) + Math.floor(y / 4)) % 2 ? 220 : 30;
}
}
return { width, height, luma };
}
function fill(
frame: LumaFrame,
box: { x: number; y: number; w: number; h: number },
value: number,
) {
const out = { ...frame, luma: frame.luma.slice() };
for (
let y = Math.floor(box.y * frame.height);
y < Math.ceil((box.y + box.h) * frame.height);
y++
) {
for (
let x = Math.floor(box.x * frame.width);
x < Math.ceil((box.x + box.w) * frame.width);
x++
) {
out.luma[y * frame.width + x] = value;
}
}
return out;
}
const region = { x: 0.25, y: 0.25, w: 0.5, h: 0.5 };
describe("PNG masks", () => {
it("decodes the masks the segmenter writes", () => {
const mask = decodePngLuma(encodeMaskPng(40, 20, { x: 0.5, y: 0.25, w: 0.25, h: 0.5 }));
expect(mask).toMatchObject({ width: 40, height: 20 });
expect(maskBounds(mask)).toEqual({ x: 0.5, y: 0.25, w: 0.25, h: 0.5 });
expect(maskBounds(decodePngLuma(encodeMaskPng(8, 8, { x: 0, y: 0, w: 0, h: 0 })))).toBeNull();
expect(() => decodePngLuma(Buffer.from("nope"))).toThrow("not a PNG");
});
});
describe("residual visibility", () => {
it("is 1 for untouched detail and 0 for a filled or flat region", () => {
const source = checker();
expect(residualVisibility({ source, treated: source, box: region })).toBe(1);
expect(residualVisibility({ source, treated: fill(source, region, 128), box: region })).toBe(0);
const flat = fill(source, region, 90);
expect(
residualVisibility({ source: flat, treated: flat, box: { x: 0.3, y: 0.3, w: 0.4, h: 0.4 } }),
).toBe(0);
});
it("counts only the part a crop still shows and only mask pixels", () => {
const source = checker();
// The treated frame is the right half of the source, untouched.
const half: LumaFrame = { width: 32, height: 64, luma: new Uint8Array(32 * 64) };
for (let y = 0; y < 64; y++)
half.luma.set(source.luma.subarray(y * 64 + 32, y * 64 + 64), y * 32);
const left = { x: 0, y: 0.25, w: 0.4, h: 0.5 };
expect(residualVisibility({ source, treated: half, box: left, offset: { x: 32, y: 0 } })).toBe(
0,
);
expect(
residualVisibility({ source, treated: half, box: region, offset: { x: 32, y: 0 } }),
).toBe(1);
const mask = decodePngLuma(encodeMaskPng(64, 64, { x: 0.5, y: 0.25, w: 0.25, h: 0.5 }));
const treated = fill(source, { x: 0.25, y: 0.25, w: 0.25, h: 0.5 }, 128);
expect(residualVisibility({ source, treated, box: region, mask })).toBe(1);
});
it("detects frames that are not the same moment", () => {
const source = checker();
const treated = fill(source, region, 128);
expect(controlAgreement({ source, treated, boxes: [region] })).toBe(1);
expect(controlAgreement({ source, treated: checker(64, 64, 2), boxes: [region] })).toBeLessThan(
0.8,
);
});
});
describe("temporal stability", () => {
it("drops when the treated region flickers more than the frame", () => {
const a = checker();
expect(temporalStability([a, a, a], region)).toBe(1);
const flicker = [fill(a, region, 0), fill(a, region, 255), fill(a, region, 0)];
expect(temporalStability(flicker, region)).toBe(0);
expect(temporalStability([a], region)).toBe(1);
});
});

169
packages/adapters/src/media-privacy-pixels.ts
View File

@ -1,11 +1,12 @@
import { readFile } from "node:fs/promises"; import { readFile } from "node:fs/promises";
import { inflateSync } from "node:zlib"; import { inflateSync } from "node:zlib";
import type { NormalizedBox } from "@rakazo/contracts"; import type { NormalizedBox } from "@rakazo/contracts";
import { runMediaTool } from "./media-folder-source-probe.js";
/* /*
* Pixel checks for the privacy gate. Frames come from MediaRenderer.extractFrame * Pixel checks for the privacy gate and mark quality. Video frames are decoded
* as PNG; the decoder below covers the PNGs ffmpeg and the mask writers produce * by ffmpeg straight to 8-bit luma; mask PNGs from the segmenter go through the
* (8-bit gray, gray+alpha, RGB, RGBA; not interlaced, not palette). * decoder below (8-bit gray, gray+alpha, RGB, RGBA; not interlaced, not palette).
*/ */
export interface LumaFrame { export interface LumaFrame {
@ -98,7 +99,47 @@ export async function readPngLuma(file: string): Promise<LumaFrame> {
return decodePngLuma(await readFile(file)); return decodePngLuma(await readFile(file));
} }
/** Bounding box of the non-black pixels of a mask, or null when the mask is empty. */ /**
* Decode the frame shown at `tMs` as luma, scaled to `width` x `height` with
* nearest-neighbour sampling (a no-op at the display size). Throws unless
* ffmpeg returns a full frame.
*/
export async function extractLuma(
file: string,
input: { tMs: number; width: number; height: number; still?: boolean },
options: { ffmpegPath?: string; signal?: AbortSignal } = {},
): Promise<LumaFrame> {
const seek = input.still ? [] : ["-ss", (Math.max(0, input.tMs) / 1000).toFixed(3)];
const { stdout } = await runMediaTool(
options.ffmpegPath ?? "ffmpeg",
[
"-v",
"error",
...seek,
"-i",
file,
"-frames:v",
"1",
"-vf",
`scale=${input.width}:${input.height}:flags=neighbor`,
"-f",
"rawvideo",
"-pix_fmt",
"gray",
"-",
],
{ signal: options.signal },
);
const size = input.width * input.height;
if (stdout.length < size) throw new Error(`no frame at ${input.tMs} ms`);
return {
width: input.width,
height: input.height,
luma: new Uint8Array(stdout.subarray(0, size)),
};
}
/** Bounding box of the bright (>= 128) pixels of a mask, or null when the mask is empty. */
export function maskBounds(mask: LumaFrame): NormalizedBox | null { export function maskBounds(mask: LumaFrame): NormalizedBox | null {
let minX = mask.width; let minX = mask.width;
let minY = mask.height; let minY = mask.height;
@ -126,18 +167,39 @@ export function maskBounds(mask: LumaFrame): NormalizedBox | null {
export const DETAIL_MIN_STEP = 12; export const DETAIL_MIN_STEP = 12;
/** A detail pixel within this many luma levels of the source counts as unchanged. */ /** A detail pixel within this many luma levels of the source counts as unchanged. */
export const UNCHANGED_MAX_DIFF = 2; export const UNCHANGED_MAX_DIFF = 2;
/** Looser tolerance for the control area outside treated regions (codec noise). */
const CONTROL_MAX_DIFF = 6;
/** Regions with fewer detail pixels show nothing identifying. */ /** Regions with fewer detail pixels show nothing identifying. */
const MIN_DETAIL_PIXELS = 16; const MIN_DETAIL_PIXELS = 16;
function pixelRange(box: NormalizedBox, width: number, height: number) {
return {
x0: Math.max(0, Math.floor(box.x * width)),
y0: Math.max(0, Math.floor(box.y * height)),
x1: Math.min(width, Math.ceil((box.x + box.w) * width)),
y1: Math.min(height, Math.ceil((box.y + box.h) * height)),
};
}
function detailStep(frame: LumaFrame, x: number, y: number): number {
const at = y * frame.width + x;
const value = frame.luma[at] ?? 0;
const right = x + 1 < frame.width ? (frame.luma[at + 1] ?? value) : value;
const down = y + 1 < frame.height ? (frame.luma[at + frame.width] ?? value) : value;
return Math.abs(value - right) + Math.abs(value - down);
}
/** /**
* Residual visibility of a region (0..1): the share of the source's detail * Residual visibility of a region (0..1): the share of the source's detail
* pixels that the treated frame left unchanged. Blur, pixelation, fills, and * pixels (edges) that the treated frame left unchanged, value and edge
* inpainting all rewrite edges, so a treated region scores near 0, an untouched * strength alike. Blur, pixelation, fills, and inpainting all rewrite edges,
* one 1. A flat region without detail scores 0: there is nothing to recognise. * so a treated region scores near 0, an untouched one 1. A flat region
* without detail scores 0: there is nothing to recognise.
* *
* `source` and `treated` are pixel-aligned at full resolution; when `treated` * `source` and `treated` are pixel-aligned at full resolution; when `treated`
* is a crop of the source, `offset` is the crop origin in source pixels. * is a crop of the source, `offset` is the crop origin in source pixels and
* Optional `mask` (normalized over the source frame) limits the region to mask pixels. * detail outside the crop is invisible. Optional `mask` (normalized over the
* source frame) limits the region to mask pixels.
*/ */
export function residualVisibility(input: { export function residualVisibility(input: {
source: LumaFrame; source: LumaFrame;
@ -148,10 +210,7 @@ export function residualVisibility(input: {
}): number { }): number {
const { source, treated, box, mask } = input; const { source, treated, box, mask } = input;
const offset = input.offset ?? { x: 0, y: 0 }; const offset = input.offset ?? { x: 0, y: 0 };
const x0 = Math.max(0, Math.floor(box.x * source.width)); const { x0, y0, x1, y1 } = pixelRange(box, source.width, source.height);
const y0 = Math.max(0, Math.floor(box.y * source.height));
const x1 = Math.min(source.width - 1, Math.ceil((box.x + box.w) * source.width) - 1);
const y1 = Math.min(source.height - 1, Math.ceil((box.y + box.h) * source.height) - 1);
let detail = 0; let detail = 0;
let unchanged = 0; let unchanged = 0;
for (let y = y0; y < y1; y++) { for (let y = y0; y < y1; y++) {
@ -161,19 +220,91 @@ export function residualVisibility(input: {
const my = Math.min(mask.height - 1, Math.floor((y / source.height) * mask.height)); const my = Math.min(mask.height - 1, Math.floor((y / source.height) * mask.height));
if ((mask.luma[my * mask.width + mx] ?? 0) < 128) continue; if ((mask.luma[my * mask.width + mx] ?? 0) < 128) continue;
} }
const at = y * source.width + x; const step = detailStep(source, x, y);
const value = source.luma[at] ?? 0;
const step =
Math.abs(value - (source.luma[at + 1] ?? value)) +
Math.abs(value - (source.luma[at + source.width] ?? value));
if (step < DETAIL_MIN_STEP) continue; if (step < DETAIL_MIN_STEP) continue;
const tx = x - offset.x; const tx = x - offset.x;
const ty = y - offset.y; const ty = y - offset.y;
if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue; if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue;
detail += 1; detail += 1;
if (Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= UNCHANGED_MAX_DIFF) const value = source.luma[y * source.width + x] ?? 0;
// Visible means the pixel and its edge survived, not a value that happens to match.
if (
Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= UNCHANGED_MAX_DIFF &&
Math.abs(step - detailStep(treated, tx, ty)) <= 2 * UNCHANGED_MAX_DIFF
)
unchanged += 1; unchanged += 1;
} }
} }
return detail < MIN_DETAIL_PIXELS ? 0 : unchanged / detail; return detail < MIN_DETAIL_PIXELS ? 0 : unchanged / detail;
} }
/**
* Share of detail pixels outside every treated box that the treated frame kept
* (within codec tolerance). A low value means the two frames are not the same
* moment or framing, so a residual measured between them proves nothing.
* Returns 1 when the control area has too little detail to judge.
*/
export function controlAgreement(input: {
source: LumaFrame;
treated: LumaFrame;
boxes: NormalizedBox[];
offset?: { x: number; y: number };
}): number {
const { source, treated, boxes } = input;
const offset = input.offset ?? { x: 0, y: 0 };
const ranges = boxes.map((box) => pixelRange(box, source.width, source.height));
let detail = 0;
let kept = 0;
for (let y = 0; y < source.height; y++) {
for (let x = 0; x < source.width; x++) {
const tx = x - offset.x;
const ty = y - offset.y;
if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue;
if (ranges.some((r) => x >= r.x0 - 2 && x < r.x1 + 2 && y >= r.y0 - 2 && y < r.y1 + 2))
continue;
if (detailStep(source, x, y) < DETAIL_MIN_STEP) continue;
detail += 1;
const value = source.luma[y * source.width + x] ?? 0;
if (Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= CONTROL_MAX_DIFF)
kept += 1;
}
}
return detail < MIN_DETAIL_PIXELS ? 1 : kept / detail;
}
/**
* Temporal stability of a treated region over consecutive frames (0..1):
* 1 when the region changes no more between neighbours than the rest of the
* frame, falling to 0 as it flickers 24+ luma levels more on average.
*/
export function temporalStability(frames: LumaFrame[], box: NormalizedBox): number {
if (frames.length < 2) return 1;
let excess = 0;
for (let i = 1; i < frames.length; i++) {
const a = frames[i - 1]!;
const b = frames[i]!;
if (a.width !== b.width || a.height !== b.height) throw new Error("frame sizes differ");
const r = pixelRange(box, a.width, a.height);
let inside = 0;
let insideCount = 0;
let outside = 0;
let outsideCount = 0;
for (let y = 0; y < a.height; y++) {
for (let x = 0; x < a.width; x++) {
const at = y * a.width + x;
const diff = Math.abs((a.luma[at] ?? 0) - (b.luma[at] ?? 0));
if (x >= r.x0 && x < r.x1 && y >= r.y0 && y < r.y1) {
inside += diff;
insideCount += 1;
} else {
outside += diff;
outsideCount += 1;
}
}
}
const meanInside = insideCount ? inside / insideCount : 0;
const meanOutside = outsideCount ? outside / outsideCount : 0;
excess += Math.max(0, meanInside - meanOutside);
}
return Math.max(0, Math.min(1, 1 - excess / (frames.length - 1) / 24));
}

210
packages/adapters/src/media-privacy-plan.test.ts Normal file
View File

@ -0,0 +1,210 @@
import type { FaceRedaction } from "@rakazo/contracts";
import { encodeVectors } from "@rakazo/media";
import { describe, expect, it } from "vitest";
import { mergeTracks, syntheticSeedFor } from "./media-anonymize.js";
import { testEdl } from "./media-privacy-fixtures.js";
import {
boxAt,
cropPixels,
effectiveModes,
ensureMinSize,
GAP_HOLD_GROWTH,
matchPerson,
outsideCrop,
padBox,
parseStoredKeyframes,
protectKeyframes,
resolveProtection,
sourceToOutput,
} from "./media-privacy-plan.js";
const box = { x: 0.4, y: 0.4, w: 0.1, h: 0.1 };
function face(
id: string,
target: FaceRedaction["target"],
mode: FaceRedaction["mode"],
): FaceRedaction {
return { id, target, mode, paddingPct: 20, featherPct: 10, minSizePct: 2 };
}
describe("protectKeyframes", () => {
it("interpolates short gaps and holds a grown box over long ones for review", () => {
const result = protectKeyframes(
[
{ tMs: 0, box },
{ tMs: 400, box: { ...box, x: 0.44 } },
{ tMs: 1400, box: { ...box, x: 0.5 } },
],
[{ startMs: 0, endMs: 2000 }],
2,
);
expect(result.gaps).toEqual([{ startMs: 400, endMs: 1400 }]);
expect(result.keyframes.map((k) => k.tMs)).toEqual([0, 100, 200, 300, 400, 401, 1399, 1400]);
const held = result.keyframes.find((k) => k.tMs === 401)!.box;
expect(held.w).toBeCloseTo(0.1 * (1 + 2 * GAP_HOLD_GROWTH), 3);
});
it("treats weak detections as lost but still covers them", () => {
const result = protectKeyframes(
[
{ tMs: 0, box, confidence: 0.9 },
{ tMs: 400, box: { ...box, x: 0.8 }, confidence: 0.2 },
{ tMs: 800, box, confidence: 0.9 },
],
[{ startMs: 0, endMs: 1000 }],
2,
);
expect(result.gaps).toEqual([{ startMs: 0, endMs: 800 }]);
const held = result.keyframes.find((k) => k.tMs === 1)!.box;
expect(held.x + held.w).toBeGreaterThanOrEqual(0.9);
});
it("ignores detections far outside the clip ranges and enforces a minimum size", () => {
const result = protectKeyframes(
[
{ tMs: 0, box: { x: 0.5, y: 0.5, w: 0.001, h: 0.001 } },
{ tMs: 5000, box },
],
[{ startMs: 0, endMs: 100 }],
5,
);
expect(result.keyframes).toHaveLength(1);
expect(result.keyframes[0]!.box.w).toBeGreaterThanOrEqual(0.05);
expect(ensureMinSize({ x: 0.99, y: 0.99, w: 0.01, h: 0.01 }, 10)).toEqual({
x: 0.9,
y: 0.9,
w: 0.1,
h: 0.1,
});
});
});
describe("protection", () => {
const defaults = new Map([
["alice", "pixelate" as const],
["bob", "none" as const],
]);
const faces = [
face("by-track", { trackId: "t1" }, "solid"),
face("by-person", { personId: "bob" }, "partial_eyes"),
face("strangers", { unassignedFaces: true }, "pixelate"),
];
it("prefers track over person over default over unassigned", () => {
const resolve = (storedId: string | null, personId: string | null, protectUnassigned = false) =>
resolveProtection({ storedId, personId }, { faces, defaults, protectUnassigned })?.mode;
expect(resolve("t1", "alice")).toBe("solid");
expect(resolve("t2", "bob")).toBe("partial_eyes");
expect(resolve("t2", "alice")).toBe("pixelate");
expect(resolve(null, null)).toBe("pixelate");
expect(
resolveProtection(
{ storedId: null, personId: null },
{ faces: [], defaults, protectUnassigned: true },
),
).toMatchObject({ mode: "blur", origin: "unassigned" });
expect(
resolveProtection(
{ storedId: null, personId: null },
{ faces: [], defaults, protectUnassigned: false },
),
).toBeNull();
expect(effectiveModes(faces, defaults).get("bob")).toBe("partial_eyes");
});
it("matches faces to people only above the similarity threshold", () => {
const centroids = [
{ personId: "alice", centroid: encodeVectors([[1, 0, 0]]) },
{ personId: "bob", centroid: encodeVectors([[0, 1, 0]]) },
];
expect(matchPerson([0.9, 0.1, 0], centroids)).toBe("alice");
expect(matchPerson([0.5, 0.5, 0.7], centroids)).toBeNull();
expect(matchPerson(null, centroids)).toBeNull();
});
it("merges fresh tracks into stored ones and keeps what the fresh pass missed", () => {
const stored = [
{ id: "s1", storedId: "s1", personId: "alice", keyframes: [{ tMs: 0, box }] },
{ id: "s2", storedId: "s2", personId: null, keyframes: [{ tMs: 0, box: { ...box, x: 0 } }] },
];
const merged = mergeTracks(
"a1",
stored,
[
{
keyframes: [
{ tMs: 0, box, confidence: 0.9 },
{ tMs: 100, box, confidence: 0.9 },
],
embedding: null,
},
{ keyframes: [{ tMs: 0, box: { ...box, y: 0.8 }, confidence: 0.9 }], embedding: [1, 0, 0] },
],
[{ personId: "carol", centroid: encodeVectors([[1, 0, 0]]) }],
);
expect(merged.map((t) => [t.id, t.personId, t.keyframes.length])).toEqual([
["s1", "alice", 3],
["fresh-a1-2", "carol", 1],
["s2", null, 1],
]);
expect(syntheticSeedFor("alice")).toBe(syntheticSeedFor("alice"));
expect(syntheticSeedFor("alice")).not.toBe(syntheticSeedFor("bob"));
});
});
describe("geometry", () => {
it("crops with the output aspect and knows what lies outside", () => {
const geometry = { sourceWidth: 1920, sourceHeight: 1080, outWidth: 1080, outHeight: 1920 };
const crop = cropPixels(geometry, { cx: 0.8, cy: 0.5, scale: 1 });
expect(crop).toEqual({ x: 1232, y: 0, w: 608, h: 1080 });
expect(outsideCrop({ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }, crop, 1920, 1080)).toBe(true);
expect(outsideCrop({ x: 0.6, y: 0.1, w: 0.2, h: 0.2 }, crop, 1920, 1080)).toBe(false);
expect(padBox(box, 20)).toEqual({
x: 0.39,
y: 0.39,
w: expect.closeTo(0.12),
h: expect.closeTo(0.12),
});
});
it("maps source spans onto the output timeline", () => {
const edl = testEdl([
{ id: "a", assetId: "x", inMs: 1000, outMs: 3000 },
{ id: "b", assetId: "y", inMs: 0, outMs: 1000 },
]);
const [a, b] = edl.tracks.video;
expect(sourceToOutput(edl, a!, { startMs: 1500, endMs: 5000 })).toEqual({
startMs: 500,
endMs: 2000,
});
expect(sourceToOutput(edl, b!, { startMs: 200, endMs: 300 })).toEqual({
startMs: 2200,
endMs: 2300,
});
expect(sourceToOutput(edl, a!, { startMs: 0, endMs: 500 })).toBeNull();
});
it("finds the box between keyframes", () => {
const keyframes = [
{ tMs: 0, box },
{ tMs: 100, box: { ...box, x: 0.5 } },
];
expect(boxAt(keyframes, 50)).toEqual({
x: 0.4,
y: 0.4,
w: expect.closeTo(0.2),
h: expect.closeTo(0.1),
});
expect(boxAt(keyframes, 150)).toEqual({ ...box, x: 0.5 });
expect(boxAt(keyframes, 500)).toBeNull();
});
it("reads stored keyframe files in both shapes", () => {
expect(parseStoredKeyframes([{ tMs: 0, box, extra: 1 }])).toEqual([{ tMs: 0, box }]);
expect(parseStoredKeyframes({ keyframes: [{ tMs: 5, box, confidence: 0.4 }] })).toEqual([
{ tMs: 5, box, confidence: 0.4 },
]);
expect(() => parseStoredKeyframes({ frames: [] })).toThrow();
});
});

328
packages/adapters/src/media-privacy-plan.ts
View File

@ -8,22 +8,18 @@ import type {
FaceRedaction, FaceRedaction,
NormalizedBox, NormalizedBox,
} from "@rakazo/contracts"; } from "@rakazo/contracts";
import { import { AiAlterationSchema, AnonymizeModeSchema, BoxKeyframeSchema } from "@rakazo/contracts";
AiAlterationSchema, import { cosineSimilarity, decodeVectors, introMs, MAX_TRACKING_GAP_MS } from "@rakazo/media";
AnonymizeModeSchema,
BoxKeyframeSchema,
MarkTreatmentSchema,
} from "@rakazo/contracts";
import { cosineSimilarity, decodeVectors, MAX_TRACKING_GAP_MS } from "@rakazo/media";
import { z } from "zod"; import { z } from "zod";
/* /*
* Pure planning helpers for face, mark, and region protection, plus the * Pure planning helpers for face, mark, and region protection, plus the
* privacy record that prepareSources leaves next to the render output for the * privacy record that prepareSources leaves next to the render output for the
* gate. Times are source times of the asset unless a name says otherwise. * gate. Times are source times of the asset unless a name says otherwise;
* boxes are normalized to the displayed (rotation-applied) frame.
*/ */
/** Tracking rate for the re-detection pass on clip sources (faces the stored analysis missed). */ /** Tracking rate of the fresh detection pass on clip sources. */
export const FACE_TRACK_FPS = 10; export const FACE_TRACK_FPS = 10;
/** /**
* Minimum cosine similarity between a face embedding and a person centroid to * Minimum cosine similarity between a face embedding and a person centroid to
@ -32,12 +28,20 @@ export const FACE_TRACK_FPS = 10;
* appears in the project. * appears in the project.
*/ */
export const FACE_MATCH_MIN_COSINE = 0.6; export const FACE_MATCH_MIN_COSINE = 0.6;
/** Detections below this confidence count as lost (spec 7: hold and grow the last mask). */
export const FACE_MIN_CONFIDENCE = 0.5;
/** A fresh track inherits a stored track's person when their boxes overlap this much on average. */
export const TRACK_MATCH_MIN_IOU = 0.3;
/** Step for linear interpolation across short detection gaps (<= MAX_TRACKING_GAP_MS). */ /** Step for linear interpolation across short detection gaps (<= MAX_TRACKING_GAP_MS). */
export const INTERPOLATE_STEP_MS = 100; export const INTERPOLATE_STEP_MS = 100;
/** Longer gaps hold the last box, enlarged by this share on every side (spec 7). */ /** Lost detections hold the last box, enlarged by this share on every side (spec 7). */
export const GAP_HOLD_GROWTH = 0.25; export const GAP_HOLD_GROWTH = 0.25;
/** Padding around marks and regions, in percent of the box. */ /** Padding around marks and regions, in percent of the box. */
export const MARK_PADDING_PCT = 10; export const MARK_PADDING_PCT = 10;
/** Person-bound marks re-found below this segmentation score go to review and block the gate. */
export const MARK_REVIEW_MIN_SCORE = 0.6;
/** Treatment quality below this suggests (and, while rendering, applies) a fallback. */
export const MARK_QUALITY_MIN = 0.7;
/** Name of the privacy record prepareSources writes into the render work directory. */ /** Name of the privacy record prepareSources writes into the render work directory. */
export const PRIVACY_RECORD_FILE = "privacy.json"; export const PRIVACY_RECORD_FILE = "privacy.json";
@ -54,11 +58,15 @@ export interface Protection {
origin: string; origin: string;
} }
export type TrackKeyframe = BoxKeyframe & { confidence?: number };
export interface CandidateTrack { export interface CandidateTrack {
/** Stored track id, or null for faces found by the re-detection pass. */ /** Stable id: the stored track id, or `fresh-<n>` for faces only the fresh pass found. */
id: string;
/** Stored track id when the face matches an analysed track, for EDL track targets. */
storedId: string | null; storedId: string | null;
personId: string | null; personId: string | null;
keyframes: BoxKeyframe[]; keyframes: TrackKeyframe[];
} }
export interface TimeWindow { export interface TimeWindow {
@ -84,8 +92,9 @@ function fromRedaction(redaction: FaceRedaction): Protection {
/** /**
* Protection for one face track. Most specific wins: EDL track target, EDL * Protection for one face track. Most specific wins: EDL track target, EDL
* person target, EDL unassigned target, the person's default, and finally a * person target, the person's default, then the EDL unassigned target, and
* blur for unassigned faces while any protected person appears in the project. * finally a blur for unassigned faces while any protected person appears in
* the project.
*/ */
export function resolveProtection( export function resolveProtection(
track: Pick<CandidateTrack, "storedId" | "personId">, track: Pick<CandidateTrack, "storedId" | "personId">,
@ -95,10 +104,12 @@ export function resolveProtection(
protectUnassigned: boolean; protectUnassigned: boolean;
}, },
): Protection | null { ): Protection | null {
const byTrack = context.faces.find( if (track.storedId) {
(face) => "trackId" in face.target && face.target.trackId === track.storedId, const byTrack = context.faces.find(
); (face) => "trackId" in face.target && face.target.trackId === track.storedId,
if (byTrack && track.storedId) return fromRedaction(byTrack); );
if (byTrack) return fromRedaction(byTrack);
}
if (track.personId) { if (track.personId) {
const byPerson = context.faces.find( const byPerson = context.faces.find(
(face) => "personId" in face.target && face.target.personId === track.personId, (face) => "personId" in face.target && face.target.personId === track.personId,
@ -110,10 +121,23 @@ export function resolveProtection(
} }
const unassigned = context.faces.find((face) => "unassignedFaces" in face.target); const unassigned = context.faces.find((face) => "unassignedFaces" in face.target);
if (unassigned) return fromRedaction(unassigned); if (unassigned) return fromRedaction(unassigned);
if (context.protectUnassigned) return { mode: "blur", ...DEFAULT_PROTECTION, origin: "unassigned" }; if (context.protectUnassigned)
return { mode: "blur", ...DEFAULT_PROTECTION, origin: "unassigned" };
return null; return null;
} }
/** Effective mode per person: an EDL person target wins over the person's default. */
export function effectiveModes(
faces: FaceRedaction[],
defaults: Map<string, AnonymizeMode>,
): Map<string, AnonymizeMode> {
const modes = new Map(defaults);
for (const face of faces) {
if ("personId" in face.target) modes.set(face.target.personId, face.mode);
}
return modes;
}
/** Best person for an embedding by cosine to the centroids, or null below the threshold. */ /** Best person for an embedding by cosine to the centroids, or null below the threshold. */
export function matchPerson( export function matchPerson(
embedding: ArrayLike<number> | null, embedding: ArrayLike<number> | null,
@ -145,15 +169,20 @@ export function growBox(box: NormalizedBox, share: number): NormalizedBox {
}; };
} }
/** Padding as the redaction providers apply it: paddingPct / 2 of the size on each side. */
export function padBox(box: NormalizedBox, paddingPct: number): NormalizedBox {
return growBox(box, Math.max(0, paddingPct) / 200);
}
/** Enlarge a box around its centre until both sides reach `minSizePct` of the frame. */ /** Enlarge a box around its centre until both sides reach `minSizePct` of the frame. */
export function ensureMinSize(box: NormalizedBox, minSizePct: number): NormalizedBox { export function ensureMinSize(box: NormalizedBox, minSizePct: number): NormalizedBox {
const min = minSizePct / 100; const min = minSizePct / 100;
if (box.w >= min && box.h >= min) return box; if (box.w >= min && box.h >= min) return box;
const w = Math.max(box.w, min); const w = Math.min(1, Math.max(box.w, min));
const h = Math.max(box.h, min); const h = Math.min(1, Math.max(box.h, min));
const x = clampUnit(box.x + box.w / 2 - w / 2); const x = clampUnit(Math.min(box.x + box.w / 2 - w / 2, 1 - w));
const y = clampUnit(box.y + box.h / 2 - h / 2); const y = clampUnit(Math.min(box.y + box.h / 2 - h / 2, 1 - h));
return { x, y, w: Math.min(w, 1 - x), h: Math.min(h, 1 - y) }; return { x, y, w, h };
} }
export function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox { export function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox {
@ -167,8 +196,12 @@ export function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox {
}; };
} }
export function padBox(box: NormalizedBox, paddingPct: number): NormalizedBox { export function iou(a: NormalizedBox, b: NormalizedBox): number {
return growBox(box, Math.max(0, paddingPct) / 200); const w = Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x);
const h = Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y);
if (w <= 0 || h <= 0) return 0;
const inter = w * h;
return inter / (a.w * a.h + b.w * b.h - inter);
} }
function lerpBox(a: NormalizedBox, b: NormalizedBox, f: number): NormalizedBox { function lerpBox(a: NormalizedBox, b: NormalizedBox, f: number): NormalizedBox {
@ -176,67 +209,116 @@ function lerpBox(a: NormalizedBox, b: NormalizedBox, f: number): NormalizedBox {
return { x: mix(a.x, b.x), y: mix(a.y, b.y), w: mix(a.w, b.w), h: mix(a.h, b.h) }; return { x: mix(a.x, b.x), y: mix(a.y, b.y), w: mix(a.w, b.w), h: mix(a.h, b.h) };
} }
const round4 = (value: number) => Math.round(value * 10_000) / 10_000; /** Round outward so a rounded box never covers less than the exact one. */
const roundBox = (box: NormalizedBox): NormalizedBox => ({ export function roundBoxOut(box: NormalizedBox): NormalizedBox {
x: round4(box.x), const x = Math.floor(box.x * 10_000) / 10_000;
y: round4(box.y), const y = Math.floor(box.y * 10_000) / 10_000;
w: round4(box.w), return {
h: round4(box.h), x,
}); y,
w: Math.min(1 - x, Math.ceil((box.x + box.w - x) * 10_000) / 10_000),
h: Math.min(1 - y, Math.ceil((box.y + box.h - y) * 10_000) / 10_000),
};
}
function nearWindows(tMs: number, windows: TimeWindow[]): boolean {
return windows.some(
(w) => tMs >= w.startMs - MAX_TRACKING_GAP_MS && tMs <= w.endMs + MAX_TRACKING_GAP_MS,
);
}
/** /**
* Keyframes for redaction inside the windows (clip ranges plus the bridging * Keyframes for redaction near the windows (clip ranges plus the bridging
* limit on both sides). Gaps up to MAX_TRACKING_GAP_MS are interpolated; * limit on both sides). Detections below FACE_MIN_CONFIDENCE count as lost.
* longer ones hold the last box grown by GAP_HOLD_GROWTH until the face is * Gaps up to MAX_TRACKING_GAP_MS between confident detections are
* found again and are returned so the gate blocks them (needs review). * interpolated; longer ones hold the last confident box grown by
* GAP_HOLD_GROWTH (unioned with any weak detections) until the face is found
* again, and are returned so the gate blocks them as needing review.
*/ */
export function protectKeyframes( export function protectKeyframes(
keyframes: BoxKeyframe[], keyframes: TrackKeyframe[],
windows: TimeWindow[], windows: TimeWindow[],
minSizePct: number, minSizePct: number,
): { keyframes: BoxKeyframe[]; gaps: TimeWindow[] } { ): { keyframes: BoxKeyframe[]; gaps: TimeWindow[] } {
const inside = [...keyframes] const inside = [...keyframes]
.filter((k) => .filter((k) => nearWindows(k.tMs, windows))
windows.some(
(w) => k.tMs >= w.startMs - MAX_TRACKING_GAP_MS && k.tMs <= w.endMs + MAX_TRACKING_GAP_MS,
),
)
.sort((a, b) => a.tMs - b.tMs) .sort((a, b) => a.tMs - b.tMs)
.map((k) => ({ tMs: k.tMs, box: ensureMinSize(k.box, minSizePct) })); .map((k) => ({
tMs: k.tMs,
box: ensureMinSize(k.box, minSizePct),
confident: (k.confidence ?? 1) >= FACE_MIN_CONFIDENCE,
}));
const out: BoxKeyframe[] = []; const out: BoxKeyframe[] = [];
const gaps: TimeWindow[] = []; const gaps: TimeWindow[] = [];
inside.forEach((current, index) => { const confident = inside.filter((k) => k.confident);
out.push({ tMs: current.tMs, box: roundBox(current.box) }); const weak = inside.filter((k) => !k.confident);
const next = inside[index + 1]; if (confident.length === 0) {
// Only weak detections: cover them grown and send the whole span to review.
for (const k of weak)
out.push({ tMs: k.tMs, box: roundBoxOut(growBox(k.box, GAP_HOLD_GROWTH)) });
const first = weak[0];
const last = weak.at(-1);
if (first && last) gaps.push({ startMs: first.tMs, endMs: Math.max(last.tMs, first.tMs + 1) });
return { keyframes: out, gaps };
}
confident.forEach((current, index) => {
out.push({ tMs: current.tMs, box: roundBoxOut(current.box) });
const next = confident[index + 1];
if (!next) return; if (!next) return;
const span = next.tMs - current.tMs; const span = next.tMs - current.tMs;
const between = weak.filter((k) => k.tMs > current.tMs && k.tMs < next.tMs);
if (span <= MAX_TRACKING_GAP_MS) { if (span <= MAX_TRACKING_GAP_MS) {
for (let t = current.tMs + INTERPOLATE_STEP_MS; t < next.tMs; t += INTERPOLATE_STEP_MS) { for (let t = current.tMs + INTERPOLATE_STEP_MS; t < next.tMs; t += INTERPOLATE_STEP_MS) {
out.push({ let box = lerpBox(current.box, next.box, (t - current.tMs) / span);
tMs: t, for (const w of between)
box: roundBox(lerpBox(current.box, next.box, (t - current.tMs) / span)), if (Math.abs(w.tMs - t) < INTERPOLATE_STEP_MS) box = unionBox(box, w.box);
}); out.push({ tMs: t, box: roundBoxOut(box) });
} }
return; return;
} }
const held = roundBox(growBox(current.box, GAP_HOLD_GROWTH)); let held = growBox(current.box, GAP_HOLD_GROWTH);
out.push({ tMs: current.tMs + 1, box: held }, { tMs: next.tMs - 1, box: held }); for (const w of between) held = unionBox(held, w.box);
const box = roundBoxOut(held);
out.push({ tMs: current.tMs + 1, box }, { tMs: next.tMs - 1, box });
gaps.push({ startMs: current.tMs, endMs: next.tMs }); gaps.push({ startMs: current.tMs, endMs: next.tMs });
}); });
return { keyframes: out, gaps }; return { keyframes: out, gaps };
} }
/** Box a keyframe list covers at `tMs`: the union of the bracketing keyframes, or null outside. */
export function boxAt(keyframes: BoxKeyframe[], tMs: number, holdMs = 100): NormalizedBox | null {
let before: BoxKeyframe | null = null;
let after: BoxKeyframe | null = null;
for (const k of keyframes) {
if (k.tMs <= tMs && (!before || k.tMs > before.tMs)) before = k;
if (k.tMs >= tMs && (!after || k.tMs < after.tMs)) after = k;
}
if (before && after) return unionBox(before.box, after.box);
if (before && tMs - before.tMs <= holdMs) return before.box;
return null;
}
/** Mean IoU of two tracks over the times both observe (nearest keyframe within 60 ms). */
export function trackOverlap(a: BoxKeyframe[], b: BoxKeyframe[]): number {
const scores: number[] = [];
for (const k of a) {
let nearest: BoxKeyframe | null = null;
for (const other of b) {
if (Math.abs(other.tMs - k.tMs) > 60) continue;
if (!nearest || Math.abs(other.tMs - k.tMs) < Math.abs(nearest.tMs - k.tMs)) nearest = other;
}
if (nearest) scores.push(iou(k.box, nearest.box));
}
return scores.length === 0 ? 0 : scores.reduce((sum, s) => sum + s, 0) / scores.length;
}
/** Source windows each clip of an asset shows. */ /** Source windows each clip of an asset shows. */
export function clipWindows(clips: EdlVideoClip[]): TimeWindow[] { export function clipWindows(clips: EdlVideoClip[]): TimeWindow[] {
return clips.map((clip) => ({ startMs: clip.inMs, endMs: clip.outMs })); return clips.map((clip) => ({ startMs: clip.inMs, endMs: clip.outMs }));
} }
export function overlapsWindows(keyframes: BoxKeyframe[], windows: TimeWindow[]): boolean { export function overlapsWindows(span: TimeWindow, windows: TimeWindow[]): boolean {
return keyframes.some((k) => return windows.some((w) => span.startMs <= w.endMs && w.startMs <= span.endMs);
windows.some(
(w) => k.tMs >= w.startMs - MAX_TRACKING_GAP_MS && k.tMs <= w.endMs + MAX_TRACKING_GAP_MS,
),
);
} }
/** /**
@ -255,11 +337,21 @@ export function partialCheckBand(box: NormalizedBox, mode: PartialMode): Normali
}; };
} }
/** Crop window in source pixels with the output aspect, as the compiler sizes it. */ export interface PixelBox {
x: number;
y: number;
w: number;
h: number;
}
/**
* Static crop window in display pixels with the output aspect, as the
* compiler sizes it for a reframe keyframe. Even sizes and offsets for yuv420p.
*/
export function cropPixels( export function cropPixels(
geometry: { sourceWidth: number; sourceHeight: number; outWidth: number; outHeight: number }, geometry: { sourceWidth: number; sourceHeight: number; outWidth: number; outHeight: number },
keyframe: { cx: number; cy: number; scale: number }, keyframe: { cx: number; cy: number; scale: number },
): { x: number; y: number; w: number; h: number } { ): PixelBox {
const outAspect = geometry.outWidth / geometry.outHeight; const outAspect = geometry.outWidth / geometry.outHeight;
let w: number; let w: number;
let h: number; let h: number;
@ -272,11 +364,10 @@ export function cropPixels(
} }
const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2); const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2);
const even = (v: number) => Math.max(0, Math.floor(v / 2) * 2); const even = (v: number) => Math.max(0, Math.floor(v / 2) * 2);
const cw = Math.min(geometry.sourceWidth - (geometry.sourceWidth % 2), roundEven(w / keyframe.scale)); const maxW = geometry.sourceWidth - (geometry.sourceWidth % 2);
const ch = Math.min( const maxH = geometry.sourceHeight - (geometry.sourceHeight % 2);
geometry.sourceHeight - (geometry.sourceHeight % 2), const cw = Math.min(maxW, roundEven(w / keyframe.scale));
roundEven(h / keyframe.scale), const ch = Math.min(maxH, roundEven(h / keyframe.scale));
);
const x = even( const x = even(
Math.min(Math.max(0, keyframe.cx * geometry.sourceWidth - cw / 2), geometry.sourceWidth - cw), Math.min(Math.max(0, keyframe.cx * geometry.sourceWidth - cw / 2), geometry.sourceWidth - cw),
); );
@ -286,17 +377,22 @@ export function cropPixels(
return { x, y, w: cw, h: ch }; return { x, y, w: cw, h: ch };
} }
/** Map source gaps of one clip onto the output timeline (intro included). */ /** Whether a normalized box lies completely outside a pixel crop. */
export function gapToOutput( export function outsideCrop(box: NormalizedBox, crop: PixelBox, width: number, height: number) {
edl: Edl, const x0 = box.x * width;
clip: EdlVideoClip, const y0 = box.y * height;
gap: TimeWindow, const x1 = (box.x + box.w) * width;
): { startMs: number; endMs: number } | null { const y1 = (box.y + box.h) * height;
const start = Math.max(gap.startMs, clip.inMs); return x1 <= crop.x || x0 >= crop.x + crop.w || y1 <= crop.y || y0 >= crop.y + crop.h;
const end = Math.min(gap.endMs, clip.outMs); }
if (end <= start) return null;
const intro = edl.intro?.durationMs ?? 0; /** Map a source window of one clip onto the output timeline (intro included), or null. */
const toOutput = (t: number) => Math.round(intro + clip.atMs + (t - clip.inMs) / clip.speed); export function sourceToOutput(edl: Edl, clip: EdlVideoClip, span: TimeWindow): TimeWindow | null {
const start = Math.max(span.startMs, clip.inMs);
const end = Math.min(span.endMs, clip.outMs);
if (end < start) return null;
const toOutput = (t: number) =>
Math.round(introMs(edl) + clip.atMs + (t - clip.inMs) / clip.speed);
return { startMs: toOutput(start), endMs: toOutput(end) }; return { startMs: toOutput(start), endMs: toOutput(end) };
} }
@ -311,45 +407,55 @@ const PixelBoxSchema = z.object({
h: z.number().int().min(1), h: z.number().int().min(1),
}); });
const SpanSchema = { startMs: z.number(), endMs: z.number() };
/** One protected region the gate verifies pixel by pixel. */
export const PrivacyCheckSchema = z.object({
kind: z.enum(["face", "mark", "region"]),
/** Track, mark, or region id. */
id: z.string(),
personId: z.string().nullable(),
markId: z.string().nullable(),
/** Boxes the treatment promised to alter, normalized to the base frame. */
keyframes: z.array(BoxKeyframeSchema),
/** Absolute mask directory (`<ms>.png` per frame) or null for box-only regions. */
maskDir: z.string().nullable(),
});
export type PrivacyCheck = z.infer<typeof PrivacyCheckSchema>;
export const PrivacyRecordSchema = z.object({ export const PrivacyRecordSchema = z.object({
version: z.literal(1), version: z.literal(1),
assets: z.record( assets: z.record(
z.string(), z.string(),
z.object({ z.object({
/** Unredacted file the prepared source derives from (pixel reference). */
basePath: z.string(), basePath: z.string(),
preparedPath: z.string(), preparedPath: z.string(),
width: z.number().int().positive(),
height: z.number().int().positive(),
still: z.boolean(), still: z.boolean(),
/** Static crop applied for reframe_out, in base display pixels. */ /** Static crop applied for reframe_out, in base display pixels. */
crop: PixelBoxSchema.nullable(), crop: PixelBoxSchema.nullable(),
marks: z.array( checks: z.array(PrivacyCheckSchema),
z.object({
markId: z.string(),
treatment: MarkTreatmentSchema,
/** Absolute mask directory (`<ms>.png` per frame) or null for box-only regions. */
maskDir: z.string().nullable(),
keyframes: z.array(BoxKeyframeSchema),
}),
),
partialFaces: z.array(
z.object({
personId: z.string().nullable(),
mode: z.enum(["partial_eyes", "partial_upper", "partial_lower"]),
keyframes: z.array(BoxKeyframeSchema),
}),
),
}), }),
), ),
/** Source-time gaps over MAX_TRACKING_GAP_MS per clip. */ /** Detection gaps over MAX_TRACKING_GAP_MS per asset (source time), for review. */
trackingGaps: z.array( trackingGaps: z.array(
z.object({ z.object({ assetId: z.string(), personId: z.string().nullable(), ...SpanSchema }),
clipId: z.string(),
startMs: z.number(),
endMs: z.number(),
personId: z.string().nullable(),
}),
), ),
/** Person-bound marks re-found with low confidence, for review. */ /** Person-bound marks re-found with low confidence, for review. */
reviews: z.array(z.object({ markId: z.string(), assetId: z.string(), score: z.number() })), reviews: z.array(z.object({ markId: z.string(), assetId: z.string(), score: z.number() })),
/** Faces that stay detectable by design (synthetic identity, partial modes), source time. */
expectedFaces: z.array(
z.object({
assetId: z.string(),
personId: z.string().nullable(),
mode: AnonymizeModeSchema,
...SpanSchema,
}),
),
/** Faces in the clip ranges that belong to no person (consent cannot be shown). */
unassignedFaces: z.array(z.object({ assetId: z.string(), ...SpanSchema })),
aiAlterations: z.array(AiAlterationSchema), aiAlterations: z.array(AiAlterationSchema),
/** Effective protection per person (EDL target over person default). */ /** Effective protection per person (EDL target over person default). */
protectedPeople: z.record(z.string(), AnonymizeModeSchema), protectedPeople: z.record(z.string(), AnonymizeModeSchema),
@ -363,29 +469,31 @@ export async function writePrivacyRecord(workDir: string, record: PrivacyRecord)
await writeFile(path.join(workDir, PRIVACY_RECORD_FILE), JSON.stringify(record)); await writeFile(path.join(workDir, PRIVACY_RECORD_FILE), JSON.stringify(record));
} }
export async function readPrivacyRecord(workDir: string): Promise<PrivacyRecord> { /** The record prepareSources wrote, or null when it is missing. Malformed records throw. */
export async function readPrivacyRecord(workDir: string): Promise<PrivacyRecord | null> {
let text: string; let text: string;
try { try {
text = await readFile(path.join(workDir, PRIVACY_RECORD_FILE), "utf8"); text = await readFile(path.join(workDir, PRIVACY_RECORD_FILE), "utf8");
} catch (error) { } catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
throw new Error(
"privacy record is missing: sources must be prepared in the render's work directory",
);
}
throw error; throw error;
} }
return PrivacyRecordSchema.parse(JSON.parse(text)); return PrivacyRecordSchema.parse(JSON.parse(text));
} }
/** Stored face keyframe files: an array of keyframes or a tracking result with `keyframes`. */ /** Stored face keyframe files: an array of keyframes or a tracking result with `keyframes`. */
const StoredKeyframeSchema = BoxKeyframeSchema.extend({ confidence: z.number().optional() });
const StoredKeyframesSchema = z.union([ const StoredKeyframesSchema = z.union([
z.array(BoxKeyframeSchema.passthrough()), z.array(StoredKeyframeSchema.passthrough()),
z.object({ keyframes: z.array(BoxKeyframeSchema.passthrough()) }).passthrough(), z.object({ keyframes: z.array(StoredKeyframeSchema.passthrough()) }).passthrough(),
]); ]);
export function parseStoredKeyframes(json: unknown): BoxKeyframe[] { export function parseStoredKeyframes(json: unknown): TrackKeyframe[] {
const parsed = StoredKeyframesSchema.parse(json); const parsed = StoredKeyframesSchema.parse(json);
const list = Array.isArray(parsed) ? parsed : parsed.keyframes; const list = Array.isArray(parsed) ? parsed : parsed.keyframes;
return list.map((k) => ({ tMs: k.tMs, box: k.box })); return list.map((k) => ({
tMs: k.tMs,
box: k.box,
...(k.confidence !== undefined ? { confidence: k.confidence } : {}),
}));
} }

39
packages/adapters/src/media-privacy.ts Normal file
View File

@ -0,0 +1,39 @@
import { rm } from "node:fs/promises";
import path from "node:path";
import { prepareSources } from "./media-anonymize.js";
import type { MediaPrivacyService, MediaServiceDeps } from "./media-context.js";
import { runGate } from "./media-gate.js";
import { createMark, updateMark } from "./media-marks.js";
import type { PrivacyToolOptions } from "./media-privacy-io.js";
import { resolveStored } from "./media-privacy-io.js";
/*
* MediaPrivacyService: anonymization before rendering, the gate after it,
* identifying marks, and deleting a person with their biometric data. Face
* data never leaves this host: matching, tracking, and clustering run against
* the local inference provider and database only.
*/
export function createMediaPrivacyService(
deps: MediaServiceDeps,
tools: PrivacyToolOptions = {},
): MediaPrivacyService {
const withTools = { ...deps, ...tools };
return {
prepareSources: (input) => prepareSources(withTools, input),
gate: (input) => runGate(withTools, input),
createMark: (actor, input) => createMark(withTools, actor, input),
updateMark: (actor, input) => updateMark(withTools, actor, input),
async removePerson(actor, personId) {
const removed = await deps.repos.removePersonCompletely(actor.spaceId, personId);
// Rows are gone; now the files: track keyframes and each mark's directory
// (masks plus previews). A path outside the media root throws untouched.
const markDirs = removed.maskDirs.map((dir) =>
path.basename(dir) === "masks" ? path.dirname(dir) : dir,
);
for (const stored of [...removed.keyframesPaths, ...markDirs]) {
await rm(await resolveStored(deps.mediaRoot, stored), { recursive: true, force: true });
}
},
};
}