cynarAI/evOo-local
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

merge: W2 privacy

Browse Source
This commit is contained in:
Claude 2026-10-03 18:57:49 +00:00
commit 90f18cea52
No known key found for this signature in database
17 changed files with 5122 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

424
packages/adapters/src/media-anonymize.test.ts Normal file
View File

@ -0,0 +1,424 @@
import { copyFile } from "node:fs/promises";
import path from "node:path";
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
import type { AnonymizeMode, Edl } from "@rakazo/contracts";
import { afterEach, describe, expect, it, vi } from "vitest";
import { prepareSources } from "./media-anonymize.js";
import { ffmpegAvailable } from "./media-emulator-clips.js";
import { runGate } from "./media-gate.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import {
ACTOR,
cleanProbe,
createPrivacyHarness,
testEdl,
testProfile,
} from "./media-privacy-fixtures.js";
import { PrivacyUnavailableError } from "./media-privacy-io.js";
import { extractLuma, residualVisibility } from "./media-privacy-pixels.js";
import { readPrivacyRecord } from "./media-privacy-plan.js";
const hasFfmpeg = ffmpegAvailable();
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
/** One asset with one face of person "alice" (stored track), one clip over it. */
async function setup(
mode: AnonymizeMode,
options: {
fileName?: string;
edl?: (assetId: string) => Edl;
inference?: (emulator: PrivacyHarness["inference"]) => MediaInferenceProvider | null;
} = {},
) {
harness = await createPrivacyHarness(options.inference);
const asset = await harness.addAsset(options.fileName ?? "clip__faces-1____person-alice__.mp4");
harness.addPerson("alice", mode);
await harness.analyseFaces(asset.id, ["alice"]);
await harness.learnPerson("alice", asset.id);
const edl =
options.edl?.(asset.id) ?? testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
const workDir = await harness.workDir(`render-${mode}`);
return { h: harness, asset, edl, workDir };
}
async function gateOn(
h: PrivacyHarness,
input: { edl: Edl; workDir: string; prepared: string; outputName?: string },
profile = testProfile(),
) {
// The compiler is not under test: the prepared source stands in for the render output.
const outputPath = path.join(input.workDir, input.outputName ?? "output.mp4");
await copyFile(input.prepared, outputPath);
return runGate(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl: input.edl,
profile,
outputPath,
ffprobe: cleanProbe(input.edl),
copy: { title: null, caption: null, tags: [] },
bundleHasDisclosure: false,
autoFixRound: 0,
});
}
describe.skipIf(!hasFfmpeg)("prepareSources", () => {
it.each([
"blur",
"pixelate",
"solid",
"sticker",
"partial_eyes",
"partial_upper",
"partial_lower",
] as const)("applies %s to a protected person and the gate verifies it", async (mode) => {
const { h, asset, edl, workDir } = await setup(mode);
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const source = prepared.sources.get(asset.id);
expect(source?.path).toContain(`${path.sep}privacy${path.sep}`);
expect(source).toMatchObject({ width: 320, height: 180, rotation: 0, still: false });
const record = await readPrivacyRecord(workDir);
expect(record?.protectedPeople).toEqual({ alice: mode });
const check = record?.assets[asset.id]?.checks[0];
expect(check).toMatchObject({ kind: "face", personId: "alice" });
// The protected box changed, the rest of the frame did not.
const base = await extractLuma(path.join(h.root, asset.storagePath!), {
tMs: 1000,
width: 320,
height: 180,
});
const treated = await extractLuma(source!.path, { tMs: 1000, width: 320, height: 180 });
const box = check!.keyframes.find((k) => k.tMs >= 1000)!.box;
expect(residualVisibility({ source: base, treated, box })).toBeLessThan(0.05);
expect(
residualVisibility({ source: base, treated, box: { x: 0.6, y: 0.6, w: 0.35, h: 0.35 } }),
).toBeGreaterThan(0.95);
const report = await gateOn(h, { edl, workDir, prepared: source!.path });
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
expect(report.passed).toBe(true);
});
it("crops a reframe_out face out of frame", async () => {
const { h, asset, workDir } = await setup("reframe_out");
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
width: 180,
height: 320,
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const source = prepared.sources.get(asset.id)!;
expect(prepared.fallbacks).toEqual([]);
expect(source.width / source.height).toBeCloseTo(180 / 320, 1);
const record = await readPrivacyRecord(workDir);
expect(record?.assets[asset.id]?.crop).not.toBeNull();
const report = await gateOn(h, { edl, workDir, prepared: source.path });
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
});
it("falls back from reframe_out to blur and says so when no crop avoids the face", async () => {
const { h, asset, edl, workDir } = await setup("none", {
edl: (assetId) =>
testEdl([{ id: "c1", assetId, inMs: 0, outMs: 2000 }], {
redactions: {
faces: [
{
id: "big",
target: { personId: "alice" },
mode: "reframe_out",
paddingPct: 100,
featherPct: 0,
minSizePct: 50,
},
],
marks: [],
regions: [],
},
}),
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.fallbacks).toEqual([
"person alice: no crop keeps the face out of frame; blurred instead",
]);
const source = prepared.sources.get(asset.id)!;
expect(source.width).toBe(320);
const report = await gateOn(h, { edl, workDir, prepared: source.path });
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
});
it("blocks the export when tracking loses the face for more than 500 ms", async () => {
const { h, asset, edl, workDir } = await setup("blur", {
fileName: "clip__faces-1____person-alice____noface-gap-600-1400__.mp4",
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const record = await readPrivacyRecord(workDir);
expect(record?.trackingGaps).toEqual([
expect.objectContaining({ assetId: asset.id, personId: "alice" }),
]);
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "tracking_gap", severity: "block", personId: "alice" }),
);
});
it("falls back to blur when synthetic identity is unavailable and never fakes it", async () => {
const { h, asset, edl, workDir } = await setup("synthetic_identity");
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.fallbacks).toEqual([
"person alice: synthetic identity is not available; blurred instead",
]);
expect(prepared.aiAlterations).not.toContain("synthetic_identity");
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.passed).toBe(true);
});
it("uses a licensed synthetic identity with a stable seed and no target image", async () => {
const synthesize = vi.fn();
const { h, asset, edl, workDir } = await setup("synthetic_identity", {
inference: (emulator) => {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.capabilities = async () => {
const caps = await emulator.capabilities();
caps.operations.synthesizeIdentity = {
available: true,
experimental: true,
model: "test-synth",
license: "Apache-2.0",
commercialUse: true,
};
return caps;
};
provider.synthesizeIdentity = async (request) => {
synthesize(request);
// Stand-in synthesis: a solid fill over the track (real models paint a face).
return emulator.redact({
path: request.path,
outPath: request.outPath,
redactions: [
{ keyframes: request.track, mode: "solid", paddingPct: 20, featherPct: 0 },
],
});
};
return provider;
},
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.aiAlterations).toContain("synthetic_identity");
expect(synthesize).toHaveBeenCalledTimes(1);
const request = synthesize.mock.calls[0]![0] as Record<string, unknown>;
expect(Object.keys(request).sort()).toEqual(["outPath", "path", "seed", "signal", "track"]);
const seed = h.store.people[0]!.syntheticSeed;
expect(request.seed).toBe(seed);
expect(seed).not.toBeNull();
const record = await readPrivacyRecord(workDir);
expect(record?.expectedFaces).toEqual([
expect.objectContaining({ personId: "alice", mode: "synthetic_identity" }),
]);
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.passed).toBe(true);
});
it("fails closed without a media worker", async () => {
const { h, edl, workDir } = await setup("blur", { inference: () => null });
await expect(
prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
}),
).rejects.toBeInstanceOf(PrivacyUnavailableError);
});
it("returns the untouched source when nothing needs protection", async () => {
const { h, asset, edl, workDir } = await setup("none");
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.sources.get(asset.id)?.path).toBe(path.join(h.root, asset.storagePath!));
expect(prepared.aiAlterations).toEqual([]);
});
it("blurs unassigned faces next to a protected person and records voice disguise", async () => {
const { h, asset, workDir } = await setup("blur", {
fileName: "clip__faces-2____person-alice__.mp4",
});
// Track 2 stays unassigned (analysis stored it without a person).
h.store.tracks[1]!.personId = null;
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
redactions: { faces: [], marks: [], regions: [], voice: { method: "pitch", semitones: 3 } },
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.aiAlterations).toEqual(["voice_disguise"]);
const record = await readPrivacyRecord(workDir);
expect(record?.protectUnassigned).toBe(true);
expect(record?.assets[asset.id]?.checks.map((c) => c.personId)).toEqual(["alice", null]);
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
});
it("protects a photo and renders it from a redacted still", async () => {
const { h, asset, edl, workDir } = await setup("pixelate", {
fileName: "photo__faces-1____person-alice__.png",
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const source = prepared.sources.get(asset.id)!;
expect(source).toMatchObject({ still: true, durationMs: 0, width: 320, height: 180 });
expect(source.path.endsWith(".png")).toBe(true);
const record = await readPrivacyRecord(workDir);
const box = record!.assets[asset.id]!.checks[0]!.keyframes[0]!.box;
const base = await extractLuma(path.join(h.root, asset.storagePath!), {
tMs: 0,
width: 320,
height: 180,
still: true,
});
const treated = await extractLuma(source.path, {
tMs: 0,
width: 320,
height: 180,
still: true,
});
expect(residualVisibility({ source: base, treated, box })).toBeLessThan(0.05);
});
it("applies EDL regions and covers one that no crop can avoid", async () => {
const { h, asset, workDir } = await setup("none");
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
redactions: {
faces: [],
marks: [],
regions: [
{
id: "plate",
clipId: "c1",
keyframes: [{ tMs: 0, box: { x: 0.7, y: 0.7, w: 0.2, h: 0.2 } }],
treatment: "blur",
},
{
id: "everything",
clipId: "c1",
keyframes: [{ tMs: 0, box: { x: 0, y: 0, w: 1, h: 1 } }],
treatment: "reframe_out",
},
],
},
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
expect(prepared.fallbacks).toEqual([
"region everything: no crop keeps it out of frame; covered instead",
]);
const record = await readPrivacyRecord(workDir);
expect(record?.assets[asset.id]?.checks.map((c) => c.id)).toEqual(["plate", "everything"]);
});
it("blocks when a provider claims to redact but leaves the face visible", async () => {
const { h, asset, edl, workDir } = await setup("blur", {
inference: (emulator) => {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.redact = async (request) => {
await copyFile(request.path, request.outPath);
return { outPath: request.outPath, framesProcessed: 50 };
};
return provider;
},
});
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const report = await gateOn(h, {
edl,
workDir,
prepared: prepared.sources.get(asset.id)!.path,
});
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "face_not_covered", personId: "alice" }),
);
});
});

928
packages/adapters/src/media-anonymize.ts Normal file
View File

@ -0,0 +1,928 @@
import { createHash } from "node:crypto";
import { mkdir, rm } from "node:fs/promises";
import path from "node:path";
import type { RedactionInstruction } from "@rakazo/adapter-kit";
import type {
AiAlteration,
AnonymizeMode,
BoxKeyframe,
Edl,
EdlVideoClip,
MarkTreatment,
MediaCapabilities,
MediaRenderQuality,
NormalizedBox,
NormalizedPoint,
} from "@rakazo/contracts";
import type { MediaRepos } from "@rakazo/db";
import type { CompileSource } from "@rakazo/media";
import { MAX_TRACKING_GAP_MS, reframeExcluding } from "@rakazo/media";
import type { MediaActor, MediaPreparedSources, MediaServiceDeps } from "./media-context.js";
import {
anchorFromFace,
faceAt,
firstConfident,
placeAnchor,
promptBox,
segmentMark,
} from "./media-marks.js";
import type { PrivacyToolOptions, ProbedSource } from "./media-privacy-io.js";
import {
analysisFile,
copyInto,
cropFile,
firstFrame,
inferenceCapabilities,
isInsideRoot,
maskKeyframes,
operationAvailable,
PrivacyUnavailableError,
probeSource,
readTrackKeyframes,
resolveAssetFile,
resolveStored,
} from "./media-privacy-io.js";
import type {
CandidateTrack,
PixelBox,
PrivacyCheck,
PrivacyRecord,
Protection,
TimeWindow,
} from "./media-privacy-plan.js";
import {
boxAt,
clipWindows,
cropPixels,
effectiveModes,
FACE_TRACK_FPS,
iou,
isPartial,
MARK_PADDING_PCT,
MARK_QUALITY_MIN,
MARK_REVIEW_MIN_SCORE,
matchPerson,
outsideCrop,
overlapsWindows,
PRIVACY_RECORD_FILE,
padBox,
partialCheckBand,
protectKeyframes,
resolveProtection,
TRACK_MATCH_MIN_IOU,
trackOverlap,
unionBox,
writePrivacyRecord,
} from "./media-privacy-plan.js";
/*
* prepareSources (spec 7): resolve every clip's file and apply all face, mark,
* and region protection before the compiler sees it. The compiler applies no
* redactions, so anything this function cannot honor throws
* (PrivacyUnavailableError) instead of returning an unredacted source.
*
* Per asset, in this order: synthetic identities, inpainted marks, one
* redaction pass (blur, pixelate, solid, sticker, partial bands, covers) by the
* inference provider, then a static crop for reframe_out. Faces are found from
* the stored tracks plus a fresh tracking pass; tracking gaps over 500 ms are
* covered by the grown last box and recorded so the gate blocks them. Voice
* disguise is applied by the compiler on the program audio and only recorded
* here. Everything the gate must verify goes into privacy.json in workDir.
*/
export type AnonymizeDeps = Pick<
MediaServiceDeps,
"repos" | "inference" | "mediaRoot" | "sources"
> &
PrivacyToolOptions;
type AssetRow = Awaited<ReturnType<MediaRepos["getAssetRecord"]>>;
type PersonRow = Awaited<ReturnType<MediaRepos["listPersonRecords"]>>[number];
/** reframe_out boxes are unioned per bucket of this length so the crop search stays small. */
const REFRAME_BUCKET_MS = 1000;
/** Landmark samples per partial-mode track. */
const MAX_LANDMARK_SAMPLES = 120;
interface AssetPlan {
asset: AssetRow;
clips: EdlVideoClip[];
windows: TimeWindow[];
base: string;
probe: ProbedSource;
dir: string;
candidates: CandidateTrack[];
instructions: RedactionInstruction[];
checks: PrivacyCheck[];
synthetic: Array<{ personId: string | null; keyframes: BoxKeyframe[]; seed: number }>;
inpaint: Array<{ markId: string; maskDir: string; keyframes: BoxKeyframe[] }>;
reframe: Array<{
check: PrivacyCheck;
/** Boxes the crop must exclude. */
boxes: Array<{ tMs: number; box: NormalizedBox }>;
/** Applied when no crop can exclude the boxes. */
fallback: () => void;
}>;
}
export interface PrepareInput {
actor: MediaActor;
projectId: string;
edl: Edl;
quality: MediaRenderQuality;
workDir: string;
signal?: AbortSignal;
}
export async function prepareSources(
deps: AnonymizeDeps,
input: PrepareInput,
): Promise<MediaPreparedSources> {
const { actor, edl, signal } = input;
const spaceId = actor.spaceId;
const workDir = await resolveStored(deps.mediaRoot, input.workDir);
await mkdir(workDir, { recursive: true });
const privacyDir = path.join(workDir, "privacy");
await rm(privacyDir, { recursive: true, force: true });
// A record from an earlier attempt must never vouch for this one.
await rm(path.join(workDir, PRIVACY_RECORD_FILE), { force: true });
const clipsByAsset = new Map<string, EdlVideoClip[]>();
for (const clip of edl.tracks.video) {
clipsByAsset.set(clip.assetId, [...(clipsByAsset.get(clip.assetId) ?? []), clip]);
}
const assetIds = [...clipsByAsset.keys()].sort();
const assets = await deps.repos.listAssetRecords(spaceId, assetIds);
if (assets.length !== assetIds.length)
throw new PrivacyUnavailableError("a clip asset is missing");
const project = await deps.repos.getProjectRecord(spaceId, input.projectId);
const kit = project.brandKitId
? (await deps.repos.getBrandKit(spaceId, project.brandKitId)).kit
: null;
const people = await deps.repos.listPersonRecords(spaceId);
const peopleById = new Map(people.map((person) => [person.id, person]));
const defaults = new Map(
people.map((person) => [person.id, person.anonymizeByDefault as AnonymizeMode]),
);
const modes = effectiveModes(edl.redactions.faces, defaults);
const faceWork =
edl.redactions.faces.length > 0 || [...modes.values()].some((mode) => mode !== "none");
const capabilities = await inferenceCapabilities(deps.inference, signal);
const stored = await deps.repos.faceTrackRecords(spaceId, assetIds);
const centroids = faceWork ? await deps.repos.personCentroids(spaceId) : [];
const fallbacks: string[] = [];
const alterations = new Set<AiAlteration>();
if (edl.redactions.voice) alterations.add("voice_disguise");
const record: PrivacyRecord = {
version: 1,
assets: {},
trackingGaps: [],
reviews: [],
expectedFaces: [],
unassignedFaces: [],
aiAlterations: [],
protectedPeople: {},
visiblePeople: [],
protectUnassigned: false,
};
// 1. Files and face candidates per asset.
const plans: AssetPlan[] = [];
for (const asset of assets) {
signal?.throwIfAborted();
const clips = clipsByAsset.get(asset.id) ?? [];
const base = await resolveAssetFile(deps, spaceId, asset, input.quality);
const probe = await probeSource(base, asset.kind, { ...deps, signal });
const windows = probe.still ? [{ startMs: 0, endMs: 0 }] : clipWindows(clips);
const plan: AssetPlan = {
asset,
clips,
windows,
base,
probe,
dir: path.join(privacyDir, asset.id),
candidates: [],
instructions: [],
checks: [],
synthetic: [],
inpaint: [],
reframe: [],
};
const storedTracks = stored.filter((track) => track.assetId === asset.id);
const fromStore: CandidateTrack[] = [];
for (const track of storedTracks) {
fromStore.push({
id: track.id,
storedId: track.id,
personId: track.personId,
keyframes: await readTrackKeyframes(deps.mediaRoot, track.keyframesPath),
});
}
if (faceWork) {
if (deps.inference && operationAvailable(capabilities, "trackFaces")) {
const file = (await analysisFile(deps, spaceId, asset)) ?? (await workingCopy(deps, plan));
const fresh = await deps.inference.trackFaces({
path: file,
sampleFps: FACE_TRACK_FPS,
signal,
});
plan.candidates = mergeTracks(asset.id, fromStore, fresh.tracks, centroids);
} else {
const analysed = await deps.repos.latestAnalysis(spaceId, asset.id, "faces");
if (!analysed) {
throw new PrivacyUnavailableError(
`faces of asset ${asset.id} were never detected and the media worker is unavailable`,
);
}
plan.candidates = fromStore;
}
} else {
plan.candidates = fromStore;
}
plan.candidates = plan.candidates.filter((track) => visibleIn(track, plan));
plans.push(plan);
}
// 2. Who is visible and who is protected.
const visible = new Set<string>();
for (const plan of plans) {
for (const track of plan.candidates) {
if (track.personId) visible.add(track.personId);
else {
const span = spanOf(track);
if (span) record.unassignedFaces.push({ assetId: plan.asset.id, ...span });
}
}
}
record.visiblePeople = [...visible].sort();
for (const [personId, mode] of modes) {
if (mode !== "none" && visible.has(personId)) record.protectedPeople[personId] = mode;
}
record.protectUnassigned = Object.keys(record.protectedPeople).length > 0;
const colors = {
fill: kit?.colors.primary,
sticker: kit?.colors.accent ?? kit?.colors.primary,
};
// 3. Faces.
const syntheticReady = operationAvailable(capabilities, "synthesizeIdentity");
for (const plan of plans) {
for (const track of plan.candidates) {
const protection = resolveProtection(track, {
faces: edl.redactions.faces,
defaults,
protectUnassigned: record.protectUnassigned,
});
if (!protection) continue;
const protectedTrack = protectKeyframes(track.keyframes, plan.windows, protection.minSizePct);
const keyframes = plan.probe.still
? stillKeyframes(protectedTrack.keyframes)
: protectedTrack.keyframes;
if (keyframes.length === 0) continue;
if (!plan.probe.still) {
for (const gap of protectedTrack.gaps) {
record.trackingGaps.push({ assetId: plan.asset.id, personId: track.personId, ...gap });
}
}
await planFace(deps, plan, {
track,
protection,
keyframes,
person: track.personId ? (peopleById.get(track.personId) ?? null) : null,
syntheticReady,
capabilities,
colors,
fallbacks,
record,
signal,
});
}
}
// 4. Marks (asset marks and the standing marks of people seen in the asset).
const edlMarks = new Map(edl.redactions.marks.map((mark) => [mark.markId, mark.treatment]));
for (const plan of plans) {
const marks = await deps.repos.marksForAsset(spaceId, plan.asset.id);
for (const mark of marks) {
const treatment: MarkTreatment = edlMarks.get(mark.id) ?? mark.treatment;
if (treatment === "none") continue;
const located = await locateMark(deps, plan, mark, signal);
if (!located) continue;
if (located.review) {
record.reviews.push({ markId: mark.id, assetId: plan.asset.id, score: located.score });
}
const keyframes = plan.probe.still
? stillKeyframes(located.keyframes)
: located.keyframes.filter((k) => nearWindows(k.tMs, plan.windows));
if (keyframes.length === 0) continue;
const check: PrivacyCheck = {
kind: "mark",
id: mark.id,
personId: mark.personId,
markId: mark.id,
keyframes,
maskDir: located.maskDir,
};
const cover = () => {
plan.instructions.push({
keyframes,
mode: "cover",
paddingPct: MARK_PADDING_PCT,
featherPct: 0,
maskDir: located.maskDir,
...(colors.fill ? { color: colors.fill } : {}),
});
plan.checks.push(check);
};
if (treatment === "blur") {
plan.instructions.push({
keyframes,
mode: "blur",
paddingPct: MARK_PADDING_PCT,
featherPct: 0,
maskDir: located.maskDir,
});
plan.checks.push(check);
} else if (treatment === "cover") {
cover();
} else if (treatment === "reframe_out") {
plan.reframe.push({
check,
boxes: keyframes.map((k) => ({ tMs: k.tMs, box: padBox(k.box, MARK_PADDING_PCT) })),
fallback: () => {
fallbacks.push(`mark ${mark.id}: no crop keeps it out of frame; covered instead`);
cover();
},
});
} else if (operationAvailable(capabilities, "inpaintVideo")) {
plan.inpaint.push({ markId: mark.id, maskDir: located.maskDir, keyframes });
plan.checks.push(check);
} else {
fallbacks.push(`mark ${mark.id}: inpainting is unavailable; covered instead`);
cover();
}
}
}
// 5. Regions drawn in the EDL.
for (const region of edl.redactions.regions) {
const clip = edl.tracks.video.find((candidate) => candidate.id === region.clipId);
const plan = plans.find((candidate) => candidate.asset.id === clip?.assetId);
if (!clip || !plan) throw new PrivacyUnavailableError(`region ${region.id} has no clip`);
const keyframes = plan.probe.still ? stillKeyframes(region.keyframes) : region.keyframes;
const check: PrivacyCheck = {
kind: "region",
id: region.id,
personId: null,
markId: null,
keyframes,
maskDir: null,
};
const apply = (mode: "blur" | "cover") => {
plan.instructions.push({
keyframes,
mode,
paddingPct: 0,
featherPct: 0,
...(mode === "cover" && colors.fill ? { color: colors.fill } : {}),
});
plan.checks.push(check);
};
if (region.treatment === "reframe_out") {
plan.reframe.push({
check,
boxes: keyframes,
fallback: () => {
fallbacks.push(`region ${region.id}: no crop keeps it out of frame; covered instead`);
apply("cover");
},
});
} else {
apply(region.treatment);
}
}
// 6. Apply per asset.
const sources = new Map<string, CompileSource>();
for (const plan of plans) {
signal?.throwIfAborted();
const crop = chooseCrop(plan, edl);
const hasWork =
plan.synthetic.length > 0 ||
plan.inpaint.length > 0 ||
plan.instructions.length > 0 ||
crop !== null;
if (!hasWork) {
sources.set(plan.asset.id, compileSource(plan.base, plan.probe, plan.probe.rotation));
continue;
}
const prepared = await applyPlan(deps, plan, crop, {
capabilities,
alterations,
fallbacks,
signal,
});
const probed = await probeSource(prepared, plan.asset.kind, { ...deps, signal });
verifyPrepared(plan, crop, probed);
sources.set(plan.asset.id, compileSource(prepared, probed, 0));
record.assets[plan.asset.id] = {
basePath: plan.base,
preparedPath: prepared,
width: plan.probe.width,
height: plan.probe.height,
still: plan.probe.still,
crop,
checks: plan.checks,
};
}
record.aiAlterations = [...alterations].sort();
await writePrivacyRecord(workDir, record);
return { sources, aiAlterations: record.aiAlterations, fallbacks };
}
// ---------------------------------------------------------------------------
// Faces
// ---------------------------------------------------------------------------
/**
* Fresh tracks take over stored ones they overlap (keeping the stored id and
* person); unmatched fresh tracks are matched to people by embedding; stored
* tracks the fresh pass missed stay (fail closed).
*/
export function mergeTracks(
assetId: string,
stored: CandidateTrack[],
fresh: Array<{
keyframes: Array<BoxKeyframe & { confidence: number }>;
embedding: number[] | null;
}>,
centroids: Array<{ personId: string; centroid: Uint8Array }>,
): CandidateTrack[] {
const taken = new Set<string>();
const merged: CandidateTrack[] = [];
fresh.forEach((track, index) => {
let best: { candidate: CandidateTrack; score: number } | null = null;
for (const candidate of stored) {
if (taken.has(candidate.id)) continue;
const score = trackOverlap(track.keyframes, candidate.keyframes);
if (score >= TRACK_MATCH_MIN_IOU && (!best || score > best.score))
best = { candidate, score };
}
if (best) {
taken.add(best.candidate.id);
merged.push({
...best.candidate,
personId: best.candidate.personId ?? matchPerson(track.embedding, centroids),
keyframes: [...track.keyframes, ...best.candidate.keyframes].sort((a, b) => a.tMs - b.tMs),
});
return;
}
merged.push({
id: `fresh-${assetId}-${index + 1}`,
storedId: null,
personId: matchPerson(track.embedding, centroids),
keyframes: track.keyframes,
});
});
for (const candidate of stored) if (!taken.has(candidate.id)) merged.push(candidate);
return merged;
}
function nearWindows(tMs: number, windows: TimeWindow[]): boolean {
return windows.some(
(w) => tMs >= w.startMs - MAX_TRACKING_GAP_MS && tMs <= w.endMs + MAX_TRACKING_GAP_MS,
);
}
function spanOf(track: CandidateTrack): TimeWindow | null {
if (track.keyframes.length === 0) return null;
const times = track.keyframes.map((k) => k.tMs);
return { startMs: Math.min(...times), endMs: Math.max(...times) };
}
function visibleIn(track: CandidateTrack, plan: AssetPlan): boolean {
if (plan.probe.still) return track.keyframes.length > 0;
const span = spanOf(track);
return span !== null && overlapsWindows(span, plan.windows);
}
/** A still shows one moment: every box at once, held over the whole image. */
function stillKeyframes(keyframes: BoxKeyframe[]): BoxKeyframe[] {
const first = keyframes[0];
if (!first) return [];
const box = keyframes.reduce((acc, k) => unionBox(acc, k.box), first.box);
return [
{ tMs: 0, box },
{ tMs: 1000, box },
];
}
/** Stable per-person seed for a synthetic identity (stored on first use). */
export function syntheticSeedFor(id: string): number {
return createHash("sha256").update(`synthetic:${id}`).digest().readInt32LE(0);
}
async function planFace(
deps: AnonymizeDeps,
plan: AssetPlan,
input: {
track: CandidateTrack;
protection: Protection;
keyframes: BoxKeyframe[];
person: PersonRow | null;
syntheticReady: boolean;
capabilities: MediaCapabilities | null;
colors: { fill?: string; sticker?: string };
fallbacks: string[];
record: PrivacyRecord;
signal?: AbortSignal;
},
): Promise<void> {
const { track, protection, keyframes } = input;
const who = track.personId ? `person ${track.personId}` : `face ${track.id}`;
let mode = protection.mode;
const padded = keyframes.map((k) => ({ tMs: k.tMs, box: padBox(k.box, protection.paddingPct) }));
const check: PrivacyCheck = {
kind: "face",
id: track.id,
personId: track.personId,
markId: null,
keyframes: padded,
maskDir: null,
};
const span = spanOf({ ...track, keyframes });
if (mode === "synthetic_identity") {
if (input.syntheticReady) {
let seed: number;
if (input.person) {
seed = input.person.syntheticSeed ?? syntheticSeedFor(input.person.id);
if (input.person.syntheticSeed === null) {
await deps.repos.updatePerson(plan.asset.spaceId, input.person.id, {
syntheticSeed: seed,
});
input.person.syntheticSeed = seed;
}
} else {
seed = syntheticSeedFor(track.id);
}
plan.synthetic.push({ personId: track.personId, keyframes, seed });
plan.checks.push(check);
if (span) {
input.record.expectedFaces.push({
assetId: plan.asset.id,
personId: track.personId,
mode,
...span,
});
}
return;
}
input.fallbacks.push(`${who}: synthetic identity is not available; blurred instead`);
mode = "blur";
}
if (mode === "reframe_out") {
plan.reframe.push({
check,
boxes: padded,
fallback: () => {
input.fallbacks.push(`${who}: no crop keeps the face out of frame; blurred instead`);
plan.instructions.push({
keyframes,
mode: "blur",
paddingPct: protection.paddingPct,
featherPct: protection.featherPct,
});
plan.checks.push(check);
},
});
return;
}
const instruction: RedactionInstruction = {
keyframes,
mode,
paddingPct: protection.paddingPct,
featherPct: protection.featherPct,
};
if (mode === "solid" && input.colors.fill) instruction.color = input.colors.fill;
if (mode === "sticker" && input.colors.sticker) instruction.color = input.colors.sticker;
if (isPartial(mode)) {
const partial = mode;
const landmarks = await landmarksFor(deps, plan, keyframes, input.capabilities, input.signal);
if (landmarks.length > 0) instruction.landmarks = landmarks;
check.keyframes = padded.map((k) => ({ tMs: k.tMs, box: partialCheckBand(k.box, partial) }));
if (span) {
input.record.expectedFaces.push({
assetId: plan.asset.id,
personId: track.personId,
mode,
...span,
});
}
}
plan.instructions.push(instruction);
plan.checks.push(check);
}
/** Five-point landmarks per sampled keyframe for partial modes, when the provider has them. */
async function landmarksFor(
deps: AnonymizeDeps,
plan: AssetPlan,
keyframes: BoxKeyframe[],
capabilities: MediaCapabilities | null,
signal?: AbortSignal,
): Promise<Array<{ tMs: number; points: NormalizedPoint[] }>> {
if (!deps.inference || !operationAvailable(capabilities, "faceLandmarks")) return [];
const step = Math.max(1, Math.ceil(keyframes.length / MAX_LANDMARK_SAMPLES));
const timesMs = keyframes.filter((_, i) => i % step === 0).map((k) => k.tMs);
const file =
(await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan));
const result = await deps.inference.faceLandmarks({ path: file, timesMs, signal });
const out: Array<{ tMs: number; points: NormalizedPoint[] }> = [];
for (const frame of result.frames) {
const expected = boxAt(keyframes, frame.tMs);
if (!expected) continue;
const best = frame.faces
.map((face) => ({ face, score: iou(face.box, expected) }))
.sort((a, b) => b.score - a.score)[0];
if (best && best.score > 0.2) out.push({ tMs: frame.tMs, points: best.face.landmarks });
}
return out;
}
// ---------------------------------------------------------------------------
// Marks
// ---------------------------------------------------------------------------
type MarkRow = Awaited<ReturnType<MediaRepos["marksForAsset"]>>[number];
/**
* Mask and keyframes of a mark in this asset. Marks drawn on the asset use
* their stored mask (segmenting now when none exists); standing person marks
* are re-prompted at the same place relative to the person's face. Low
* segmentation confidence sends the mark to review.
*/
async function locateMark(
deps: AnonymizeDeps,
plan: AssetPlan,
mark: MarkRow,
signal?: AbortSignal,
): Promise<{ maskDir: string; keyframes: BoxKeyframe[]; score: number; review: boolean } | null> {
const outDir = path.join(plan.dir, `mark-${mark.id}`);
if (mark.assetId === plan.asset.id) {
if (mark.maskDir) {
const maskDir = await resolveStored(deps.mediaRoot, mark.maskDir);
return { maskDir, keyframes: await maskKeyframes(maskDir), score: 1, review: false };
}
if (!mark.prompt) throw new PrivacyUnavailableError(`mark ${mark.id} has no prompt`);
const file =
(await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan));
const segmented = await segmentMark(deps, { file, prompt: mark.prompt, outDir, signal });
return {
maskDir: segmented.maskDir,
keyframes: segmented.keyframes,
score: segmented.score,
review: false,
};
}
// A standing mark of a person seen in this asset.
if (!mark.personId) return null;
const tracks = plan.candidates.filter((track) => track.personId === mark.personId);
if (tracks.length === 0) return null;
const anchorFace = await originFace(deps, plan.asset.spaceId, mark);
const target = tracks
.map((track) =>
firstConfident(track.keyframes, Math.min(...plan.windows.map((w) => w.startMs))),
)
.filter((k) => k !== null)
.sort((a, b) => a.tMs - b.tMs)[0];
const placed =
anchorFace && target && mark.prompt
? placeAnchor(anchorFromFace(promptBox(mark.prompt), anchorFace), target.box)
: null;
if (!placed || !target) {
// Cannot relate the mark to the face here: cover the person's face region and review.
const keyframes = tracks.flatMap((track) =>
track.keyframes.map((k) => ({ tMs: k.tMs, box: k.box })),
);
return { maskDir: await emptyMaskDir(outDir), keyframes, score: 0, review: true };
}
const file =
(await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan));
const segmented = await segmentMark(deps, {
file,
prompt: { tMs: target.tMs, box: placed },
outDir,
signal,
});
return {
maskDir: segmented.maskDir,
keyframes: segmented.keyframes,
score: segmented.score,
review: segmented.score < MARK_REVIEW_MIN_SCORE,
};
}
/** The person's face in the frame a standing mark was drawn on. */
async function originFace(
deps: AnonymizeDeps,
spaceId: string,
mark: MarkRow,
): Promise<NormalizedBox | null> {
if (!mark.assetId || !mark.prompt || !mark.personId) return null;
const tracks = (await deps.repos.faceTrackRecords(spaceId, [mark.assetId])).filter(
(t) => t.personId === mark.personId,
);
for (const track of tracks) {
const face = faceAt(
await readTrackKeyframes(deps.mediaRoot, track.keyframesPath),
mark.prompt.tMs,
);
if (face) return face;
}
return null;
}
async function emptyMaskDir(dir: string): Promise<string> {
await mkdir(dir, { recursive: true });
return dir;
}
// ---------------------------------------------------------------------------
// Applying
// ---------------------------------------------------------------------------
/** In-root copy of the base file for providers (once per asset). */
async function workingCopy(deps: AnonymizeDeps, plan: AssetPlan): Promise<string> {
if (!(await isInsideRoot(deps.mediaRoot, plan.base)))
plan.base = await copyInto(plan.base, plan.dir);
return plan.base;
}
/** Static crop that keeps every reframe_out box out of frame; applies fallbacks otherwise. */
function chooseCrop(plan: AssetPlan, edl: Edl): PixelBox | null {
if (plan.reframe.length === 0) return null;
const boxes = bucketBoxes(plan.reframe.flatMap((item) => item.boxes));
const geometry = {
sourceWidth: plan.probe.width,
sourceHeight: plan.probe.height,
outWidth: edl.output.width,
outHeight: edl.output.height,
durationMs: plan.probe.durationMs,
};
const keyframe = reframeExcluding(boxes, geometry)?.[0];
const crop = keyframe ? cropPixels(geometry, keyframe) : null;
if (
crop &&
boxes.every(({ box }) => outsideCrop(box, crop, plan.probe.width, plan.probe.height))
) {
for (const item of plan.reframe) plan.checks.push(item.check);
return crop;
}
for (const item of plan.reframe) item.fallback();
return null;
}
/** Union boxes per second so the crop search stays small; unions only widen the excluded area. */
function bucketBoxes(boxes: Array<{ tMs: number; box: NormalizedBox }>) {
const buckets = new Map<number, NormalizedBox>();
for (const { tMs, box } of boxes) {
const key = Math.floor(tMs / REFRAME_BUCKET_MS);
const current = buckets.get(key);
buckets.set(key, current ? unionBox(current, box) : box);
}
return [...buckets.entries()].map(([key, box]) => ({ tMs: key * REFRAME_BUCKET_MS, box }));
}
function stepPath(plan: AssetPlan, file: string, step: string, ext = ".mp4"): string {
const stem = path
.basename(file, path.extname(file))
.replace(/\.(?:synth|inpaint|redacted|cropped)[-\w]*$/, "");
return path.join(plan.dir, `${stem}.${step}${ext}`);
}
async function applyPlan(
deps: AnonymizeDeps,
plan: AssetPlan,
crop: PixelBox | null,
context: {
capabilities: MediaCapabilities | null;
alterations: Set<AiAlteration>;
fallbacks: string[];
signal?: AbortSignal;
},
): Promise<string> {
const { signal } = context;
await mkdir(plan.dir, { recursive: true });
const needsProvider = plan.synthetic.length + plan.inpaint.length + plan.instructions.length > 0;
if (needsProvider && !deps.inference) {
throw new PrivacyUnavailableError("redaction needs the media worker, which is not configured");
}
let current = needsProvider ? await workingCopy(deps, plan) : plan.base;
const inference = deps.inference;
if (inference) {
for (const [index, synthetic] of plan.synthetic.entries()) {
const outPath = stepPath(plan, current, `synth-${index + 1}`);
current = (
await inference.synthesizeIdentity({
path: current,
track: synthetic.keyframes,
seed: synthetic.seed,
outPath,
signal,
})
).outPath;
context.alterations.add("synthetic_identity");
}
for (const mark of plan.inpaint) {
const outPath = stepPath(plan, current, `inpaint-${mark.markId}`);
const result = await inference.inpaintVideo({
path: current,
maskDir: mark.maskDir,
outPath,
signal,
});
if (result.quality.score >= MARK_QUALITY_MIN) {
current = result.outPath;
context.alterations.add("inpaint");
continue;
}
// Never ship artifacts silently: low quality falls back to a cover.
context.fallbacks.push(
`mark ${mark.markId}: inpainting scored ${result.quality.score.toFixed(2)}; covered instead`,
);
await rm(result.outPath, { force: true });
plan.instructions.push({
keyframes: mark.keyframes,
mode: "cover",
paddingPct: MARK_PADDING_PCT,
featherPct: 0,
maskDir: mark.maskDir,
});
}
if (plan.instructions.length > 0) {
if (!operationAvailable(context.capabilities, "redact")) {
throw new PrivacyUnavailableError("the media worker cannot redact");
}
const outPath = stepPath(plan, current, "redacted");
const result = await inference.redact({
path: current,
redactions: plan.instructions,
outPath,
signal,
});
if (result.framesProcessed <= 0)
throw new PrivacyUnavailableError("redaction processed no frames");
current = result.outPath;
}
}
if (crop) {
const outPath = stepPath(plan, current, "cropped", plan.probe.still ? ".png" : ".mp4");
await cropFile(current, outPath, crop, plan.probe.still, { ...deps, signal });
current = outPath;
}
if (plan.probe.still && path.extname(current).toLowerCase() !== ".png" && current !== plan.base) {
const outPath = stepPath(plan, current, "still", ".png");
await firstFrame(current, outPath, { ...deps, signal });
current = outPath;
}
return current;
}
/** The prepared file must show the whole clip range at the expected size. */
function verifyPrepared(plan: AssetPlan, crop: PixelBox | null, probed: ProbedSource): void {
const width = crop ? crop.w : plan.probe.width;
const height = crop ? crop.h : plan.probe.height;
if (probed.width !== width || probed.height !== height) {
throw new PrivacyUnavailableError(
`redacted asset ${plan.asset.id} is ${probed.width}x${probed.height}, expected ${width}x${height}`,
);
}
if (plan.probe.still) return;
const neededMs = Math.max(...plan.clips.map((clip) => clip.outMs));
const frameMs = 1000 / Math.max(1, plan.probe.fps);
if (probed.durationMs + 2 * frameMs < Math.min(neededMs, plan.probe.durationMs)) {
throw new PrivacyUnavailableError(`redacted asset ${plan.asset.id} is shorter than its clips`);
}
}
function compileSource(
file: string,
probe: ProbedSource,
rotation: CompileSource["rotation"],
): CompileSource {
const quarter = rotation === 90 || rotation === 270;
return {
path: file,
// CompileSource wants the coded size; probe reports the display size.
width: quarter ? probe.height : probe.width,
height: quarter ? probe.width : probe.height,
durationMs: probe.durationMs,
fps: probe.fps,
hasAudio: probe.hasAudio,
rotation,
still: probe.still,
};
}

97
packages/adapters/src/media-consent.test.ts Normal file
View File

@ -0,0 +1,97 @@
import { afterEach, describe, expect, it } from "vitest";
import {
assertExportConsent,
ConsentRequiredError,
consentIssues,
consentRequirements,
enforceConsentFlags,
participantsOf,
} from "./media-consent.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import { ACTOR, createPrivacyHarness, testEdl, testProfile } from "./media-privacy-fixtures.js";
const now = new Date("2026-06-01T12:00:00Z");
const past = new Date("2026-01-01T00:00:00Z");
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
describe("consent rules", () => {
it("always requires consent and age proof for paid video", () => {
expect(consentRequirements(testProfile())).toEqual({ consent: false, age: false });
expect(consentRequirements(testProfile({ kind: "paid_video" }))).toEqual({
consent: true,
age: true,
});
expect(enforceConsentFlags(testProfile({ kind: "paid_video" })).consent).toEqual({
requirePerformerConsent: true,
requireAgeVerification: true,
});
});
it("accepts only recorded, past proof with a verification method", () => {
const paid = testProfile({ kind: "paid_video" });
const ok = {
id: "a",
consentRecordedAt: past,
ageVerifiedAt: past,
ageVerificationMethod: "platform_verified",
};
expect(consentIssues(paid, [ok], [], now)).toEqual([]);
const future = { ...ok, consentRecordedAt: new Date("2027-01-01T00:00:00Z") };
const noMethod = { ...ok, id: "b", ageVerificationMethod: "none" };
expect(consentIssues(paid, [future, noMethod], [{ atMs: 10 }], now).map((i) => i.code)).toEqual(
["consent_missing", "age_unverified", "unknown_performer"],
);
expect(consentIssues(testProfile(), [noMethod], [{}], now)).toEqual([]);
});
it("collects the people and unassigned faces inside the clip ranges", () => {
const edl = testEdl([{ id: "c1", assetId: "a1", inMs: 1000, outMs: 2000 }]);
expect(
participantsOf(edl, [
{ assetId: "a1", personId: "alice", startMs: 0, endMs: 1200 },
{ assetId: "a1", personId: "bob", startMs: 2500, endMs: 3000 },
{ assetId: "a1", personId: null, startMs: 1500, endMs: 1600 },
{ assetId: "a2", personId: "carol", startMs: 0, endMs: 5000 },
]),
).toEqual({
personIds: ["alice"],
unassigned: [{ assetId: "a1", startMs: 1500, endMs: 1600 }],
});
});
});
describe("export consent", () => {
it("blocks a paid export until every participant has consent and age proof", async () => {
harness = await createPrivacyHarness(() => null);
const alice = harness.addPerson("alice", "none");
harness.store.tracks.push({
id: "t1",
assetId: "a1",
personId: "alice",
startMs: 0,
endMs: 1000,
confidenceMin: 0.9,
keyframesPath: "x.json",
modelVersion: "v1",
});
const edl = testEdl([{ id: "c1", assetId: "a1", inMs: 0, outMs: 1000 }]);
const paid = testProfile({ kind: "paid_video" });
await expect(assertExportConsent(harness.deps, ACTOR, { edl, profile: paid })).rejects.toThrow(
ConsentRequiredError,
);
await expect(
assertExportConsent(harness.deps, ACTOR, { edl, profile: testProfile() }),
).resolves.toBeUndefined();
alice.consentRecordedAt = past;
alice.ageVerifiedAt = past;
alice.ageVerificationMethod = "document_on_file";
await expect(
assertExportConsent(harness.deps, ACTOR, { edl, profile: paid }),
).resolves.toBeUndefined();
});
});

161
packages/adapters/src/media-consent.ts Normal file
View File

@ -0,0 +1,161 @@
import type { Edl, GateIssue, PlatformProfile } from "@rakazo/contracts";
import type { MediaActor, MediaServiceDeps } from "./media-context.js";
import { clipWindows, overlapsWindows } from "./media-privacy-plan.js";
/*
* Consent and age proof (spec 0 and 6.4). A paid-video profile always needs
* documented consent and age verification of every recognised participant,
* whatever its own flags say; other profiles follow their consent flags.
* Faces that belong to no person can never show consent and block.
*/
export interface ConsentRequirements {
consent: boolean;
age: boolean;
}
export function consentRequirements(profile: PlatformProfile): ConsentRequirements {
const paid = profile.kind === "paid_video";
return {
consent: paid || profile.consent.requirePerformerConsent,
age: paid || profile.consent.requireAgeVerification,
};
}
/** The profile with its consent flags raised to what the hard rules demand. */
export function enforceConsentFlags(profile: PlatformProfile): PlatformProfile {
const required = consentRequirements(profile);
return {
...profile,
consent: { requirePerformerConsent: required.consent, requireAgeVerification: required.age },
};
}
export interface PersonConsentState {
id: string;
consentRecordedAt: Date | string | null;
ageVerifiedAt: Date | string | null;
ageVerificationMethod: string;
}
/** Valid proof: a recorded date that is not in the future. */
function proven(value: Date | string | null, now: Date): boolean {
if (!value) return false;
const at = value instanceof Date ? value : new Date(value);
return !Number.isNaN(at.getTime()) && at.getTime() <= now.getTime();
}
export function hasConsent(person: PersonConsentState, now: Date): boolean {
return proven(person.consentRecordedAt, now);
}
export function hasAgeProof(person: PersonConsentState, now: Date): boolean {
return person.ageVerificationMethod !== "none" && proven(person.ageVerifiedAt, now);
}
/**
* Consent issues for the participants of an export. `unassigned` counts faces
* no person could be matched to; each blocks because nobody can consent for it.
*/
export function consentIssues(
profile: PlatformProfile,
people: PersonConsentState[],
unassigned: Array<{ atMs?: number; endMs?: number }>,
now: Date,
): GateIssue[] {
const required = consentRequirements(profile);
if (!required.consent && !required.age) return [];
const issues: GateIssue[] = [];
for (const person of [...people].sort((a, b) => a.id.localeCompare(b.id))) {
if (required.consent && !hasConsent(person, now)) {
issues.push({
check: "consent",
severity: "block",
code: "consent_missing",
message: "no recorded consent",
personId: person.id,
});
}
if (required.age && !hasAgeProof(person, now)) {
issues.push({
check: "consent",
severity: "block",
code: "age_unverified",
message: "age is not verified",
personId: person.id,
});
}
}
for (const face of unassigned) {
issues.push({
check: "consent",
severity: "block",
code: "unknown_performer",
message: "a visible face is not assigned to a person with consent",
...(face.atMs !== undefined ? { atMs: face.atMs } : {}),
...(face.endMs !== undefined ? { endMs: face.endMs } : {}),
});
}
return issues;
}
/**
* People and unassigned faces in the clip ranges of an EDL, from the stored
* face tracks of its assets.
*/
export function participantsOf(
edl: Edl,
tracks: Array<{ assetId: string; personId: string | null; startMs: number; endMs: number }>,
): {
personIds: string[];
unassigned: Array<{ assetId: string; startMs: number; endMs: number }>;
} {
const personIds = new Set<string>();
const unassigned: Array<{ assetId: string; startMs: number; endMs: number }> = [];
for (const track of tracks) {
const windows = clipWindows(edl.tracks.video.filter((clip) => clip.assetId === track.assetId));
if (!overlapsWindows(track, windows)) continue;
if (track.personId) personIds.add(track.personId);
else unassigned.push({ assetId: track.assetId, startMs: track.startMs, endMs: track.endMs });
}
return { personIds: [...personIds].sort(), unassigned };
}
export class ConsentRequiredError extends Error {
constructor(readonly issues: GateIssue[]) {
super(
`export blocked: ${issues.length} participant${issues.length === 1 ? "" : "s"} without documented consent or age proof`,
);
this.name = "ConsentRequiredError";
}
}
/**
* Consent issues for exporting a render to a profile, re-read at export time
* so a withdrawn consent stops a bundle that passed the gate earlier.
*/
export async function exportConsentIssues(
deps: Pick<MediaServiceDeps, "repos" | "now">,
actor: MediaActor,
input: { edl: Edl; profile: PlatformProfile },
): Promise<GateIssue[]> {
const required = consentRequirements(input.profile);
if (!required.consent && !required.age) return [];
const assetIds = [...new Set(input.edl.tracks.video.map((clip) => clip.assetId))];
const tracks = await deps.repos.faceTrackRecords(actor.spaceId, assetIds);
const { personIds, unassigned } = participantsOf(input.edl, tracks);
const people = personIds.length
? await deps.repos.listPersonRecords(actor.spaceId, personIds)
: [];
return consentIssues(input.profile, people, unassigned, deps.now());
}
/** Throws ConsentRequiredError when the export may not happen. */
export async function assertExportConsent(
deps: Pick<MediaServiceDeps, "repos" | "now">,
actor: MediaActor,
input: { edl: Edl; profile: PlatformProfile },
): Promise<void> {
const issues = await exportConsentIssues(deps, actor, input);
if (issues.length > 0) throw new ConsentRequiredError(issues);
}

188
packages/adapters/src/media-gate-autofix.test.ts Normal file
View File

@ -0,0 +1,188 @@
import type { Edl, GateReport } from "@rakazo/contracts";
import { MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
import { afterEach, describe, expect, it, vi } from "vitest";
import { autoFixRender, nonCompliantChange, proposeAutoFix } from "./media-gate-autofix.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import { ACTOR, createPrivacyHarness, testEdl } from "./media-privacy-fixtures.js";
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
function edlWithFace(): Edl {
return testEdl(
[
{ id: "c1", assetId: "a1", inMs: 0, outMs: 4000 },
{ id: "c2", assetId: "a2", inMs: 0, outMs: 4000 },
],
{
redactions: {
faces: [
{
id: "f1",
target: { personId: "alice" },
mode: "blur",
paddingPct: 20,
featherPct: 10,
minSizePct: 2,
},
],
marks: [{ markId: "m1", treatment: "inpaint" }],
regions: [],
},
},
);
}
function blocked(issues: GateReport["issues"], round = 0): GateReport {
return {
profileId: "test-profile",
passed: false,
checkedAt: "2026-06-01T12:00:00.000Z",
sampledFrames: 10,
issues,
autoFixRound: round,
};
}
const contentHit = {
check: "content" as const,
severity: "block" as const,
code: "content_threshold",
message: "explicit_nudity 0.90 above 0.5",
atMs: 7000,
endMs: 7500,
category: "explicit_nudity" as const,
score: 0.9,
};
describe("proposeAutoFix", () => {
it("only widens protection, covers marks, and cuts offending footage", () => {
const edl = edlWithFace();
const proposal = proposeAutoFix({
edl,
report: blocked([
contentHit,
{
check: "anonymization",
severity: "block",
code: "face_not_covered",
message: "face covered 80 %",
atMs: 100,
personId: "alice",
},
{
check: "anonymization",
severity: "block",
code: "mark_visible",
message: "visible",
atMs: 200,
markId: "m1",
},
]),
round: 0,
alternatives: [],
});
expect("edl" in proposal).toBe(true);
if (!("edl" in proposal)) return;
expect(proposal.edl.redactions.faces[0]!.paddingPct).toBe(45);
expect(proposal.edl.redactions.marks[0]!.treatment).toBe("cover");
expect(proposal.edl.tracks.video.find((c) => c.id === "c2")!.outMs).toBe(2500);
expect(proposal.changes.length).toBeGreaterThanOrEqual(3);
expect(nonCompliantChange(edl, proposal.edl)).toBeNull();
});
it("stops after the round limit", () => {
const proposal = proposeAutoFix({
edl: edlWithFace(),
report: blocked([contentHit], MAX_AUTO_FIX_ROUNDS),
round: MAX_AUTO_FIX_ROUNDS,
alternatives: [],
});
expect(proposal).toEqual({ reason: "round_limit", changes: [] });
});
it("rejects any change that weakens protection or adds content", () => {
const edl = edlWithFace();
const weaker = structuredClone(edl);
weaker.redactions.faces[0]!.paddingPct = 5;
expect(nonCompliantChange(edl, weaker)).toMatch(/shrank/);
const removed = structuredClone(edl);
removed.redactions.faces = [];
expect(nonCompliantChange(edl, removed)).toMatch(/removed/);
const unmasked = structuredClone(edl);
unmasked.redactions.marks[0]!.treatment = "blur";
expect(nonCompliantChange(edl, unmasked)).toMatch(/changed to blur/);
const grown = structuredClone(edl);
grown.tracks.video[0]!.outMs = 9000;
expect(nonCompliantChange(edl, grown)).toMatch(/grew/);
const overlay = structuredClone(edl);
overlay.overlays.push({ id: "o1", kind: "text", atMs: 0, durationMs: 100, text: "x" });
expect(nonCompliantChange(edl, overlay)).toMatch(/overlays/);
const swapped = structuredClone(edl);
swapped.redactions.faces[0]!.mode = "partial_eyes";
expect(nonCompliantChange(edl, swapped)).toMatch(/mode/);
});
});
describe("autoFixRender", () => {
async function blockedRender(round: number) {
harness = await createPrivacyHarness(() => null);
const edl = edlWithFace();
harness.store.plans.push({ id: "plan-1", projectId: "project-1", edl });
harness.store.renders.push({
id: "render-1",
planId: "plan-1",
projectId: "project-1",
quality: "draft",
status: "blocked",
autoFixRound: round,
gateReport: blocked([contentHit], round),
});
harness.store.renders.push({ id: "render-2", status: "queued", autoFixRound: 0 });
const submitPlan = vi.fn(async () => ({
ok: true as const,
plan: { id: "plan-2" } as never,
}));
const start = vi.fn(async () => ({ id: "render-2" }) as never);
return {
services: { deps: harness.deps, marketing: { submitPlan }, render: { start } },
submitPlan,
start,
};
}
it("submits a new plan with the reasons and renders it as the next round", async () => {
const { services, submitPlan, start } = await blockedRender(1);
const result = await autoFixRender(services, ACTOR, "render-1");
expect(result.status).toBe("rerendering");
expect(submitPlan).toHaveBeenCalledWith(
ACTOR,
expect.objectContaining({
projectId: "project-1",
createdBy: "agent",
rationale: expect.stringMatching(/^Auto-fix round 2 of 3:\n- clip c2: /),
}),
);
expect(start).toHaveBeenCalledWith(ACTOR, { planId: "plan-2", quality: "draft" });
expect(harness!.store.renders[1]!.autoFixRound).toBe(2);
});
it("hands over to the user after three rounds", async () => {
const { services, submitPlan } = await blockedRender(MAX_AUTO_FIX_ROUNDS);
const result = await autoFixRender(services, ACTOR, "render-1");
expect(result).toEqual({ status: "user_required", reason: "round_limit", changes: [] });
expect(submitPlan).not.toHaveBeenCalled();
});
it("does nothing for a render that is not blocked", async () => {
const { services } = await blockedRender(0);
harness!.store.renders[0]!.status = "done";
expect(await autoFixRender(services, ACTOR, "render-1")).toMatchObject({
status: "user_required",
reason: "not_blocked",
});
});
});

193
packages/adapters/src/media-gate-autofix.ts Normal file
View File

@ -0,0 +1,193 @@
import type { Edl, EdlError, GateReport, MediaRender, MediaRenderQuality } from "@rakazo/contracts";
import { EdlSchema, GateReportSchema, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
import { ACTIVE_RENDER_STATUSES } from "@rakazo/db";
import { planAutoFix } from "@rakazo/media";
import type {
MediaActor,
MediaMarketingService,
MediaRenderService,
MediaServiceDeps,
} from "./media-context.js";
/*
* Auto-fix for blocked renders (spec 6): only compliance-preserving repairs
* (trim or drop a clip, take another segment, change the crop, widen a
* redaction, cover a mark) through planAutoFix, re-checked here so no round
* can weaken protection or add anything new. Each round submits a new plan
* version with the reasons as rationale and starts a new render, which runs
* the gate again. After MAX_AUTO_FIX_ROUNDS the user takes over.
*/
export type AutoFixResult =
| { status: "rerendering"; round: number; render: MediaRender; changes: string[] }
| {
status: "user_required";
reason: "not_blocked" | "round_limit" | "no_safe_fix" | "plan_rejected";
changes: string[];
errors?: EdlError[];
};
export interface AutoFixDeps {
deps: Pick<MediaServiceDeps, "repos">;
marketing: Pick<MediaMarketingService, "submitPlan">;
render: Pick<MediaRenderService, "start">;
}
/**
* Why a proposed fix is not a pure compliance repair, or null when it is.
* Allowed: fewer or shorter clips, other segments, other crops, wider padding
* on the same face redactions, added blur redactions, marks switched to cover.
* Forbidden: anything that removes or weakens protection or adds content.
*/
export function nonCompliantChange(before: Edl, after: Edl): string | null {
const facesBefore = new Map(before.redactions.faces.map((f) => [f.id, f]));
for (const face of before.redactions.faces) {
const next = after.redactions.faces.find((f) => f.id === face.id);
if (!next) return `face redaction ${face.id} was removed`;
if (JSON.stringify(next.target) !== JSON.stringify(face.target))
return `face redaction ${face.id} changed its target`;
if (next.mode !== face.mode) return `face redaction ${face.id} changed its mode`;
if (next.paddingPct < face.paddingPct || next.minSizePct < face.minSizePct)
return `face redaction ${face.id} shrank`;
}
for (const face of after.redactions.faces) {
if (!facesBefore.has(face.id) && face.mode !== "blur")
return `added face redaction ${face.id} is not a blur`;
}
for (const mark of before.redactions.marks) {
const next = after.redactions.marks.find((m) => m.markId === mark.markId);
if (!next) return `mark ${mark.markId} lost its treatment`;
if (next.treatment !== mark.treatment && next.treatment !== "cover")
return `mark ${mark.markId} changed to ${next.treatment}`;
}
if (after.redactions.marks.length !== before.redactions.marks.length)
return "mark treatments were added";
const clipIds = new Set(after.tracks.video.map((clip) => clip.id));
for (const region of before.redactions.regions) {
if (!clipIds.has(region.clipId)) continue;
const next = after.redactions.regions.find((r) => r.id === region.id);
if (!next || JSON.stringify(next) !== JSON.stringify(region))
return `region ${region.id} changed`;
}
if (after.redactions.regions.length > before.redactions.regions.length)
return "regions were added";
if (JSON.stringify(after.redactions.voice) !== JSON.stringify(before.redactions.voice))
return "voice disguise changed";
const beforeClips = new Map(before.tracks.video.map((clip) => [clip.id, clip]));
for (const clip of after.tracks.video) {
const old = beforeClips.get(clip.id);
if (!old) return `clip ${clip.id} was added`;
if (clip.speed !== old.speed || clip.muted !== old.muted)
return `clip ${clip.id} changed speed or sound`;
if (JSON.stringify(clip.color) !== JSON.stringify(old.color))
return `clip ${clip.id} changed its colors`;
if (clip.assetId === old.assetId && (clip.inMs < old.inMs || clip.outMs > old.outMs)) {
if (clip.segmentId === old.segmentId) return `clip ${clip.id} grew`;
}
}
if (after.overlays.length > before.overlays.length) return "overlays were added";
for (const overlay of after.overlays) {
const old = before.overlays.find((o) => o.id === overlay.id);
if (!old || old.text !== overlay.text || old.kind !== overlay.kind)
return `overlay ${overlay.id} changed`;
}
if (
JSON.stringify(after.tracks.audio) !== JSON.stringify(before.tracks.audio) ||
JSON.stringify(after.captions) !== JSON.stringify(before.captions) ||
JSON.stringify(after.intro) !== JSON.stringify(before.intro) ||
JSON.stringify(after.outro) !== JSON.stringify(before.outro) ||
JSON.stringify(after.output) !== JSON.stringify(before.output)
) {
return "audio, captions, bookends, or output settings changed";
}
for (const alteration of after.metadata.aiAlterations) {
if (!before.metadata.aiAlterations.includes(alteration))
return `AI alteration ${alteration} was added`;
}
return null;
}
/** Plan one auto-fix round for a blocked render, or say why the user must take over. */
export function proposeAutoFix(input: {
edl: Edl;
report: GateReport;
round: number;
alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>;
}): { edl: Edl; changes: string[] } | { reason: "round_limit" | "no_safe_fix"; changes: string[] } {
if (input.round >= MAX_AUTO_FIX_ROUNDS) return { reason: "round_limit", changes: [] };
const fix = planAutoFix(
input.edl,
{ ...input.report, autoFixRound: input.round },
input.alternatives,
);
if (!fix) return { reason: "no_safe_fix", changes: [] };
const parsed = EdlSchema.safeParse(fix.edl);
if (!parsed.success) return { reason: "no_safe_fix", changes: fix.changes };
const violation = nonCompliantChange(input.edl, parsed.data);
if (violation)
return { reason: "no_safe_fix", changes: [...fix.changes, `rejected: ${violation}`] };
return { edl: parsed.data, changes: fix.changes };
}
/**
* Run one auto-fix round for a blocked render: new plan version (createdBy
* agent, rationale lists every change) and a new render carrying the next
* round number, whose gate decides again.
*/
export async function autoFixRender(
services: AutoFixDeps,
actor: MediaActor,
renderId: string,
): Promise<AutoFixResult> {
const { repos } = services.deps;
const row = await repos.getRenderRecord(actor.spaceId, renderId);
const report = GateReportSchema.nullable().parse(row.gateReport ?? null);
if (row.status !== "blocked" || !report || report.passed) {
return { status: "user_required", reason: "not_blocked", changes: [] };
}
const plan = await repos.getPlan(actor.spaceId, row.planId);
const project = await repos.getProjectRecord(actor.spaceId, row.projectId);
const assetIds = [
...new Set([...project.assetIds, ...plan.edl.tracks.video.map((clip) => clip.assetId)]),
];
const segments = await repos.listSegments(actor.spaceId, assetIds);
const alternatives = segments
.filter((segment) => !segment.paidOnly)
.map((segment) => ({
segmentId: segment.id,
assetId: segment.assetId,
startMs: segment.startMs,
endMs: segment.endMs,
}));
const proposal = proposeAutoFix({ edl: plan.edl, report, round: row.autoFixRound, alternatives });
if ("reason" in proposal) {
return { status: "user_required", reason: proposal.reason, changes: proposal.changes };
}
const round = row.autoFixRound + 1;
const rationale = [
`Auto-fix round ${round} of ${MAX_AUTO_FIX_ROUNDS}:`,
...proposal.changes,
].join("\n- ");
const submitted = await services.marketing.submitPlan(actor, {
projectId: row.projectId,
edl: proposal.edl,
rationale,
createdBy: "agent",
});
if (!submitted.ok) {
return {
status: "user_required",
reason: "plan_rejected",
changes: proposal.changes,
errors: submitted.errors,
};
}
const started = await services.render.start(actor, {
planId: submitted.plan.id,
quality: row.quality as MediaRenderQuality,
});
const updated = await repos.updateRender(actor.spaceId, started.id, { autoFixRound: round }, [
...ACTIVE_RENDER_STATUSES,
]);
return { status: "rerendering", round, render: updated ?? started, changes: proposal.changes };
}

250
packages/adapters/src/media-gate.test.ts Normal file
View File

@ -0,0 +1,250 @@
import { copyFile, rm } from "node:fs/promises";
import path from "node:path";
import type { MediaInferenceProvider, TrackFacesResult } from "@rakazo/adapter-kit";
import type { AnonymizeMode, Edl, PlatformProfile } from "@rakazo/contracts";
import { decodeVectors } from "@rakazo/media";
import { afterEach, describe, expect, it } from "vitest";
import { prepareSources } from "./media-anonymize.js";
import { ffmpegAvailable } from "./media-emulator-clips.js";
import { cutTimes, GATE_PIXEL_STEP_MS, pixelSampleTimes, runGate } from "./media-gate.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import {
ACTOR,
cleanProbe,
createPrivacyHarness,
testEdl,
testProfile,
} from "./media-privacy-fixtures.js";
import { PRIVACY_RECORD_FILE } from "./media-privacy-plan.js";
const hasFfmpeg = ffmpegAvailable();
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
/** Faces the fake detector reports on the render output. */
let outputFaces: TrackFacesResult = { sampleFps: 4, tracks: [] };
let classifierAvailable = true;
function withOutputDetector(emulator: PrivacyHarness["inference"]): MediaInferenceProvider {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.trackFaces = async (request) =>
path.basename(request.path).startsWith("output") ? outputFaces : emulator.trackFaces(request);
provider.capabilities = async () => {
const caps = await emulator.capabilities();
caps.operations.classifyContent.available = classifierAvailable;
return caps;
};
return provider;
}
async function prepared(
mode: AnonymizeMode,
options: { file?: string; unassigned?: boolean } = {},
) {
outputFaces = { sampleFps: 4, tracks: [] };
classifierAvailable = true;
harness = await createPrivacyHarness(withOutputDetector);
const asset = await harness.addAsset(options.file ?? "clip__faces-2____person-alice__.mp4");
harness.addPerson("alice", mode);
await harness.analyseFaces(asset.id, ["alice", options.unassigned ? null : "alice"]);
await harness.learnPerson("alice", asset.id);
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
const workDir = await harness.workDir(`gate-${mode}`);
const result = await prepareSources(harness.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
const output = path.join(workDir, "output.mp4");
await copyFile(result.sources.get(asset.id)!.path, output);
return { h: harness, asset, edl, workDir, output };
}
function gate(
h: PrivacyHarness,
input: { edl: Edl; output: string },
options: {
profile?: PlatformProfile;
copy?: { title: string | null; caption: string | null; tags: string[] };
probe?: Partial<ReturnType<typeof cleanProbe>>;
} = {},
) {
return runGate(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl: input.edl,
profile: options.profile ?? testProfile(),
outputPath: input.output,
ffprobe: { ...cleanProbe(input.edl), ...options.probe },
copy: options.copy ?? { title: null, caption: null, tags: [] },
bundleHasDisclosure: false,
autoFixRound: 0,
});
}
const box = { x: 0.4, y: 0.4, w: 0.2, h: 0.2 };
const strangerEmbedding = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
describe("gate sampling", () => {
it("samples each clip range and spreads long ones", () => {
const edl = testEdl([
{ id: "a", assetId: "x", inMs: 0, outMs: 1000 },
{ id: "b", assetId: "x", inMs: 5000, outMs: 6000 },
]);
expect(cutTimes(edl)).toEqual([0, 999, 1000, 1999]);
expect(pixelSampleTimes(edl.tracks.video, false)).toEqual([0, 500, 960, 5000, 5500, 5960]);
expect(pixelSampleTimes(edl.tracks.video, true)).toEqual([0]);
const long = pixelSampleTimes([{ ...edl.tracks.video[0]!, outMs: 600_000 }], false);
expect(long).toHaveLength(40);
expect(long[1]! - long[0]!).toBeGreaterThanOrEqual(GATE_PIXEL_STEP_MS);
});
});
describe.skipIf(!hasFfmpeg)("runGate", () => {
it("passes a protected render with a clean classifier", async () => {
const { h, edl, output } = await prepared("blur");
const report = await gate(h, { edl, output });
expect(report.issues).toEqual([]);
expect(report.passed).toBe(true);
expect(report.sampledFrames).toBeGreaterThan(0);
});
it("classifies the rendered output and blocks above the threshold", async () => {
const { h, edl, workDir, output } = await prepared("blur");
const flagged = path.join(workDir, "output__nsfw-explicit_nudity-0.9__.mp4");
await copyFile(output, flagged);
const report = await gate(h, { edl, output: flagged });
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({
check: "content",
code: "content_threshold",
category: "explicit_nudity",
score: 0.9,
atMs: 0,
}),
);
});
it("blocks when the classifier is unavailable", async () => {
const { h, edl, output } = await prepared("blur");
classifierAvailable = false;
const report = await gate(h, { edl, output });
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "classifier_unavailable", severity: "block" }),
);
});
it("blocks a recognisable protected face and an unknown face in the output", async () => {
const { h, edl, output } = await prepared("blur");
const alice = h.store.people[0]!;
const embedding = [...decodeVectors(alice.centroid!, 16)[0]!];
outputFaces = {
sampleFps: 4,
tracks: [
{ trackId: "o1", keyframes: [{ tMs: 250, box, confidence: 0.9 }], embedding },
{
trackId: "o2",
keyframes: [{ tMs: 750, box, confidence: 0.9 }],
embedding: strangerEmbedding,
},
],
};
const report = await gate(h, { edl, output });
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "face_not_covered", personId: "alice", atMs: 250 }),
);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "unknown_face", atMs: 750 }),
);
});
it("lets a partially covered face stay detectable without counting it as unknown", async () => {
const { h, edl, workDir, output } = await prepared("partial_eyes");
expect(path.basename(workDir)).toBe("gate-partial_eyes");
outputFaces = {
sampleFps: 4,
tracks: [
{
trackId: "o1",
keyframes: [{ tMs: 250, box, confidence: 0.9 }],
embedding: strangerEmbedding,
},
],
};
const report = await gate(h, { edl, output });
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
});
it("requires consent and age proof of every participant for paid video", async () => {
const { h, edl, output } = await prepared("none");
// The profile's own flags are off; paid video turns them on regardless.
const paid = testProfile({ kind: "paid_video" });
const blocked = await gate(h, { edl, output }, { profile: paid });
expect(blocked.passed).toBe(false);
expect(blocked.issues.filter((i) => i.check === "consent").map((i) => i.code)).toEqual([
"consent_missing",
"age_unverified",
]);
const alice = h.store.people[0]!;
alice.consentRecordedAt = new Date("2026-01-01T00:00:00Z");
alice.ageVerifiedAt = new Date("2026-01-01T00:00:00Z");
alice.ageVerificationMethod = "document_on_file";
const allowed = await gate(h, { edl, output }, { profile: paid });
expect(allowed.issues.filter((i) => i.check === "consent")).toEqual([]);
expect(allowed.passed).toBe(true);
});
it("blocks consent profiles when a face belongs to nobody", async () => {
const { h, edl, output } = await prepared("none", { unassigned: true });
const profile = testProfile({
consent: { requirePerformerConsent: true, requireAgeVerification: true },
});
const alice = h.store.people[0]!;
alice.consentRecordedAt = new Date("2026-01-01T00:00:00Z");
alice.ageVerifiedAt = new Date("2026-01-01T00:00:00Z");
alice.ageVerificationMethod = "platform_verified";
const report = await gate(h, { edl, output }, { profile });
expect(report.issues).toEqual([
expect.objectContaining({ check: "consent", code: "unknown_performer", atMs: 0 }),
]);
});
it("blocks banned text and a duration outside the profile", async () => {
const { h, edl, output } = await prepared("blur");
const profile = testProfile({
duration: { hardMinMs: 0, hardMaxMs: 1500 },
text: {
captionMaxChars: 100,
hashtagLimit: 3,
linkPolicy: "forbidden",
bannedPatterns: [{ pattern: "free\\s+money", reason: "spam" }],
},
});
const report = await gate(
h,
{ edl, output },
{ profile, copy: { title: "Free money", caption: "see example.com", tags: [] } },
);
const codes = report.issues.map((issue) => issue.code);
expect(codes).toEqual(
expect.arrayContaining(["duration_too_long", "banned_pattern", "link_forbidden"]),
);
expect(report.passed).toBe(false);
});
it("blocks when the sources were not prepared for this render", async () => {
const { h, edl, workDir, output } = await prepared("blur");
await rm(path.join(workDir, PRIVACY_RECORD_FILE));
const report = await gate(h, { edl, output });
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "privacy_unverified", severity: "block" }),
);
});
});

396
packages/adapters/src/media-gate.ts Normal file
View File

@ -0,0 +1,396 @@
import path from "node:path";
import type {
ContentCategory,
Edl,
EdlVideoClip,
FfprobeReport,
GateIssue,
GateReport,
NormalizedBox,
PlatformProfile,
} from "@rakazo/contracts";
import type { GateInput } from "@rakazo/media";
import { clipLengthMs, evaluateGate, introMs, MAX_MARK_RESIDUAL } from "@rakazo/media";
import { assertInside } from "@rakazo/media/node";
import { consentIssues, enforceConsentFlags, hasAgeProof, hasConsent } from "./media-consent.js";
import type { MediaActor, MediaServiceDeps } from "./media-context.js";
import type { PrivacyToolOptions } from "./media-privacy-io.js";
import { inferenceCapabilities, maskTimes, operationAvailable } from "./media-privacy-io.js";
import type { LumaFrame } from "./media-privacy-pixels.js";
import {
controlAgreement,
extractLuma,
readPngLuma,
residualVisibility,
} from "./media-privacy-pixels.js";
import type { PrivacyCheck, PrivacyRecord, TimeWindow } from "./media-privacy-plan.js";
import {
boxAt,
isPartial,
matchPerson,
readPrivacyRecord,
sourceToOutput,
} from "./media-privacy-plan.js";
/*
* Gate execution on the rendered output (spec 6). Content is classified on the
* output file at the profile's sample rate plus every cut; faces are tracked on
* the output and matched to people locally. Protection is verified pixel by
* pixel: for every protected region prepareSources recorded, the prepared
* frame is compared with the unredacted frame at the same moment. The
* compiler only crops, scales, and composites prepared frames, so a region
* altered there stays altered in the output. Anything the gate cannot check
* blocks (fail closed).
*/
export type GateDeps = Pick<MediaServiceDeps, "repos" | "inference" | "mediaRoot" | "now"> &
PrivacyToolOptions;
/** Face tracking rate on the output. */
export const GATE_FACE_FPS = 4;
/** Pixel verification step inside each clip range. */
export const GATE_PIXEL_STEP_MS = 500;
/** Pixel samples per asset at most (spread evenly). */
export const MAX_PIXEL_SAMPLES_PER_ASSET = 40;
/** Below this agreement outside the treated regions, two frames are not comparable. */
const MIN_CONTROL_AGREEMENT = 0.8;
export interface GateRunInput {
actor: MediaActor;
projectId: string;
edl: Edl;
profile: PlatformProfile;
outputPath: string;
ffprobe: FfprobeReport;
copy: { title: string | null; caption: string | null; tags: string[] };
bundleHasDisclosure: boolean;
autoFixRound: number;
signal?: AbortSignal;
}
type Faces = GateInput["faces"];
type Marks = GateInput["marks"];
function block(issue: Omit<GateIssue, "severity">): GateIssue {
return { severity: "block", ...issue };
}
/** Output times of every cut: clip starts and ends, intro end. */
export function cutTimes(edl: Edl): number[] {
const intro = introMs(edl);
const times = new Set<number>();
if (intro > 0) times.add(intro);
for (const clip of edl.tracks.video) {
times.add(intro + clip.atMs);
times.add(Math.max(0, intro + clip.atMs + clipLengthMs(clip) - 1));
}
return [...times].sort((a, b) => a - b);
}
function clipsOf(edl: Edl, assetId: string): EdlVideoClip[] {
return edl.tracks.video.filter((clip) => clip.assetId === assetId);
}
/** Output windows of a source span across every clip of its asset. */
function toOutput(edl: Edl, assetId: string, span: TimeWindow): TimeWindow[] {
return clipsOf(edl, assetId).flatMap((clip) => {
const mapped = sourceToOutput(edl, clip, span);
return mapped ? [mapped] : [];
});
}
/** Evaluate the full gate for a render output. */
export async function runGate(deps: GateDeps, input: GateRunInput): Promise<GateReport> {
const { actor, edl, signal } = input;
const profile = enforceConsentFlags(input.profile);
const outputPath = await assertInside(deps.mediaRoot, input.outputPath);
const record = await readPrivacyRecord(path.dirname(outputPath));
const capabilities = await inferenceCapabilities(deps.inference, signal);
const extra: GateIssue[] = [];
const now = deps.now();
if (!record) {
extra.push(
block({
check: "anonymization",
code: "privacy_unverified",
message: "sources were not prepared for this render, so protection cannot be shown",
}),
);
}
// Content on the rendered output.
let content: GateInput["content"] = [];
let sampledFrames = 0;
if (Object.keys(profile.content.thresholds).length > 0) {
if (deps.inference && operationAvailable(capabilities, "classifyContent")) {
const classified = await deps.inference.classifyContent({
path: outputPath,
sampleFps: profile.content.sampleFps,
extraTimesMs: cutTimes(edl),
signal,
});
content = classified.frames.map((frame) => ({
tMs: frame.tMs,
scores: frame.scores as Partial<Record<ContentCategory, number>>,
}));
sampledFrames = content.length;
} else {
extra.push(
block({
check: "content",
code: "classifier_unavailable",
message: "the content classifier is unavailable, so the output cannot be checked",
}),
);
}
}
// Faces re-detected on the output.
const faces: Faces = [];
const matchedPeople = new Set<string>();
const protectedPeople = record?.protectedPeople ?? {};
const consentRequired =
profile.consent.requirePerformerConsent || profile.consent.requireAgeVerification;
const needFaces = Object.keys(protectedPeople).length > 0 || consentRequired;
if (needFaces) {
if (deps.inference && operationAvailable(capabilities, "trackFaces")) {
const tracked = await deps.inference.trackFaces({
path: outputPath,
sampleFps: GATE_FACE_FPS,
signal,
});
const centroids = await deps.repos.personCentroids(actor.spaceId);
const expected = expectedOutputFaces(edl, record);
const unmatchedAt = new Map<number, number>();
const slot = (tMs: number) => Math.round(tMs / (1000 / GATE_FACE_FPS));
for (const track of tracked.tracks) {
const personId = matchPerson(track.embedding, centroids);
if (personId) matchedPeople.add(personId);
const mode = personId ? protectedPeople[personId] : undefined;
for (const keyframe of track.keyframes) {
if (personId) {
// Partial modes and synthetic identities stay detectable by design;
// their regions are verified pixel by pixel below.
if (mode && (isPartial(mode) || mode === "synthetic_identity")) continue;
faces.push({ tMs: keyframe.tMs, box: keyframe.box, personId, coverage: mode ? 0 : 1 });
continue;
}
const key = slot(keyframe.tMs);
const seen = unmatchedAt.get(key) ?? 0;
unmatchedAt.set(key, seen + 1);
const allowance = expected.filter(
(w) => keyframe.tMs >= w.startMs && keyframe.tMs <= w.endMs,
).length;
if (seen < allowance) continue;
faces.push({ tMs: keyframe.tMs, box: keyframe.box, personId: null, coverage: 0 });
}
}
} else {
extra.push(
block({
check: "anonymization",
code: "face_check_unavailable",
message: "face detection is unavailable, so faces in the output cannot be checked",
}),
);
}
}
// Pixel verification of every recorded protection.
const marks: Marks = [];
if (record) {
const verified = await verifyPixels(deps, edl, record, signal);
for (const sample of verified) {
const covered = sample.residual <= MAX_MARK_RESIDUAL;
const { check } = sample;
if (check.kind === "face") {
if (check.personId && protectedPeople[check.personId]) {
faces.push({
tMs: sample.outputMs,
box: sample.box,
personId: check.personId,
coverage: covered ? 1 : 1 - sample.residual,
});
} else if (!covered) {
extra.push(
block({
check: "anonymization",
code: "face_not_covered",
message: `face covered ${((1 - sample.residual) * 100).toFixed(0)} %`,
atMs: sample.outputMs,
endMs: sample.outputMs,
score: 1 - sample.residual,
...(check.personId ? { personId: check.personId } : {}),
}),
);
}
} else {
marks.push({
markId: check.markId ?? check.id,
tMs: sample.outputMs,
residualVisibility: sample.residual,
});
}
}
for (const review of record.reviews) {
const window = toOutput(edl, review.assetId, {
startMs: 0,
endMs: Number.MAX_SAFE_INTEGER,
})[0];
extra.push(
block({
check: "anonymization",
code: "mark_needs_review",
message: `mark found with low confidence (${review.score.toFixed(2)}); review it`,
markId: review.markId,
...(window ? { atMs: window.startMs, endMs: window.endMs } : {}),
}),
);
}
}
const trackingGaps: GateInput["trackingGaps"] = (record?.trackingGaps ?? []).flatMap((gap) =>
toOutput(edl, gap.assetId, gap).map((window) => ({ ...window, personId: gap.personId })),
);
// People in the output and their consent state.
const ids = [...new Set([...(record?.visiblePeople ?? []), ...matchedPeople])].sort();
const persons = ids.length ? await deps.repos.listPersonRecords(actor.spaceId, ids) : [];
const people: GateInput["people"] = persons.map((person) => ({
id: person.id,
anonymizeByDefault: protectedPeople[person.id] ?? "none",
consentRecordedAt: hasConsent(person, now)
? new Date(person.consentRecordedAt!).toISOString()
: null,
ageVerifiedAt: hasAgeProof(person, now) ? new Date(person.ageVerifiedAt!).toISOString() : null,
}));
if (record && consentRequired) {
const unassigned = record.unassignedFaces.flatMap((face) =>
toOutput(edl, face.assetId, face).map((w) => ({ atMs: w.startMs, endMs: w.endMs })),
);
extra.push(...consentIssues(profile, [], unassigned, now));
}
const report = evaluateGate({
profile,
edl,
ffprobe: input.ffprobe,
content,
faces,
trackingGaps,
marks,
people,
copy: input.copy,
aiAlterations: record?.aiAlterations ?? [],
bundleHasDisclosure: input.bundleHasDisclosure,
watermarkBox: null,
sampledFrames,
autoFixRound: input.autoFixRound,
now,
});
const issues = [...report.issues, ...extra];
return { ...report, issues, passed: !issues.some((issue) => issue.severity === "block") };
}
/** Output windows where a face stays detectable by design (synthetic identity, partial modes). */
function expectedOutputFaces(edl: Edl, record: PrivacyRecord | null): TimeWindow[] {
return (record?.expectedFaces ?? []).flatMap((face) => toOutput(edl, face.assetId, face));
}
interface PixelSample {
check: PrivacyCheck;
outputMs: number;
box: NormalizedBox;
residual: number;
}
/** Source times to verify per asset: every GATE_PIXEL_STEP_MS inside its clips, spread evenly. */
export function pixelSampleTimes(clips: EdlVideoClip[], still: boolean): number[] {
if (still) return [0];
const times = new Set<number>();
for (const clip of clips) {
for (let t = clip.inMs; t < clip.outMs; t += GATE_PIXEL_STEP_MS) times.add(t);
times.add(Math.max(clip.inMs, clip.outMs - 40));
}
const sorted = [...times].sort((a, b) => a - b);
if (sorted.length <= MAX_PIXEL_SAMPLES_PER_ASSET) return sorted;
const step = (sorted.length - 1) / (MAX_PIXEL_SAMPLES_PER_ASSET - 1);
return Array.from(
{ length: MAX_PIXEL_SAMPLES_PER_ASSET },
(_, i) => sorted[Math.round(i * step)]!,
);
}
async function verifyPixels(
deps: GateDeps,
edl: Edl,
record: PrivacyRecord,
signal?: AbortSignal,
): Promise<PixelSample[]> {
const samples: PixelSample[] = [];
const maskCache = new Map<string, number[]>();
for (const [assetId, entry] of Object.entries(record.assets).sort((a, b) =>
a[0].localeCompare(b[0]),
)) {
if (entry.checks.length === 0) continue;
const clips = clipsOf(edl, assetId);
const treatedSize = entry.crop
? { width: entry.crop.w, height: entry.crop.h }
: { width: entry.width, height: entry.height };
const offset = entry.crop ? { x: entry.crop.x, y: entry.crop.y } : undefined;
for (const tMs of pixelSampleTimes(clips, entry.still)) {
signal?.throwIfAborted();
const active = entry.checks.flatMap((check) => {
const box = entry.still ? (check.keyframes[0]?.box ?? null) : boxAt(check.keyframes, tMs);
return box ? [{ check, box }] : [];
});
if (active.length === 0) continue;
const tool = { ffmpegPath: deps.ffmpegPath, signal };
const source = await extractLuma(
entry.basePath,
{ tMs, width: entry.width, height: entry.height, still: entry.still },
tool,
);
const treated = await extractLuma(
entry.preparedPath,
{ tMs, ...treatedSize, still: entry.still },
tool,
);
const aligned =
controlAgreement({ source, treated, boxes: active.map((a) => a.box), offset }) >=
MIN_CONTROL_AGREEMENT;
const outputTimes = entry.still
? clips.map((clip) => introMs(edl) + clip.atMs)
: clips.flatMap((clip) => {
const mapped = sourceToOutput(edl, clip, { startMs: tMs, endMs: tMs });
return mapped ? [mapped.startMs] : [];
});
for (const { check, box } of active) {
let residual = 1;
if (aligned) {
const mask = check.maskDir ? await maskAt(check.maskDir, tMs, maskCache) : undefined;
residual = residualVisibility({ source, treated, box, offset, mask });
}
for (const outputMs of outputTimes) samples.push({ check, outputMs, box, residual });
}
}
}
return samples;
}
/** The mask frame nearest at or before `tMs` (or the first one), if the directory has any. */
async function maskAt(
maskDir: string,
tMs: number,
cache: Map<string, number[]>,
): Promise<LumaFrame | undefined> {
let times = cache.get(maskDir);
if (!times) {
times = await maskTimes(maskDir).catch(() => []);
cache.set(maskDir, times);
}
if (times.length === 0) return undefined;
const before = times.filter((t) => t <= tMs).at(-1) ?? times[0]!;
return readPngLuma(path.join(maskDir, `${before}.png`));
}

300
packages/adapters/src/media-marks.test.ts Normal file
View File

@ -0,0 +1,300 @@
import { copyFile, stat } from "node:fs/promises";
import path from "node:path";
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
import type { Edl } from "@rakazo/contracts";
import { afterEach, describe, expect, it, vi } from "vitest";
import { prepareSources } from "./media-anonymize.js";
import { ffmpegAvailable } from "./media-emulator-clips.js";
import { runGate } from "./media-gate.js";
import { anchorFromFace, createMark, placeAnchor, promptBox, updateMark } from "./media-marks.js";
import { createMediaPrivacyService } from "./media-privacy.js";
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
import {
ACTOR,
cleanProbe,
createPrivacyHarness,
testEdl,
testProfile,
} from "./media-privacy-fixtures.js";
import { PrivacyUnavailableError } from "./media-privacy-io.js";
import { readPrivacyRecord } from "./media-privacy-plan.js";
const hasFfmpeg = ffmpegAvailable();
let harness: PrivacyHarness | null = null;
afterEach(async () => {
await harness?.cleanup();
harness = null;
});
const tattoo = { tMs: 500, box: { x: 0.6, y: 0.55, w: 0.2, h: 0.25 } };
async function render(h: PrivacyHarness, edl: Edl, name: string) {
const workDir = await h.workDir(name);
const prepared = await prepareSources(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
quality: "final",
workDir,
});
return { workDir, prepared };
}
async function gate(h: PrivacyHarness, edl: Edl, workDir: string, prepared: string) {
const outputPath = path.join(workDir, "output.mp4");
await copyFile(prepared, outputPath);
return runGate(h.deps, {
actor: ACTOR,
projectId: "project-1",
edl,
profile: testProfile(),
outputPath,
ffprobe: cleanProbe(edl),
copy: { title: null, caption: null, tags: [] },
bundleHasDisclosure: false,
autoFixRound: 0,
});
}
describe("mark geometry", () => {
it("turns a point prompt into a small box inside the frame", () => {
expect(promptBox({ tMs: 0, point: { x: 0.99, y: 0.01 } })).toEqual({
x: 0.88,
y: 0,
w: 0.12,
h: 0.12,
});
expect(() => promptBox({ tMs: 0 })).toThrow(PrivacyUnavailableError);
});
it("keeps a mark's place relative to the face", () => {
const anchor = anchorFromFace(
{ x: 0.5, y: 0.5, w: 0.1, h: 0.1 },
{ x: 0.4, y: 0.3, w: 0.2, h: 0.2 },
);
const placed = placeAnchor(anchor, { x: 0.1, y: 0.1, w: 0.4, h: 0.4 })!;
expect(placed.x).toBeCloseTo(0.3);
expect(placed.y).toBeCloseTo(0.5);
expect(placed.w).toBeCloseTo(0.2);
expect(placeAnchor(anchor, { x: 0.95, y: 0.95, w: 0.4, h: 0.4 })).toBeNull();
});
});
describe.skipIf(!hasFfmpeg)("marks", () => {
it("segments, propagates, scores, and stores the mask under the media root", async () => {
harness = await createPrivacyHarness();
const asset = await harness.addAsset("clip.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "blur",
});
expect(mark).toMatchObject({ treatment: "blur", qualityScore: 0.9, suggestedFallback: null });
const stored = harness.store.marks[0]!;
expect(path.isAbsolute(stored.maskDir!)).toBe(false);
expect((await stat(path.join(harness.root, stored.maskDir!, "0.png"))).isFile()).toBe(true);
});
it("suggests a fallback when inpainting scores low and rescoring follows the treatment", async () => {
harness = await createPrivacyHarness();
const asset = await harness.addAsset("clip__inpaintq-0.3__.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "inpaint",
});
expect(mark.qualityScore).toBeLessThanOrEqual(0.3);
expect(mark.suggestedFallback).toBe("cover");
const blurred = await updateMark(harness.deps, ACTOR, { markId: mark.id, treatment: "blur" });
expect(blurred).toMatchObject({
treatment: "blur",
qualityScore: 0.9,
suggestedFallback: null,
});
});
it("keeps a mark without a worker and rendering it fails closed", async () => {
harness = await createPrivacyHarness(() => null);
const asset = await harness.addAsset("clip.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "license_plate",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "cover",
});
expect(mark.qualityScore).toBeNull();
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
await expect(render(harness, edl, "no-worker")).rejects.toBeInstanceOf(PrivacyUnavailableError);
});
it("treats a marked region through the video and the gate checks what is left", async () => {
harness = await createPrivacyHarness();
const asset = await harness.addAsset("clip.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "none",
});
// The plan asks for a cover even though the mark itself says none.
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
redactions: { faces: [], marks: [{ markId: mark.id, treatment: "cover" }], regions: [] },
});
const { workDir, prepared } = await render(harness, edl, "cover");
const record = await readPrivacyRecord(workDir);
expect(record?.assets[asset.id]?.checks).toEqual([
expect.objectContaining({ kind: "mark", markId: mark.id }),
]);
const report = await gate(harness, edl, workDir, prepared.sources.get(asset.id)!.path);
expect(report.passed).toBe(true);
});
it("blocks when the treatment left the mark visible", async () => {
harness = await createPrivacyHarness((emulator) => {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.redact = async (request) => {
await copyFile(request.path, request.outPath);
return { outPath: request.outPath, framesProcessed: 50 };
};
return provider;
});
const asset = await harness.addAsset("clip.mp4");
const mark = await createMark(harness.deps, ACTOR, {
kind: "background_detail",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "blur",
});
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
const { workDir, prepared } = await render(harness, edl, "leak");
const report = await gate(harness, edl, workDir, prepared.sources.get(asset.id)!.path);
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "mark_visible", markId: mark.id }),
);
});
it("inpaints when quality holds and covers instead when it does not", async () => {
harness = await createPrivacyHarness();
const good = await harness.addAsset("good.mp4");
const poor = await harness.addAsset("poor__inpaintq-0.3__.mp4");
for (const asset of [good, poor]) {
await createMark(harness.deps, ACTOR, {
kind: "scar",
label: null,
personId: null,
assetId: asset.id,
prompt: tattoo,
treatment: "inpaint",
});
}
const edl = testEdl([
{ id: "c1", assetId: good.id, inMs: 0, outMs: 1000 },
{ id: "c2", assetId: poor.id, inMs: 0, outMs: 1000 },
]);
const { prepared } = await render(harness, edl, "inpaint");
expect(prepared.aiAlterations).toEqual(["inpaint"]);
expect(prepared.fallbacks).toEqual([
expect.stringMatching(/inpainting scored 0\.30; covered instead$/),
]);
});
it("re-finds a person's standing mark in another video relative to the face", async () => {
harness = await createPrivacyHarness();
const origin = await harness.addAsset("a__faces-1____person-alice__.mp4");
const other = await harness.addAsset("b__faces-1____person-alice__.mp4");
harness.addPerson("alice", "none");
await harness.analyseFaces(origin.id, ["alice"]);
await harness.analyseFaces(other.id, ["alice"]);
const segment = vi.spyOn(harness.inference, "segmentPrompted");
const mark = await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: "left forearm",
personId: "alice",
assetId: origin.id,
prompt: tattoo,
treatment: "blur",
});
const edl = testEdl([{ id: "c1", assetId: other.id, inMs: 0, outMs: 2000 }]);
const { workDir, prepared } = await render(harness, edl, "standing");
expect(segment).toHaveBeenCalledTimes(2);
const reprompt = segment.mock.calls[1]![0];
expect(reprompt.path).toContain("b__faces-1");
expect(reprompt.prompt.box).toBeDefined();
const record = await readPrivacyRecord(workDir);
expect(record?.reviews).toEqual([]);
expect(record?.assets[other.id]?.checks).toEqual([
expect.objectContaining({ kind: "mark", markId: mark.id }),
]);
const report = await gate(harness, edl, workDir, prepared.sources.get(other.id)!.path);
expect(report.passed).toBe(true);
});
it("sends a low-confidence re-find to review, which blocks the export", async () => {
harness = await createPrivacyHarness((emulator) => {
const provider = Object.create(emulator) as MediaInferenceProvider;
provider.segmentPrompted = async (request) => {
const result = await emulator.segmentPrompted(request);
return request.path.includes("b__") ? { ...result, score: 0.3 } : result;
};
return provider;
});
const origin = await harness.addAsset("a__faces-1____person-alice__.mp4");
const other = await harness.addAsset("b__faces-1____person-alice__.mp4");
harness.addPerson("alice", "none");
await harness.analyseFaces(origin.id, ["alice"]);
await harness.analyseFaces(other.id, ["alice"]);
const mark = await createMark(harness.deps, ACTOR, {
kind: "jewelry",
label: null,
personId: "alice",
assetId: origin.id,
prompt: tattoo,
treatment: "blur",
});
const edl = testEdl([{ id: "c1", assetId: other.id, inMs: 0, outMs: 2000 }]);
const { workDir, prepared } = await render(harness, edl, "review");
const report = await gate(harness, edl, workDir, prepared.sources.get(other.id)!.path);
expect(report.passed).toBe(false);
expect(report.issues).toContainEqual(
expect.objectContaining({ code: "mark_needs_review", markId: mark.id }),
);
});
it("removes a person with their tracks, marks, and files", async () => {
harness = await createPrivacyHarness();
const asset = await harness.addAsset("clip__faces-1____person-alice__.mp4");
harness.addPerson("alice", "blur");
await harness.analyseFaces(asset.id, ["alice"]);
await createMark(harness.deps, ACTOR, {
kind: "tattoo",
label: null,
personId: "alice",
assetId: asset.id,
prompt: tattoo,
treatment: "blur",
});
const files = [
path.join(harness.root, harness.store.tracks[0]!.keyframesPath),
path.join(harness.root, harness.store.marks[0]!.maskDir!),
];
const service = createMediaPrivacyService(harness.deps);
await service.removePerson(ACTOR, "alice");
expect(harness.store.people).toEqual([]);
expect(harness.store.tracks).toEqual([]);
for (const file of files) await expect(stat(file)).rejects.toThrow();
});
});

322
packages/adapters/src/media-marks.ts Normal file
View File

@ -0,0 +1,322 @@
import { mkdir, rm } from "node:fs/promises";
import path from "node:path";
import type { SegmentPromptedResult } from "@rakazo/adapter-kit";
import type {
BoxKeyframe,
MarkTreatment,
MediaMark,
MediaMarkPrompt,
NormalizedBox,
} from "@rakazo/contracts";
import type { CreateMarkInputType, MediaActor, MediaServiceDeps } from "./media-context.js";
import type { PrivacyToolOptions } from "./media-privacy-io.js";
import {
analysisFile,
inferenceCapabilities,
operationAvailable,
PrivacyUnavailableError,
probeSource,
resolveStored,
toStored,
} from "./media-privacy-io.js";
import { extractLuma, temporalStability } from "./media-privacy-pixels.js";
import type { TrackKeyframe } from "./media-privacy-plan.js";
import { boxAt, FACE_MIN_CONFIDENCE, MARK_QUALITY_MIN, unionBox } from "./media-privacy-plan.js";
/*
* Identifying marks (spec 7): the user marks once by click or box, the
* segmenter (SAM2 in the sidecar) propagates the mask through the video, and
* the treatment is scored. A score under MARK_QUALITY_MIN never ships
* silently: the mark carries a suggested fallback and rendering applies it.
*/
/** Side of the box a point prompt stands for, normalized. */
const POINT_PROMPT_SIZE = 0.12;
/** Frames sampled for the local flicker measurement of an inpaint preview. */
const STABILITY_FRAMES = 6;
const STABILITY_STEP_MS = 200;
export type MarkDeps = Pick<MediaServiceDeps, "repos" | "inference" | "mediaRoot" | "sources"> &
PrivacyToolOptions;
/** Directory of a mark's masks and previews, relative to the media root. */
export function markDir(spaceId: string, markId: string): string {
return path.join(spaceId, "marks", markId);
}
/** The prompt as a box: a point prompt stands for a small box around it. */
export function promptBox(prompt: MediaMarkPrompt): NormalizedBox {
if (prompt.box) return prompt.box;
if (!prompt.point) throw new PrivacyUnavailableError("a mark needs a box or a point");
const x = Math.min(Math.max(0, prompt.point.x - POINT_PROMPT_SIZE / 2), 1 - POINT_PROMPT_SIZE);
const y = Math.min(Math.max(0, prompt.point.y - POINT_PROMPT_SIZE / 2), 1 - POINT_PROMPT_SIZE);
return { x, y, w: POINT_PROMPT_SIZE, h: POINT_PROMPT_SIZE };
}
/** Fallback to propose when a treatment scores too low. */
export function fallbackFor(treatment: MarkTreatment): MarkTreatment {
return treatment === "cover" ? "reframe_out" : "cover";
}
/**
* Score a treatment. Segmentation confidence bounds every treatment (a mask
* that does not follow the mark cannot hide it); inpainting also needs the
* provider's temporal and edge quality and a local flicker check of its
* preview. Returns the score and, under MARK_QUALITY_MIN, the fallback.
*/
export async function scoreTreatment(
deps: MarkDeps,
input: {
treatment: MarkTreatment;
segmentationScore: number;
file: string;
maskDir: string;
keyframes: BoxKeyframe[];
previewDir: string;
signal?: AbortSignal;
},
): Promise<{ qualityScore: number; suggestedFallback: MarkTreatment | null }> {
let score = input.segmentationScore;
if (input.treatment === "inpaint") {
const capabilities = await inferenceCapabilities(deps.inference, input.signal);
if (!deps.inference || !operationAvailable(capabilities, "inpaintVideo")) {
return { qualityScore: 0, suggestedFallback: "cover" };
}
await mkdir(input.previewDir, { recursive: true });
const preview = await deps.inference.inpaintVideo({
path: input.file,
maskDir: input.maskDir,
outPath: path.join(input.previewDir, "inpaint-preview.mp4"),
signal: input.signal,
});
const local = await previewStability(deps, preview.outPath, input.keyframes, input.signal);
score = Math.min(score, preview.quality.score, preview.quality.temporalStability, local);
}
const qualityScore = Math.round(score * 1000) / 1000;
return {
qualityScore,
suggestedFallback:
qualityScore < MARK_QUALITY_MIN && input.treatment !== "none"
? fallbackFor(input.treatment)
: null,
};
}
/** Flicker inside the mark region of an inpainted preview around the first keyframe. */
async function previewStability(
deps: MarkDeps,
file: string,
keyframes: BoxKeyframe[],
signal?: AbortSignal,
): Promise<number> {
const first = keyframes[0];
if (!first) return 1;
const { width, height } = await frameSize(file, deps, signal);
const frames = [];
let region: NormalizedBox = first.box;
for (let i = 0; i < STABILITY_FRAMES; i++) {
const tMs = first.tMs + i * STABILITY_STEP_MS;
const box = boxAt(keyframes, tMs, STABILITY_STEP_MS);
if (box) region = unionBox(region, box);
try {
frames.push(
await extractLuma(file, { tMs, width, height }, { ffmpegPath: deps.ffmpegPath, signal }),
);
} catch {
break;
}
}
return temporalStability(frames, region);
}
async function frameSize(file: string, deps: PrivacyToolOptions, signal?: AbortSignal) {
const probed = await probeSource(file, "video", { ffprobePath: deps.ffprobePath, signal });
return { width: probed.width, height: probed.height };
}
/** Segment a prompt and propagate it through the file into `outDir`. */
export async function segmentMark(
deps: MarkDeps,
input: { file: string; prompt: MediaMarkPrompt; outDir: string; signal?: AbortSignal },
): Promise<SegmentPromptedResult> {
const capabilities = await inferenceCapabilities(deps.inference, input.signal);
if (!deps.inference || !operationAvailable(capabilities, "segmentPrompted")) {
throw new PrivacyUnavailableError("mark segmentation needs the media worker");
}
await rm(input.outDir, { recursive: true, force: true });
await mkdir(input.outDir, { recursive: true });
return deps.inference.segmentPrompted({
path: input.file,
prompt: input.prompt,
propagate: true,
outDir: input.outDir,
signal: input.signal,
});
}
/**
* Where a person-bound mark sits relative to the person's face, from the
* frame it was drawn in: offsets and size in face-box units.
*/
export interface MarkAnchor {
dx: number;
dy: number;
w: number;
h: number;
}
export function anchorFromFace(mark: NormalizedBox, face: NormalizedBox): MarkAnchor {
return {
dx: (mark.x - face.x) / face.w,
dy: (mark.y - face.y) / face.h,
w: mark.w / face.w,
h: mark.h / face.h,
};
}
/** Place an anchor at a face, clamped to the frame. Null when it falls outside. */
export function placeAnchor(anchor: MarkAnchor, face: NormalizedBox): NormalizedBox | null {
const x0 = face.x + anchor.dx * face.w;
const y0 = face.y + anchor.dy * face.h;
const x1 = x0 + anchor.w * face.w;
const y1 = y0 + anchor.h * face.h;
const x = Math.max(0, x0);
const y = Math.max(0, y0);
const w = Math.min(1, x1) - x;
const h = Math.min(1, y1) - y;
return w > 0.001 && h > 0.001 ? { x, y, w, h } : null;
}
/** First confident face keyframe of a track at or after `fromMs`. */
export function firstConfident(keyframes: TrackKeyframe[], fromMs = 0): TrackKeyframe | null {
return (
[...keyframes]
.filter((k) => k.tMs >= fromMs && (k.confidence ?? 1) >= FACE_MIN_CONFIDENCE)
.sort((a, b) => a.tMs - b.tMs)[0] ?? null
);
}
/** Face box of a track nearest to `tMs` (within 250 ms). */
export function faceAt(keyframes: TrackKeyframe[], tMs: number): NormalizedBox | null {
let best: TrackKeyframe | null = null;
for (const k of keyframes) {
if (Math.abs(k.tMs - tMs) > 250) continue;
if (!best || Math.abs(k.tMs - tMs) < Math.abs(best.tMs - tMs)) best = k;
}
return best?.box ?? null;
}
// ---------------------------------------------------------------------------
// Service operations
// ---------------------------------------------------------------------------
/**
* Create a mark: store it, segment from the prompt, propagate through the
* asset, score the treatment, and store a fallback suggestion when the score
* is low. Without the media worker the mark is kept unsegmented; rendering a
* clip of that asset then fails closed until segmentation is possible.
*/
export async function createMark(
deps: MarkDeps,
actor: MediaActor,
input: CreateMarkInputType,
signal?: AbortSignal,
): Promise<MediaMark> {
const asset = await deps.repos.getAssetRecord(actor.spaceId, input.assetId);
const mark = await deps.repos.createMark(actor.spaceId, {
kind: input.kind,
label: input.label,
personId: input.personId,
assetId: input.assetId,
prompt: input.prompt,
treatment: input.treatment,
});
const file = await analysisFile(deps, actor.spaceId, asset);
const capabilities = await inferenceCapabilities(deps.inference, signal);
if (!file || !operationAvailable(capabilities, "segmentPrompted")) return mark;
const relative = markDir(actor.spaceId, mark.id);
const outDir = await resolveStored(deps.mediaRoot, path.join(relative, "masks"));
try {
const segmented = await segmentMark(deps, { file, prompt: input.prompt, outDir, signal });
const scored = await scoreTreatment(deps, {
treatment: input.treatment,
segmentationScore: segmented.score,
file,
maskDir: segmented.maskDir,
keyframes: segmented.keyframes,
previewDir: await resolveStored(deps.mediaRoot, relative),
signal,
});
const record = await deps.repos.recordMarkResult(actor.spaceId, mark.id, {
...scored,
maskDir: await toStored(deps.mediaRoot, segmented.maskDir),
});
const { maskDir: _maskDir, ...dto } = record;
return dto;
} catch (error) {
// The mark stays without a mask (rendering fails closed); the caller sees why.
await deps.repos.removeMark(actor.spaceId, mark.id);
await rm(await resolveStored(deps.mediaRoot, relative), { recursive: true, force: true });
throw error;
}
}
/**
* Update treatment, label, or person. A new treatment is segmented and scored
* again so its suggestion reflects that treatment.
*/
export async function updateMark(
deps: MarkDeps,
actor: MediaActor,
input: {
markId: string;
treatment?: MarkTreatment;
label?: string | null;
personId?: string | null;
},
signal?: AbortSignal,
): Promise<MediaMark> {
const before = await deps.repos.getMark(actor.spaceId, input.markId);
const updated = await deps.repos.updateMark(actor.spaceId, input.markId, {
...(input.treatment !== undefined ? { treatment: input.treatment } : {}),
...(input.label !== undefined ? { label: input.label } : {}),
...(input.personId !== undefined ? { personId: input.personId } : {}),
});
if (input.treatment === undefined || input.treatment === before.treatment) return updated;
if (!before.assetId || !before.prompt) return updated;
const asset = await deps.repos.getAssetRecord(actor.spaceId, before.assetId);
const file = await analysisFile(deps, actor.spaceId, asset);
const capabilities = await inferenceCapabilities(deps.inference, signal);
if (!file || !operationAvailable(capabilities, "segmentPrompted")) {
// Without the worker the old score says nothing about the new treatment.
const record = await deps.repos.recordMarkResult(actor.spaceId, input.markId, {
qualityScore: null,
suggestedFallback: null,
maskDir: before.maskDir,
});
const { maskDir: _maskDir, ...dto } = record;
return dto;
}
const relative = markDir(actor.spaceId, input.markId);
const segmented = await segmentMark(deps, {
file,
prompt: before.prompt,
outDir: await resolveStored(deps.mediaRoot, path.join(relative, "masks")),
signal,
});
const scored = await scoreTreatment(deps, {
treatment: input.treatment,
segmentationScore: segmented.score,
file,
maskDir: segmented.maskDir,
keyframes: segmented.keyframes,
previewDir: await resolveStored(deps.mediaRoot, relative),
signal,
});
const record = await deps.repos.recordMarkResult(actor.spaceId, input.markId, {
...scored,
maskDir: await toStored(deps.mediaRoot, segmented.maskDir),
});
const { maskDir: _maskDir, ...dto } = record;
return dto;
}

437
packages/adapters/src/media-privacy-fixtures.ts Normal file
View File

@ -0,0 +1,437 @@
import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
import type {
AnonymizeMode,
Edl,
FfprobeReport,
MarkTreatment,
MediaMarkKind,
MediaMarkPrompt,
PlatformProfile,
} from "@rakazo/contracts";
import { EdlSchema, PlatformProfileSchema } from "@rakazo/contracts";
import type { MediaRepos } from "@rakazo/db";
import { edlDurationMs, encodeVectors } from "@rakazo/media";
import type { MediaServiceDeps } from "./media-context.js";
import { EmulatorMediaInferenceProvider } from "./media-emulator.js";
import { renderTestClip } from "./media-emulator-clips.js";
import { runMediaTool } from "./media-folder-source-probe.js";
/*
* In-memory repositories and synthetic ffmpeg clips for the privacy tests.
* Clip names carry media-emulator markers (faces, person seeds, detection
* gaps, classifier scores); nothing here shows a real person.
*/
export const SPACE = "space-1";
export const USER = "user-1";
export const ACTOR = { spaceId: SPACE, userId: USER };
interface AssetRow {
id: string;
spaceId: string;
sourceId: string;
externalId: string;
kind: string;
storage: string;
storagePath: string | null;
proxyPath: string | null;
}
interface PersonRow {
id: string;
spaceId: string;
displayName: string;
anonymizeByDefault: string;
consentRecordedAt: Date | null;
ageVerifiedAt: Date | null;
ageVerificationMethod: string;
syntheticSeed: number | null;
centroid: Uint8Array | null;
}
interface TrackRow {
id: string;
assetId: string;
personId: string | null;
startMs: number;
endMs: number;
confidenceMin: number;
keyframesPath: string;
modelVersion: string;
}
interface MarkRow {
id: string;
kind: MediaMarkKind;
label: string | null;
personId: string | null;
assetId: string | null;
prompt: MediaMarkPrompt | null;
treatment: MarkTreatment;
qualityScore: number | null;
suggestedFallback: MarkTreatment | null;
maskDir: string | null;
createdAt: string;
}
export interface PrivacyHarness {
root: string;
inference: EmulatorMediaInferenceProvider;
deps: MediaServiceDeps;
store: {
assets: AssetRow[];
people: PersonRow[];
tracks: TrackRow[];
marks: MarkRow[];
renders: Array<Record<string, unknown>>;
plans: Array<Record<string, unknown>>;
segments: Array<Record<string, unknown>>;
};
/** A synthetic clip, or a still (PNG) when the name ends in .png. */
addAsset(fileName: string, options?: { seconds?: number; id?: string }): Promise<AssetRow>;
addPerson(
id: string,
mode: AnonymizeMode,
consent?: { consent?: boolean; age?: boolean },
): PersonRow;
/** Run emulator tracking on the asset and store the tracks (the analysis result). */
analyseFaces(assetId: string, personIds: Array<string | null>, sampleFps?: number): Promise<void>;
/** Store the emulator identity embedding of a person seed as the person's centroid. */
learnPerson(personId: string, assetId: string, trackIndex?: number): Promise<void>;
workDir(name: string): Promise<string>;
cleanup(): Promise<void>;
}
export function testEdl(
clips: Array<{ id: string; assetId: string; inMs: number; outMs: number; atMs?: number }>,
extra: Partial<Pick<Edl, "redactions">> & { width?: number; height?: number } = {},
): Edl {
let cursor = 0;
const video = clips.map((clip) => {
const atMs = clip.atMs ?? cursor;
cursor = atMs + clip.outMs - clip.inMs;
return { ...clip, atMs, reframe: "none" as const };
});
return EdlSchema.parse({
edlVersion: 1,
output: {
profileId: "test-profile",
width: extra.width ?? 320,
height: extra.height ?? 180,
fps: 25,
durationTargetMs: cursor,
durationHardMinMs: 0,
durationHardMaxMs: 600_000,
},
tracks: { video },
redactions: extra.redactions ?? {},
metadata: {},
});
}
export function testProfile(overrides: Record<string, unknown> = {}): PlatformProfile {
return PlatformProfileSchema.parse({
id: "test-profile",
name: "Test profile",
kind: "free_social",
reviewedAt: "2026-01-01",
video: {
aspectRatios: ["16:9", "9:16"],
width: 1920,
height: 1920,
fps: 60,
codec: "h264",
maxBitrateKbps: 20_000,
maxFileSizeMb: 500,
},
duration: { hardMinMs: 0, hardMaxMs: 600_000 },
safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 },
captions: { recommended: false },
audio: {},
text: { captionMaxChars: 2200, hashtagLimit: 30, linkPolicy: "allowed" },
content: { thresholds: { explicit_nudity: 0.5 } },
disclosure: { aiAlteredLabelRequired: false },
consent: { requirePerformerConsent: false, requireAgeVerification: false },
export: {},
...overrides,
});
}
/** An ffprobe report that passes the technical check for an EDL. */
export function cleanProbe(edl: Edl): FfprobeReport {
return {
durationMs: edlDurationMs(edl),
width: edl.output.width,
height: edl.output.height,
fps: edl.output.fps,
videoCodec: "h264",
audioCodec: null,
sizeBytes: 10_000,
integratedLufs: null,
truePeakDb: null,
tags: {},
};
}
function isolation(): Error {
return new Error("not found in space");
}
export async function createPrivacyHarness(
inferenceOverride?: (emulator: EmulatorMediaInferenceProvider) => MediaInferenceProvider | null,
): Promise<PrivacyHarness> {
const root = await realpath(await mkdtemp(path.join(tmpdir(), "media-privacy-")));
const emulator = new EmulatorMediaInferenceProvider({ root });
const store: PrivacyHarness["store"] = {
assets: [],
people: [],
tracks: [],
marks: [],
renders: [],
plans: [],
segments: [],
};
let counter = 0;
const nextId = (prefix: string) => `${prefix}${++counter}`;
const markDto = (row: MarkRow) => {
const { maskDir: _maskDir, ...dto } = row;
return dto;
};
const findMark = (id: string) => {
const row = store.marks.find((mark) => mark.id === id);
if (!row) throw isolation();
return row;
};
const repos = {
async listAssetRecords(_spaceId: string, ids: string[]) {
return store.assets.filter((asset) => ids.includes(asset.id));
},
async getAssetRecord(_spaceId: string, id: string) {
const asset = store.assets.find((row) => row.id === id);
if (!asset) throw isolation();
return asset;
},
async getSource() {
return { kind: "folder", config: {} };
},
async getProjectRecord(_spaceId: string, projectId: string) {
return {
id: projectId,
brandKitId: null,
assetIds: store.assets.map((a) => a.id),
};
},
async getBrandKit() {
throw isolation();
},
async listPersonRecords(_spaceId: string, ids?: string[]) {
return store.people.filter((person) => !ids || ids.includes(person.id));
},
async personCentroids() {
return store.people.flatMap((person) =>
person.centroid ? [{ personId: person.id, centroid: person.centroid }] : [],
);
},
async updatePerson(_spaceId: string, personId: string, patch: { syntheticSeed?: number }) {
const person = store.people.find((row) => row.id === personId);
if (!person) throw isolation();
if (patch.syntheticSeed !== undefined) person.syntheticSeed = patch.syntheticSeed;
return person;
},
async faceTrackRecords(_spaceId: string, assetIds: string[]) {
return store.tracks.filter((track) => assetIds.includes(track.assetId));
},
async listFaceTracks() {
return store.tracks;
},
async latestAnalysis(_spaceId: string, assetId: string, kind: string) {
return kind === "faces" && store.tracks.some((t) => t.assetId === assetId) ? { kind } : null;
},
async marksForAsset(_spaceId: string, assetId: string) {
const peopleHere = new Set(
store.tracks.filter((t) => t.assetId === assetId && t.personId).map((t) => t.personId),
);
return store.marks.filter(
(mark) => mark.assetId === assetId || (mark.personId && peopleHere.has(mark.personId)),
);
},
async createMark(
_spaceId: string,
input: Omit<MarkRow, "id" | "qualityScore" | "suggestedFallback" | "maskDir" | "createdAt">,
) {
const row: MarkRow = {
...input,
id: nextId("mark"),
qualityScore: null,
suggestedFallback: null,
maskDir: null,
createdAt: new Date(0).toISOString(),
};
store.marks.push(row);
return markDto(row);
},
async getMark(_spaceId: string, id: string) {
return { ...findMark(id) };
},
async updateMark(_spaceId: string, id: string, patch: Partial<MarkRow>) {
Object.assign(findMark(id), patch);
return markDto(findMark(id));
},
async recordMarkResult(
_spaceId: string,
id: string,
outcome: Pick<MarkRow, "qualityScore" | "suggestedFallback" | "maskDir">,
) {
Object.assign(findMark(id), outcome);
return { ...findMark(id) };
},
async removeMark(_spaceId: string, id: string) {
const row = findMark(id);
store.marks = store.marks.filter((mark) => mark.id !== id);
return { maskDir: row.maskDir };
},
async removePersonCompletely(_spaceId: string, personId: string) {
const tracks = store.tracks.filter((t) => t.personId === personId);
const marks = store.marks.filter((m) => m.personId === personId);
store.tracks = store.tracks.filter((t) => t.personId !== personId);
store.marks = store.marks.filter((m) => m.personId !== personId);
store.people = store.people.filter((p) => p.id !== personId);
return {
deletedFaceTracks: tracks.length,
deletedMarks: marks.length,
keyframesPaths: tracks.map((t) => t.keyframesPath),
maskDirs: marks.flatMap((m) => (m.maskDir ? [m.maskDir] : [])),
};
},
async getRenderRecord(_spaceId: string, id: string) {
const render = store.renders.find((row) => row.id === id);
if (!render) throw isolation();
return render;
},
async getPlan(_spaceId: string, id: string) {
const plan = store.plans.find((row) => row.id === id);
if (!plan) throw isolation();
return plan;
},
async listSegments() {
return store.segments;
},
async updateRender(_spaceId: string, id: string, patch: Record<string, unknown>) {
const render = store.renders.find((row) => row.id === id);
if (!render) return null;
Object.assign(render, patch);
return render;
},
};
const libraryDir = path.join(root, SPACE, "library");
await mkdir(libraryDir, { recursive: true });
const deps = {
repos: repos as unknown as MediaRepos,
inference: inferenceOverride ? inferenceOverride(emulator) : emulator,
mediaRoot: root,
sources: {},
now: () => new Date("2026-06-01T12:00:00Z"),
} as unknown as MediaServiceDeps;
const harness: PrivacyHarness = {
root,
inference: emulator,
deps,
store,
async addAsset(fileName, options = {}) {
const file = path.join(libraryDir, fileName);
const still = fileName.endsWith(".png");
if (still) {
await runMediaTool("ffmpeg", [
"-v",
"error",
"-y",
"-f",
"lavfi",
"-i",
"testsrc2=s=320x180:r=1:d=1",
"-frames:v",
"1",
file,
]);
} else {
await renderTestClip(file, { seconds: options.seconds ?? 2 });
}
const row: AssetRow = {
id: options.id ?? nextId("asset"),
spaceId: SPACE,
sourceId: "source-1",
externalId: fileName,
kind: still ? "photo" : "video",
storage: "uploaded",
storagePath: path.relative(root, file),
proxyPath: null,
};
store.assets.push(row);
return row;
},
addPerson(id, mode, consent = {}) {
const at = new Date("2026-01-01T00:00:00Z");
const row: PersonRow = {
id,
spaceId: SPACE,
displayName: id,
anonymizeByDefault: mode,
consentRecordedAt: consent.consent ? at : null,
ageVerifiedAt: consent.age ? at : null,
ageVerificationMethod: consent.age ? "document_on_file" : "none",
syntheticSeed: null,
centroid: null,
};
store.people.push(row);
return row;
},
async analyseFaces(assetId, personIds, sampleFps = 10) {
const asset = store.assets.find((row) => row.id === assetId)!;
const tracked = await emulator.trackFaces({
path: path.join(root, asset.storagePath!),
sampleFps,
});
const dir = path.join(root, SPACE, "faces", assetId);
await mkdir(dir, { recursive: true });
for (const [index, track] of tracked.tracks.entries()) {
const file = path.join(dir, `${track.trackId}.json`);
await writeFile(file, JSON.stringify({ keyframes: track.keyframes }));
const times = track.keyframes.map((k) => k.tMs);
store.tracks.push({
id: nextId("track"),
assetId,
personId: personIds[index] ?? null,
startMs: Math.min(...times),
endMs: Math.max(...times),
confidenceMin: 0.9,
keyframesPath: path.relative(root, file),
modelVersion: "emulator",
});
}
},
async learnPerson(personId, assetId, trackIndex = 0) {
const asset = store.assets.find((row) => row.id === assetId)!;
const tracked = await emulator.trackFaces({
path: path.join(root, asset.storagePath!),
sampleFps: 2,
});
const embedding = tracked.tracks[trackIndex]?.embedding;
const person = store.people.find((row) => row.id === personId)!;
person.centroid = embedding ? encodeVectors([embedding]) : null;
},
async workDir(name) {
const dir = path.join(root, SPACE, "renders", name);
await mkdir(dir, { recursive: true });
return dir;
},
async cleanup() {
await rm(root, { recursive: true, force: true });
},
};
return harness;
}

264
packages/adapters/src/media-privacy-io.ts Normal file
View File

@ -0,0 +1,264 @@
import { constants } from "node:fs";
import { copyFile, mkdir, readdir, readFile, realpath } from "node:fs/promises";
import path from "node:path";
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
import type {
BoxKeyframe,
MediaCapabilities,
MediaInferenceOperation,
MediaRenderQuality,
} from "@rakazo/contracts";
import type { MediaRepos } from "@rakazo/db";
import { assertInside, resolveInside } from "@rakazo/media/node";
import type { MediaServiceDeps } from "./media-context.js";
import { probeMedia, runMediaTool } from "./media-folder-source-probe.js";
import { maskBounds, readPngLuma } from "./media-privacy-pixels.js";
import type { PixelBox, TrackKeyframe } from "./media-privacy-plan.js";
import { parseStoredKeyframes } from "./media-privacy-plan.js";
/*
* File and tool helpers shared by anonymization, marks, and the gate. Stored
* media paths are relative to the media root; every resolution goes through
* the root checks of @rakazo/media/node. ffmpeg runs as an argv array only.
*/
/** A redaction or check that cannot be carried out. Rendering stops instead of exporting. */
export class PrivacyUnavailableError extends Error {
constructor(message: string) {
super(message);
this.name = "PrivacyUnavailableError";
}
}
export interface PrivacyToolOptions {
ffmpegPath?: string;
ffprobePath?: string;
}
/** Absolute path of a stored (root-relative) or already absolute path inside the media root. */
export async function resolveStored(root: string, stored: string): Promise<string> {
return path.isAbsolute(stored) ? assertInside(root, stored) : resolveInside(root, stored);
}
export async function isInsideRoot(root: string, file: string): Promise<boolean> {
try {
await assertInside(root, file);
return true;
} catch {
return false;
}
}
/** The path of a file relative to the media root, for storing. */
export async function toStored(root: string, absolute: string): Promise<string> {
const inside = await assertInside(root, absolute);
return path.relative(await realpath(root), inside);
}
type AssetRow = Awaited<ReturnType<MediaRepos["getAssetRecord"]>>;
/**
* The file a render reads for an asset: the 720p proxy for drafts (when one
* exists), otherwise the original. Originals still on a device cannot render.
*/
export async function resolveAssetFile(
deps: Pick<MediaServiceDeps, "repos" | "sources" | "mediaRoot">,
spaceId: string,
asset: AssetRow,
quality: MediaRenderQuality,
): Promise<string> {
if (quality === "draft" && asset.proxyPath) return resolveStored(deps.mediaRoot, asset.proxyPath);
return resolveOriginal(deps, spaceId, asset);
}
export async function resolveOriginal(
deps: Pick<MediaServiceDeps, "repos" | "sources" | "mediaRoot">,
spaceId: string,
asset: AssetRow,
): Promise<string> {
if (asset.storage === "uploaded") {
if (!asset.storagePath) throw new PrivacyUnavailableError(`asset ${asset.id} has no file yet`);
return resolveStored(deps.mediaRoot, asset.storagePath);
}
if (asset.storage === "referenced") {
const source = await deps.repos.getSource(spaceId, asset.sourceId);
const provider = deps.sources[source.kind as "folder" | "emulator"];
if (!provider) throw new PrivacyUnavailableError(`source of asset ${asset.id} is unavailable`);
return provider.resolve(source.config, asset.externalId);
}
throw new PrivacyUnavailableError(`asset ${asset.id} is still on a device; upload it first`);
}
/**
* A file inference can read: the proxy when it exists (normalized coordinates
* and the timeline match the original), else the original when it lies inside
* the media root. Null when neither does.
*/
export async function analysisFile(
deps: Pick<MediaServiceDeps, "repos" | "sources" | "mediaRoot">,
spaceId: string,
asset: AssetRow,
): Promise<string | null> {
if (asset.proxyPath) return resolveStored(deps.mediaRoot, asset.proxyPath);
try {
const original = await resolveOriginal(deps, spaceId, asset);
return (await isInsideRoot(deps.mediaRoot, original)) ? original : null;
} catch {
return null;
}
}
export interface ProbedSource {
width: number;
height: number;
durationMs: number;
fps: number;
hasAudio: boolean;
rotation: 0 | 90 | 180 | 270;
still: boolean;
}
/** Display size (rotation applied), duration, and audio of a source file. */
export async function probeSource(
file: string,
kind: string,
options: PrivacyToolOptions & { signal?: AbortSignal } = {},
): Promise<ProbedSource> {
const probed = await probeMedia(file, {
ffprobePath: options.ffprobePath,
signal: options.signal,
});
if (!probed.hasVideo || !probed.width || !probed.height)
throw new PrivacyUnavailableError("source has no readable video stream");
const still = kind === "photo" || !probed.durationMs;
const rotation = (((Math.round(probed.rotation / 90) * 90) % 360) + 360) % 360;
return {
width: probed.width,
height: probed.height,
durationMs: still ? 0 : (probed.durationMs ?? 0),
fps: probed.fps ?? 30,
hasAudio: probed.hasAudio,
rotation: rotation as ProbedSource["rotation"],
still,
};
}
/** Copy a file into `dir` under its own name (reflink when the filesystem allows). */
export async function copyInto(file: string, dir: string): Promise<string> {
await mkdir(dir, { recursive: true });
const target = path.join(dir, path.basename(file));
await copyFile(file, target, constants.COPYFILE_FICLONE);
return target;
}
/** Keyframes of a stored face track file (relative to the media root). */
export async function readTrackKeyframes(root: string, stored: string): Promise<TrackKeyframe[]> {
const file = await resolveStored(root, stored);
return parseStoredKeyframes(JSON.parse(await readFile(file, "utf8")));
}
/** Mask frame times of a mask directory, ascending. */
export async function maskTimes(maskDir: string): Promise<number[]> {
const names = await readdir(maskDir);
return names
.filter((name) => /^\d+\.png$/.test(name))
.map((name) => Number.parseInt(name, 10))
.sort((a, b) => a - b);
}
/** Bounding box per mask frame. Empty masks are skipped. */
export async function maskKeyframes(maskDir: string): Promise<BoxKeyframe[]> {
const keyframes: BoxKeyframe[] = [];
for (const tMs of await maskTimes(maskDir)) {
const box = maskBounds(await readPngLuma(path.join(maskDir, `${tMs}.png`)));
if (box) keyframes.push({ tMs, box });
}
return keyframes;
}
/** Whether an inference operation is available; synthetic identity also needs a commercial license. */
export function operationAvailable(
capabilities: MediaCapabilities | null,
operation: MediaInferenceOperation,
): boolean {
const entry = capabilities?.operations[operation];
if (!entry?.available) return false;
if (operation === "synthesizeIdentity") return entry.commercialUse === true;
return true;
}
export async function inferenceCapabilities(
inference: MediaInferenceProvider | null,
signal?: AbortSignal,
): Promise<MediaCapabilities | null> {
return inference ? inference.capabilities(signal) : null;
}
/**
* Lossless static crop of a video (or a still to PNG). Audio is copied; all
* container metadata is dropped.
*/
export async function cropFile(
input: string,
output: string,
crop: PixelBox,
still: boolean,
options: PrivacyToolOptions & { signal?: AbortSignal } = {},
): Promise<void> {
const filter = `crop=${crop.w}:${crop.h}:${crop.x}:${crop.y}`;
const args = still
? [
"-v",
"error",
"-y",
"-i",
input,
"-vf",
filter,
"-frames:v",
"1",
"-map_metadata",
"-1",
output,
]
: [
"-v",
"error",
"-y",
"-i",
input,
"-map",
"0:v:0",
"-map",
"0:a?",
"-vf",
filter,
"-c:v",
"libx264",
"-qp",
"0",
"-preset",
"ultrafast",
"-pix_fmt",
"yuv420p",
"-c:a",
"copy",
"-map_metadata",
"-1",
output,
];
await runMediaTool(options.ffmpegPath ?? "ffmpeg", args, { signal: options.signal });
}
/** First frame of a video as PNG (redaction providers write videos, stills render from images). */
export async function firstFrame(
input: string,
output: string,
options: PrivacyToolOptions & { signal?: AbortSignal } = {},
): Promise<void> {
await runMediaTool(
options.ffmpegPath ?? "ffmpeg",
["-v", "error", "-y", "-i", input, "-frames:v", "1", "-map_metadata", "-1", output],
{ signal: options.signal },
);
}

104
packages/adapters/src/media-privacy-pixels.test.ts Normal file
View File

@ -0,0 +1,104 @@
import { describe, expect, it } from "vitest";
import { encodeMaskPng } from "./media-emulator.js";
import type { LumaFrame } from "./media-privacy-pixels.js";
import {
controlAgreement,
decodePngLuma,
maskBounds,
residualVisibility,
temporalStability,
} from "./media-privacy-pixels.js";
/** A 64x64 checkerboard of 4 px squares: detail everywhere. */
function checker(width = 64, height = 64, shift = 0): LumaFrame {
const luma = new Uint8Array(width * height);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
luma[y * width + x] = (Math.floor((x + shift) / 4) + Math.floor(y / 4)) % 2 ? 220 : 30;
}
}
return { width, height, luma };
}
function fill(
frame: LumaFrame,
box: { x: number; y: number; w: number; h: number },
value: number,
) {
const out = { ...frame, luma: frame.luma.slice() };
for (
let y = Math.floor(box.y * frame.height);
y < Math.ceil((box.y + box.h) * frame.height);
y++
) {
for (
let x = Math.floor(box.x * frame.width);
x < Math.ceil((box.x + box.w) * frame.width);
x++
) {
out.luma[y * frame.width + x] = value;
}
}
return out;
}
const region = { x: 0.25, y: 0.25, w: 0.5, h: 0.5 };
describe("PNG masks", () => {
it("decodes the masks the segmenter writes", () => {
const mask = decodePngLuma(encodeMaskPng(40, 20, { x: 0.5, y: 0.25, w: 0.25, h: 0.5 }));
expect(mask).toMatchObject({ width: 40, height: 20 });
expect(maskBounds(mask)).toEqual({ x: 0.5, y: 0.25, w: 0.25, h: 0.5 });
expect(maskBounds(decodePngLuma(encodeMaskPng(8, 8, { x: 0, y: 0, w: 0, h: 0 })))).toBeNull();
expect(() => decodePngLuma(Buffer.from("nope"))).toThrow("not a PNG");
});
});
describe("residual visibility", () => {
it("is 1 for untouched detail and 0 for a filled or flat region", () => {
const source = checker();
expect(residualVisibility({ source, treated: source, box: region })).toBe(1);
expect(residualVisibility({ source, treated: fill(source, region, 128), box: region })).toBe(0);
const flat = fill(source, region, 90);
expect(
residualVisibility({ source: flat, treated: flat, box: { x: 0.3, y: 0.3, w: 0.4, h: 0.4 } }),
).toBe(0);
});
it("counts only the part a crop still shows and only mask pixels", () => {
const source = checker();
// The treated frame is the right half of the source, untouched.
const half: LumaFrame = { width: 32, height: 64, luma: new Uint8Array(32 * 64) };
for (let y = 0; y < 64; y++)
half.luma.set(source.luma.subarray(y * 64 + 32, y * 64 + 64), y * 32);
const left = { x: 0, y: 0.25, w: 0.4, h: 0.5 };
expect(residualVisibility({ source, treated: half, box: left, offset: { x: 32, y: 0 } })).toBe(
0,
);
expect(
residualVisibility({ source, treated: half, box: region, offset: { x: 32, y: 0 } }),
).toBe(1);
const mask = decodePngLuma(encodeMaskPng(64, 64, { x: 0.5, y: 0.25, w: 0.25, h: 0.5 }));
const treated = fill(source, { x: 0.25, y: 0.25, w: 0.25, h: 0.5 }, 128);
expect(residualVisibility({ source, treated, box: region, mask })).toBe(1);
});
it("detects frames that are not the same moment", () => {
const source = checker();
const treated = fill(source, region, 128);
expect(controlAgreement({ source, treated, boxes: [region] })).toBe(1);
expect(controlAgreement({ source, treated: checker(64, 64, 2), boxes: [region] })).toBeLessThan(
0.8,
);
});
});
describe("temporal stability", () => {
it("drops when the treated region flickers more than the frame", () => {
const a = checker();
expect(temporalStability([a, a, a], region)).toBe(1);
const flicker = [fill(a, region, 0), fill(a, region, 255), fill(a, region, 0)];
expect(temporalStability(flicker, region)).toBe(0);
expect(temporalStability([a], region)).toBe(1);
});
});

310
packages/adapters/src/media-privacy-pixels.ts Normal file
View File

@ -0,0 +1,310 @@
import { readFile } from "node:fs/promises";
import { inflateSync } from "node:zlib";
import type { NormalizedBox } from "@rakazo/contracts";
import { runMediaTool } from "./media-folder-source-probe.js";
/*
* Pixel checks for the privacy gate and mark quality. Video frames are decoded
* by ffmpeg straight to 8-bit luma; mask PNGs from the segmenter go through the
* decoder below (8-bit gray, gray+alpha, RGB, RGBA; not interlaced, not palette).
*/
export interface LumaFrame {
width: number;
height: number;
/** One luma byte per pixel, row-major. */
luma: Uint8Array;
}
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
const CHANNELS: Record<number, number> = { 0: 1, 2: 3, 4: 2, 6: 4 };
function paeth(a: number, b: number, c: number): number {
const p = a + b - c;
const pa = Math.abs(p - a);
const pb = Math.abs(p - b);
const pc = Math.abs(p - c);
if (pa <= pb && pa <= pc) return a;
return pb <= pc ? b : c;
}
/** Decode a PNG to luma (Rec. 601 weights). Throws on anything it cannot read exactly. */
export function decodePngLuma(png: Buffer): LumaFrame {
if (png.length < 8 || !png.subarray(0, 8).equals(PNG_SIGNATURE)) throw new Error("not a PNG");
let offset = 8;
let width = 0;
let height = 0;
let channels = 0;
const data: Buffer[] = [];
while (offset + 8 <= png.length) {
const length = png.readUInt32BE(offset);
const type = png.toString("ascii", offset + 4, offset + 8);
const body = png.subarray(offset + 8, offset + 8 + length);
offset += 12 + length;
if (type === "IHDR") {
width = body.readUInt32BE(0);
height = body.readUInt32BE(4);
const depth = body[8];
const colorType = body[9] ?? -1;
const interlace = body[12];
channels = CHANNELS[colorType] ?? 0;
if (depth !== 8 || channels === 0 || interlace !== 0)
throw new Error("unsupported PNG layout");
} else if (type === "IDAT") {
data.push(body);
} else if (type === "IEND") {
break;
}
}
if (width === 0 || height === 0 || channels === 0) throw new Error("PNG has no image header");
const raw = inflateSync(Buffer.concat(data));
const stride = width * channels;
if (raw.length < (stride + 1) * height) throw new Error("PNG data is truncated");
const pixels = Buffer.alloc(stride * height);
for (let y = 0; y < height; y++) {
const filter = raw[y * (stride + 1)];
const line = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1));
const row = y * stride;
for (let x = 0; x < stride; x++) {
const left = x >= channels ? (pixels[row + x - channels] ?? 0) : 0;
const up = y > 0 ? (pixels[row - stride + x] ?? 0) : 0;
const upLeft = y > 0 && x >= channels ? (pixels[row - stride + x - channels] ?? 0) : 0;
const value = line[x] ?? 0;
let out: number;
if (filter === 0) out = value;
else if (filter === 1) out = value + left;
else if (filter === 2) out = value + up;
else if (filter === 3) out = value + ((left + up) >> 1);
else if (filter === 4) out = value + paeth(left, up, upLeft);
else throw new Error("unknown PNG filter");
pixels[row + x] = out & 0xff;
}
}
const luma = new Uint8Array(width * height);
for (let i = 0; i < width * height; i++) {
const base = i * channels;
if (channels <= 2) {
luma[i] = pixels[base] ?? 0;
} else {
const r = pixels[base] ?? 0;
const g = pixels[base + 1] ?? 0;
const b = pixels[base + 2] ?? 0;
luma[i] = Math.round(0.299 * r + 0.587 * g + 0.114 * b);
}
}
return { width, height, luma };
}
export async function readPngLuma(file: string): Promise<LumaFrame> {
return decodePngLuma(await readFile(file));
}
/**
* Decode the frame shown at `tMs` as luma, scaled to `width` x `height` with
* nearest-neighbour sampling (a no-op at the display size). Throws unless
* ffmpeg returns a full frame.
*/
export async function extractLuma(
file: string,
input: { tMs: number; width: number; height: number; still?: boolean },
options: { ffmpegPath?: string; signal?: AbortSignal } = {},
): Promise<LumaFrame> {
const seek = input.still ? [] : ["-ss", (Math.max(0, input.tMs) / 1000).toFixed(3)];
const { stdout } = await runMediaTool(
options.ffmpegPath ?? "ffmpeg",
[
"-v",
"error",
...seek,
"-i",
file,
"-frames:v",
"1",
"-vf",
`scale=${input.width}:${input.height}:flags=neighbor`,
"-f",
"rawvideo",
"-pix_fmt",
"gray",
"-",
],
{ signal: options.signal },
);
const size = input.width * input.height;
if (stdout.length < size) throw new Error(`no frame at ${input.tMs} ms`);
return {
width: input.width,
height: input.height,
luma: new Uint8Array(stdout.subarray(0, size)),
};
}
/** Bounding box of the bright (>= 128) pixels of a mask, or null when the mask is empty. */
export function maskBounds(mask: LumaFrame): NormalizedBox | null {
let minX = mask.width;
let minY = mask.height;
let maxX = -1;
let maxY = -1;
for (let y = 0; y < mask.height; y++) {
for (let x = 0; x < mask.width; x++) {
if ((mask.luma[y * mask.width + x] ?? 0) < 128) continue;
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
}
if (maxX < 0) return null;
return {
x: minX / mask.width,
y: minY / mask.height,
w: (maxX + 1 - minX) / mask.width,
h: (maxY + 1 - minY) / mask.height,
};
}
/** A luma step at least this large marks a detail pixel (an edge someone could recognise). */
export const DETAIL_MIN_STEP = 12;
/** A detail pixel within this many luma levels of the source counts as unchanged. */
export const UNCHANGED_MAX_DIFF = 2;
/** Looser tolerance for the control area outside treated regions (codec noise). */
const CONTROL_MAX_DIFF = 6;
/** Regions with fewer detail pixels show nothing identifying. */
const MIN_DETAIL_PIXELS = 16;
function pixelRange(box: NormalizedBox, width: number, height: number) {
return {
x0: Math.max(0, Math.floor(box.x * width)),
y0: Math.max(0, Math.floor(box.y * height)),
x1: Math.min(width, Math.ceil((box.x + box.w) * width)),
y1: Math.min(height, Math.ceil((box.y + box.h) * height)),
};
}
function detailStep(frame: LumaFrame, x: number, y: number): number {
const at = y * frame.width + x;
const value = frame.luma[at] ?? 0;
const right = x + 1 < frame.width ? (frame.luma[at + 1] ?? value) : value;
const down = y + 1 < frame.height ? (frame.luma[at + frame.width] ?? value) : value;
return Math.abs(value - right) + Math.abs(value - down);
}
/**
* Residual visibility of a region (0..1): the share of the source's detail
* pixels (edges) that the treated frame left unchanged, value and edge
* strength alike. Blur, pixelation, fills, and inpainting all rewrite edges,
* so a treated region scores near 0, an untouched one 1. A flat region
* without detail scores 0: there is nothing to recognise.
*
* `source` and `treated` are pixel-aligned at full resolution; when `treated`
* is a crop of the source, `offset` is the crop origin in source pixels and
* detail outside the crop is invisible. Optional `mask` (normalized over the
* source frame) limits the region to mask pixels.
*/
export function residualVisibility(input: {
source: LumaFrame;
treated: LumaFrame;
box: NormalizedBox;
offset?: { x: number; y: number };
mask?: LumaFrame;
}): number {
const { source, treated, box, mask } = input;
const offset = input.offset ?? { x: 0, y: 0 };
const { x0, y0, x1, y1 } = pixelRange(box, source.width, source.height);
let detail = 0;
let unchanged = 0;
for (let y = y0; y < y1; y++) {
for (let x = x0; x < x1; x++) {
if (mask) {
const mx = Math.min(mask.width - 1, Math.floor((x / source.width) * mask.width));
const my = Math.min(mask.height - 1, Math.floor((y / source.height) * mask.height));
if ((mask.luma[my * mask.width + mx] ?? 0) < 128) continue;
}
const step = detailStep(source, x, y);
if (step < DETAIL_MIN_STEP) continue;
const tx = x - offset.x;
const ty = y - offset.y;
if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue;
detail += 1;
const value = source.luma[y * source.width + x] ?? 0;
// Visible means the pixel and its edge survived, not a value that happens to match.
if (
Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= UNCHANGED_MAX_DIFF &&
Math.abs(step - detailStep(treated, tx, ty)) <= 2 * UNCHANGED_MAX_DIFF
)
unchanged += 1;
}
}
return detail < MIN_DETAIL_PIXELS ? 0 : unchanged / detail;
}
/**
* Share of detail pixels outside every treated box that the treated frame kept
* (within codec tolerance). A low value means the two frames are not the same
* moment or framing, so a residual measured between them proves nothing.
* Returns 1 when the control area has too little detail to judge.
*/
export function controlAgreement(input: {
source: LumaFrame;
treated: LumaFrame;
boxes: NormalizedBox[];
offset?: { x: number; y: number };
}): number {
const { source, treated, boxes } = input;
const offset = input.offset ?? { x: 0, y: 0 };
const ranges = boxes.map((box) => pixelRange(box, source.width, source.height));
let detail = 0;
let kept = 0;
for (let y = 0; y < source.height; y++) {
for (let x = 0; x < source.width; x++) {
const tx = x - offset.x;
const ty = y - offset.y;
if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue;
if (ranges.some((r) => x >= r.x0 - 2 && x < r.x1 + 2 && y >= r.y0 - 2 && y < r.y1 + 2))
continue;
if (detailStep(source, x, y) < DETAIL_MIN_STEP) continue;
detail += 1;
const value = source.luma[y * source.width + x] ?? 0;
if (Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= CONTROL_MAX_DIFF)
kept += 1;
}
}
return detail < MIN_DETAIL_PIXELS ? 1 : kept / detail;
}
/**
* Temporal stability of a treated region over consecutive frames (0..1):
* 1 when the region changes no more between neighbours than the rest of the
* frame, falling to 0 as it flickers 24+ luma levels more on average.
*/
export function temporalStability(frames: LumaFrame[], box: NormalizedBox): number {
if (frames.length < 2) return 1;
let excess = 0;
for (let i = 1; i < frames.length; i++) {
const a = frames[i - 1]!;
const b = frames[i]!;
if (a.width !== b.width || a.height !== b.height) throw new Error("frame sizes differ");
const r = pixelRange(box, a.width, a.height);
let inside = 0;
let insideCount = 0;
let outside = 0;
let outsideCount = 0;
for (let y = 0; y < a.height; y++) {
for (let x = 0; x < a.width; x++) {
const at = y * a.width + x;
const diff = Math.abs((a.luma[at] ?? 0) - (b.luma[at] ?? 0));
if (x >= r.x0 && x < r.x1 && y >= r.y0 && y < r.y1) {
inside += diff;
insideCount += 1;
} else {
outside += diff;
outsideCount += 1;
}
}
}
const meanInside = insideCount ? inside / insideCount : 0;
const meanOutside = outsideCount ? outside / outsideCount : 0;
excess += Math.max(0, meanInside - meanOutside);
}
return Math.max(0, Math.min(1, 1 - excess / (frames.length - 1) / 24));
}

210
packages/adapters/src/media-privacy-plan.test.ts Normal file
View File

@ -0,0 +1,210 @@
import type { FaceRedaction } from "@rakazo/contracts";
import { encodeVectors } from "@rakazo/media";
import { describe, expect, it } from "vitest";
import { mergeTracks, syntheticSeedFor } from "./media-anonymize.js";
import { testEdl } from "./media-privacy-fixtures.js";
import {
boxAt,
cropPixels,
effectiveModes,
ensureMinSize,
GAP_HOLD_GROWTH,
matchPerson,
outsideCrop,
padBox,
parseStoredKeyframes,
protectKeyframes,
resolveProtection,
sourceToOutput,
} from "./media-privacy-plan.js";
const box = { x: 0.4, y: 0.4, w: 0.1, h: 0.1 };
function face(
id: string,
target: FaceRedaction["target"],
mode: FaceRedaction["mode"],
): FaceRedaction {
return { id, target, mode, paddingPct: 20, featherPct: 10, minSizePct: 2 };
}
describe("protectKeyframes", () => {
it("interpolates short gaps and holds a grown box over long ones for review", () => {
const result = protectKeyframes(
[
{ tMs: 0, box },
{ tMs: 400, box: { ...box, x: 0.44 } },
{ tMs: 1400, box: { ...box, x: 0.5 } },
],
[{ startMs: 0, endMs: 2000 }],
2,
);
expect(result.gaps).toEqual([{ startMs: 400, endMs: 1400 }]);
expect(result.keyframes.map((k) => k.tMs)).toEqual([0, 100, 200, 300, 400, 401, 1399, 1400]);
const held = result.keyframes.find((k) => k.tMs === 401)!.box;
expect(held.w).toBeCloseTo(0.1 * (1 + 2 * GAP_HOLD_GROWTH), 3);
});
it("treats weak detections as lost but still covers them", () => {
const result = protectKeyframes(
[
{ tMs: 0, box, confidence: 0.9 },
{ tMs: 400, box: { ...box, x: 0.8 }, confidence: 0.2 },
{ tMs: 800, box, confidence: 0.9 },
],
[{ startMs: 0, endMs: 1000 }],
2,
);
expect(result.gaps).toEqual([{ startMs: 0, endMs: 800 }]);
const held = result.keyframes.find((k) => k.tMs === 1)!.box;
expect(held.x + held.w).toBeGreaterThanOrEqual(0.9);
});
it("ignores detections far outside the clip ranges and enforces a minimum size", () => {
const result = protectKeyframes(
[
{ tMs: 0, box: { x: 0.5, y: 0.5, w: 0.001, h: 0.001 } },
{ tMs: 5000, box },
],
[{ startMs: 0, endMs: 100 }],
5,
);
expect(result.keyframes).toHaveLength(1);
expect(result.keyframes[0]!.box.w).toBeGreaterThanOrEqual(0.05);
expect(ensureMinSize({ x: 0.99, y: 0.99, w: 0.01, h: 0.01 }, 10)).toEqual({
x: 0.9,
y: 0.9,
w: 0.1,
h: 0.1,
});
});
});
describe("protection", () => {
const defaults = new Map([
["alice", "pixelate" as const],
["bob", "none" as const],
]);
const faces = [
face("by-track", { trackId: "t1" }, "solid"),
face("by-person", { personId: "bob" }, "partial_eyes"),
face("strangers", { unassignedFaces: true }, "pixelate"),
];
it("prefers track over person over default over unassigned", () => {
const resolve = (storedId: string | null, personId: string | null, protectUnassigned = false) =>
resolveProtection({ storedId, personId }, { faces, defaults, protectUnassigned })?.mode;
expect(resolve("t1", "alice")).toBe("solid");
expect(resolve("t2", "bob")).toBe("partial_eyes");
expect(resolve("t2", "alice")).toBe("pixelate");
expect(resolve(null, null)).toBe("pixelate");
expect(
resolveProtection(
{ storedId: null, personId: null },
{ faces: [], defaults, protectUnassigned: true },
),
).toMatchObject({ mode: "blur", origin: "unassigned" });
expect(
resolveProtection(
{ storedId: null, personId: null },
{ faces: [], defaults, protectUnassigned: false },
),
).toBeNull();
expect(effectiveModes(faces, defaults).get("bob")).toBe("partial_eyes");
});
it("matches faces to people only above the similarity threshold", () => {
const centroids = [
{ personId: "alice", centroid: encodeVectors([[1, 0, 0]]) },
{ personId: "bob", centroid: encodeVectors([[0, 1, 0]]) },
];
expect(matchPerson([0.9, 0.1, 0], centroids)).toBe("alice");
expect(matchPerson([0.5, 0.5, 0.7], centroids)).toBeNull();
expect(matchPerson(null, centroids)).toBeNull();
});
it("merges fresh tracks into stored ones and keeps what the fresh pass missed", () => {
const stored = [
{ id: "s1", storedId: "s1", personId: "alice", keyframes: [{ tMs: 0, box }] },
{ id: "s2", storedId: "s2", personId: null, keyframes: [{ tMs: 0, box: { ...box, x: 0 } }] },
];
const merged = mergeTracks(
"a1",
stored,
[
{
keyframes: [
{ tMs: 0, box, confidence: 0.9 },
{ tMs: 100, box, confidence: 0.9 },
],
embedding: null,
},
{ keyframes: [{ tMs: 0, box: { ...box, y: 0.8 }, confidence: 0.9 }], embedding: [1, 0, 0] },
],
[{ personId: "carol", centroid: encodeVectors([[1, 0, 0]]) }],
);
expect(merged.map((t) => [t.id, t.personId, t.keyframes.length])).toEqual([
["s1", "alice", 3],
["fresh-a1-2", "carol", 1],
["s2", null, 1],
]);
expect(syntheticSeedFor("alice")).toBe(syntheticSeedFor("alice"));
expect(syntheticSeedFor("alice")).not.toBe(syntheticSeedFor("bob"));
});
});
describe("geometry", () => {
it("crops with the output aspect and knows what lies outside", () => {
const geometry = { sourceWidth: 1920, sourceHeight: 1080, outWidth: 1080, outHeight: 1920 };
const crop = cropPixels(geometry, { cx: 0.8, cy: 0.5, scale: 1 });
expect(crop).toEqual({ x: 1232, y: 0, w: 608, h: 1080 });
expect(outsideCrop({ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }, crop, 1920, 1080)).toBe(true);
expect(outsideCrop({ x: 0.6, y: 0.1, w: 0.2, h: 0.2 }, crop, 1920, 1080)).toBe(false);
expect(padBox(box, 20)).toEqual({
x: 0.39,
y: 0.39,
w: expect.closeTo(0.12),
h: expect.closeTo(0.12),
});
});
it("maps source spans onto the output timeline", () => {
const edl = testEdl([
{ id: "a", assetId: "x", inMs: 1000, outMs: 3000 },
{ id: "b", assetId: "y", inMs: 0, outMs: 1000 },
]);
const [a, b] = edl.tracks.video;
expect(sourceToOutput(edl, a!, { startMs: 1500, endMs: 5000 })).toEqual({
startMs: 500,
endMs: 2000,
});
expect(sourceToOutput(edl, b!, { startMs: 200, endMs: 300 })).toEqual({
startMs: 2200,
endMs: 2300,
});
expect(sourceToOutput(edl, a!, { startMs: 0, endMs: 500 })).toBeNull();
});
it("finds the box between keyframes", () => {
const keyframes = [
{ tMs: 0, box },
{ tMs: 100, box: { ...box, x: 0.5 } },
];
expect(boxAt(keyframes, 50)).toEqual({
x: 0.4,
y: 0.4,
w: expect.closeTo(0.2),
h: expect.closeTo(0.1),
});
expect(boxAt(keyframes, 150)).toEqual({ ...box, x: 0.5 });
expect(boxAt(keyframes, 500)).toBeNull();
});
it("reads stored keyframe files in both shapes", () => {
expect(parseStoredKeyframes([{ tMs: 0, box, extra: 1 }])).toEqual([{ tMs: 0, box }]);
expect(parseStoredKeyframes({ keyframes: [{ tMs: 5, box, confidence: 0.4 }] })).toEqual([
{ tMs: 5, box, confidence: 0.4 },
]);
expect(() => parseStoredKeyframes({ frames: [] })).toThrow();
});
});

499
packages/adapters/src/media-privacy-plan.ts Normal file
View File

@ -0,0 +1,499 @@
import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import type {
AnonymizeMode,
BoxKeyframe,
Edl,
EdlVideoClip,
FaceRedaction,
NormalizedBox,
} from "@rakazo/contracts";
import { AiAlterationSchema, AnonymizeModeSchema, BoxKeyframeSchema } from "@rakazo/contracts";
import { cosineSimilarity, decodeVectors, introMs, MAX_TRACKING_GAP_MS } from "@rakazo/media";
import { z } from "zod";
/*
* Pure planning helpers for face, mark, and region protection, plus the
* privacy record that prepareSources leaves next to the render output for the
* gate. Times are source times of the asset unless a name says otherwise;
* boxes are normalized to the displayed (rotation-applied) frame.
*/
/** Tracking rate of the fresh detection pass on clip sources. */
export const FACE_TRACK_FPS = 10;
/**
* Minimum cosine similarity between a face embedding and a person centroid to
* count as that person. Below it the face is unassigned, which fails closed:
* unassigned faces are redacted and block the gate while a protected person
* appears in the project.
*/
export const FACE_MATCH_MIN_COSINE = 0.6;
/** Detections below this confidence count as lost (spec 7: hold and grow the last mask). */
export const FACE_MIN_CONFIDENCE = 0.5;
/** A fresh track inherits a stored track's person when their boxes overlap this much on average. */
export const TRACK_MATCH_MIN_IOU = 0.3;
/** Step for linear interpolation across short detection gaps (<= MAX_TRACKING_GAP_MS). */
export const INTERPOLATE_STEP_MS = 100;
/** Lost detections hold the last box, enlarged by this share on every side (spec 7). */
export const GAP_HOLD_GROWTH = 0.25;
/** Padding around marks and regions, in percent of the box. */
export const MARK_PADDING_PCT = 10;
/** Person-bound marks re-found below this segmentation score go to review and block the gate. */
export const MARK_REVIEW_MIN_SCORE = 0.6;
/** Treatment quality below this suggests (and, while rendering, applies) a fallback. */
export const MARK_QUALITY_MIN = 0.7;
/** Name of the privacy record prepareSources writes into the render work directory. */
export const PRIVACY_RECORD_FILE = "privacy.json";
export type ProtectMode = Exclude<AnonymizeMode, "none">;
export type PartialMode = "partial_eyes" | "partial_upper" | "partial_lower";
export interface Protection {
mode: ProtectMode;
paddingPct: number;
featherPct: number;
minSizePct: number;
/** Redaction id from the EDL, "default" (person setting), or "unassigned". */
origin: string;
}
export type TrackKeyframe = BoxKeyframe & { confidence?: number };
export interface CandidateTrack {
/** Stable id: the stored track id, or `fresh-<n>` for faces only the fresh pass found. */
id: string;
/** Stored track id when the face matches an analysed track, for EDL track targets. */
storedId: string | null;
personId: string | null;
keyframes: TrackKeyframe[];
}
export interface TimeWindow {
startMs: number;
endMs: number;
}
const DEFAULT_PROTECTION = { paddingPct: 20, featherPct: 10, minSizePct: 2 };
export function isPartial(mode: AnonymizeMode): mode is PartialMode {
return mode === "partial_eyes" || mode === "partial_upper" || mode === "partial_lower";
}
function fromRedaction(redaction: FaceRedaction): Protection {
return {
mode: redaction.mode,
paddingPct: redaction.paddingPct,
featherPct: redaction.featherPct,
minSizePct: redaction.minSizePct,
origin: redaction.id,
};
}
/**
* Protection for one face track. Most specific wins: EDL track target, EDL
* person target, the person's default, then the EDL unassigned target, and
* finally a blur for unassigned faces while any protected person appears in
* the project.
*/
export function resolveProtection(
track: Pick<CandidateTrack, "storedId" | "personId">,
context: {
faces: FaceRedaction[];
defaults: Map<string, AnonymizeMode>;
protectUnassigned: boolean;
},
): Protection | null {
if (track.storedId) {
const byTrack = context.faces.find(
(face) => "trackId" in face.target && face.target.trackId === track.storedId,
);
if (byTrack) return fromRedaction(byTrack);
}
if (track.personId) {
const byPerson = context.faces.find(
(face) => "personId" in face.target && face.target.personId === track.personId,
);
if (byPerson) return fromRedaction(byPerson);
const mode = context.defaults.get(track.personId);
if (mode && mode !== "none") return { mode, ...DEFAULT_PROTECTION, origin: "default" };
return null;
}
const unassigned = context.faces.find((face) => "unassignedFaces" in face.target);
if (unassigned) return fromRedaction(unassigned);
if (context.protectUnassigned)
return { mode: "blur", ...DEFAULT_PROTECTION, origin: "unassigned" };
return null;
}
/** Effective mode per person: an EDL person target wins over the person's default. */
export function effectiveModes(
faces: FaceRedaction[],
defaults: Map<string, AnonymizeMode>,
): Map<string, AnonymizeMode> {
const modes = new Map(defaults);
for (const face of faces) {
if ("personId" in face.target) modes.set(face.target.personId, face.mode);
}
return modes;
}
/** Best person for an embedding by cosine to the centroids, or null below the threshold. */
export function matchPerson(
embedding: ArrayLike<number> | null,
centroids: Array<{ personId: string; centroid: Uint8Array }>,
): string | null {
if (!embedding || embedding.length === 0) return null;
let best: { personId: string; score: number } | null = null;
for (const { personId, centroid } of centroids) {
if (centroid.byteLength !== embedding.length * 4) continue;
const vector = decodeVectors(centroid, embedding.length)[0];
if (!vector) continue;
const score = cosineSimilarity(embedding, vector);
if (score >= FACE_MATCH_MIN_COSINE && (!best || score > best.score)) best = { personId, score };
}
return best?.personId ?? null;
}
const clampUnit = (value: number) => Math.min(1, Math.max(0, value));
/** Grow a box by `share` of its size on every side, clamped to the frame. */
export function growBox(box: NormalizedBox, share: number): NormalizedBox {
const x = clampUnit(box.x - box.w * share);
const y = clampUnit(box.y - box.h * share);
return {
x,
y,
w: clampUnit(box.x + box.w + box.w * share) - x,
h: clampUnit(box.y + box.h + box.h * share) - y,
};
}
/** Padding as the redaction providers apply it: paddingPct / 2 of the size on each side. */
export function padBox(box: NormalizedBox, paddingPct: number): NormalizedBox {
return growBox(box, Math.max(0, paddingPct) / 200);
}
/** Enlarge a box around its centre until both sides reach `minSizePct` of the frame. */
export function ensureMinSize(box: NormalizedBox, minSizePct: number): NormalizedBox {
const min = minSizePct / 100;
if (box.w >= min && box.h >= min) return box;
const w = Math.min(1, Math.max(box.w, min));
const h = Math.min(1, Math.max(box.h, min));
const x = clampUnit(Math.min(box.x + box.w / 2 - w / 2, 1 - w));
const y = clampUnit(Math.min(box.y + box.h / 2 - h / 2, 1 - h));
return { x, y, w, h };
}
export function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox {
const x = Math.min(a.x, b.x);
const y = Math.min(a.y, b.y);
return {
x,
y,
w: Math.max(a.x + a.w, b.x + b.w) - x,
h: Math.max(a.y + a.h, b.y + b.h) - y,
};
}
export function iou(a: NormalizedBox, b: NormalizedBox): number {
const w = Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x);
const h = Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y);
if (w <= 0 || h <= 0) return 0;
const inter = w * h;
return inter / (a.w * a.h + b.w * b.h - inter);
}
function lerpBox(a: NormalizedBox, b: NormalizedBox, f: number): NormalizedBox {
const mix = (p: number, q: number) => p + (q - p) * f;
return { x: mix(a.x, b.x), y: mix(a.y, b.y), w: mix(a.w, b.w), h: mix(a.h, b.h) };
}
/** Round outward so a rounded box never covers less than the exact one. */
export function roundBoxOut(box: NormalizedBox): NormalizedBox {
const x = Math.floor(box.x * 10_000) / 10_000;
const y = Math.floor(box.y * 10_000) / 10_000;
return {
x,
y,
w: Math.min(1 - x, Math.ceil((box.x + box.w - x) * 10_000) / 10_000),
h: Math.min(1 - y, Math.ceil((box.y + box.h - y) * 10_000) / 10_000),
};
}
function nearWindows(tMs: number, windows: TimeWindow[]): boolean {
return windows.some(
(w) => tMs >= w.startMs - MAX_TRACKING_GAP_MS && tMs <= w.endMs + MAX_TRACKING_GAP_MS,
);
}
/**
* Keyframes for redaction near the windows (clip ranges plus the bridging
* limit on both sides). Detections below FACE_MIN_CONFIDENCE count as lost.
* Gaps up to MAX_TRACKING_GAP_MS between confident detections are
* interpolated; longer ones hold the last confident box grown by
* GAP_HOLD_GROWTH (unioned with any weak detections) until the face is found
* again, and are returned so the gate blocks them as needing review.
*/
export function protectKeyframes(
keyframes: TrackKeyframe[],
windows: TimeWindow[],
minSizePct: number,
): { keyframes: BoxKeyframe[]; gaps: TimeWindow[] } {
const inside = [...keyframes]
.filter((k) => nearWindows(k.tMs, windows))
.sort((a, b) => a.tMs - b.tMs)
.map((k) => ({
tMs: k.tMs,
box: ensureMinSize(k.box, minSizePct),
confident: (k.confidence ?? 1) >= FACE_MIN_CONFIDENCE,
}));
const out: BoxKeyframe[] = [];
const gaps: TimeWindow[] = [];
const confident = inside.filter((k) => k.confident);
const weak = inside.filter((k) => !k.confident);
if (confident.length === 0) {
// Only weak detections: cover them grown and send the whole span to review.
for (const k of weak)
out.push({ tMs: k.tMs, box: roundBoxOut(growBox(k.box, GAP_HOLD_GROWTH)) });
const first = weak[0];
const last = weak.at(-1);
if (first && last) gaps.push({ startMs: first.tMs, endMs: Math.max(last.tMs, first.tMs + 1) });
return { keyframes: out, gaps };
}
confident.forEach((current, index) => {
out.push({ tMs: current.tMs, box: roundBoxOut(current.box) });
const next = confident[index + 1];
if (!next) return;
const span = next.tMs - current.tMs;
const between = weak.filter((k) => k.tMs > current.tMs && k.tMs < next.tMs);
if (span <= MAX_TRACKING_GAP_MS) {
for (let t = current.tMs + INTERPOLATE_STEP_MS; t < next.tMs; t += INTERPOLATE_STEP_MS) {
let box = lerpBox(current.box, next.box, (t - current.tMs) / span);
for (const w of between)
if (Math.abs(w.tMs - t) < INTERPOLATE_STEP_MS) box = unionBox(box, w.box);
out.push({ tMs: t, box: roundBoxOut(box) });
}
return;
}
let held = growBox(current.box, GAP_HOLD_GROWTH);
for (const w of between) held = unionBox(held, w.box);
const box = roundBoxOut(held);
out.push({ tMs: current.tMs + 1, box }, { tMs: next.tMs - 1, box });
gaps.push({ startMs: current.tMs, endMs: next.tMs });
});
return { keyframes: out, gaps };
}
/** Box a keyframe list covers at `tMs`: the union of the bracketing keyframes, or null outside. */
export function boxAt(keyframes: BoxKeyframe[], tMs: number, holdMs = 100): NormalizedBox | null {
let before: BoxKeyframe | null = null;
let after: BoxKeyframe | null = null;
for (const k of keyframes) {
if (k.tMs <= tMs && (!before || k.tMs > before.tMs)) before = k;
if (k.tMs >= tMs && (!after || k.tMs < after.tMs)) after = k;
}
if (before && after) return unionBox(before.box, after.box);
if (before && tMs - before.tMs <= holdMs) return before.box;
return null;
}
/** Mean IoU of two tracks over the times both observe (nearest keyframe within 60 ms). */
export function trackOverlap(a: BoxKeyframe[], b: BoxKeyframe[]): number {
const scores: number[] = [];
for (const k of a) {
let nearest: BoxKeyframe | null = null;
for (const other of b) {
if (Math.abs(other.tMs - k.tMs) > 60) continue;
if (!nearest || Math.abs(other.tMs - k.tMs) < Math.abs(nearest.tMs - k.tMs)) nearest = other;
}
if (nearest) scores.push(iou(k.box, nearest.box));
}
return scores.length === 0 ? 0 : scores.reduce((sum, s) => sum + s, 0) / scores.length;
}
/** Source windows each clip of an asset shows. */
export function clipWindows(clips: EdlVideoClip[]): TimeWindow[] {
return clips.map((clip) => ({ startMs: clip.inMs, endMs: clip.outMs }));
}
export function overlapsWindows(span: TimeWindow, windows: TimeWindow[]): boolean {
return windows.some((w) => span.startMs <= w.endMs && w.startMs <= span.endMs);
}
/**
* Inner part of a face box that a partial mode must alter. It sits inside the
* band every provider covers (eyes 22-55 %, upper 0-55 %, lower 50-100 % of
* the padded box), so the gate checks only pixels the mode promised to change.
*/
export function partialCheckBand(box: NormalizedBox, mode: PartialMode): NormalizedBox {
const [top, bottom] =
mode === "partial_eyes" ? [0.3, 0.48] : mode === "partial_upper" ? [0.05, 0.5] : [0.55, 0.95];
return {
x: box.x + box.w * 0.1,
y: box.y + box.h * top,
w: box.w * 0.8,
h: box.h * (bottom - top),
};
}
export interface PixelBox {
x: number;
y: number;
w: number;
h: number;
}
/**
* Static crop window in display pixels with the output aspect, as the
* compiler sizes it for a reframe keyframe. Even sizes and offsets for yuv420p.
*/
export function cropPixels(
geometry: { sourceWidth: number; sourceHeight: number; outWidth: number; outHeight: number },
keyframe: { cx: number; cy: number; scale: number },
): PixelBox {
const outAspect = geometry.outWidth / geometry.outHeight;
let w: number;
let h: number;
if (geometry.sourceWidth / geometry.sourceHeight > outAspect) {
h = geometry.sourceHeight;
w = h * outAspect;
} else {
w = geometry.sourceWidth;
h = w / outAspect;
}
const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2);
const even = (v: number) => Math.max(0, Math.floor(v / 2) * 2);
const maxW = geometry.sourceWidth - (geometry.sourceWidth % 2);
const maxH = geometry.sourceHeight - (geometry.sourceHeight % 2);
const cw = Math.min(maxW, roundEven(w / keyframe.scale));
const ch = Math.min(maxH, roundEven(h / keyframe.scale));
const x = even(
Math.min(Math.max(0, keyframe.cx * geometry.sourceWidth - cw / 2), geometry.sourceWidth - cw),
);
const y = even(
Math.min(Math.max(0, keyframe.cy * geometry.sourceHeight - ch / 2), geometry.sourceHeight - ch),
);
return { x, y, w: cw, h: ch };
}
/** Whether a normalized box lies completely outside a pixel crop. */
export function outsideCrop(box: NormalizedBox, crop: PixelBox, width: number, height: number) {
const x0 = box.x * width;
const y0 = box.y * height;
const x1 = (box.x + box.w) * width;
const y1 = (box.y + box.h) * height;
return x1 <= crop.x || x0 >= crop.x + crop.w || y1 <= crop.y || y0 >= crop.y + crop.h;
}
/** Map a source window of one clip onto the output timeline (intro included), or null. */
export function sourceToOutput(edl: Edl, clip: EdlVideoClip, span: TimeWindow): TimeWindow | null {
const start = Math.max(span.startMs, clip.inMs);
const end = Math.min(span.endMs, clip.outMs);
if (end < start) return null;
const toOutput = (t: number) =>
Math.round(introMs(edl) + clip.atMs + (t - clip.inMs) / clip.speed);
return { startMs: toOutput(start), endMs: toOutput(end) };
}
// ---------------------------------------------------------------------------
// Privacy record (prepareSources → gate)
// ---------------------------------------------------------------------------
const PixelBoxSchema = z.object({
x: z.number().int().min(0),
y: z.number().int().min(0),
w: z.number().int().min(1),
h: z.number().int().min(1),
});
const SpanSchema = { startMs: z.number(), endMs: z.number() };
/** One protected region the gate verifies pixel by pixel. */
export const PrivacyCheckSchema = z.object({
kind: z.enum(["face", "mark", "region"]),
/** Track, mark, or region id. */
id: z.string(),
personId: z.string().nullable(),
markId: z.string().nullable(),
/** Boxes the treatment promised to alter, normalized to the base frame. */
keyframes: z.array(BoxKeyframeSchema),
/** Absolute mask directory (`<ms>.png` per frame) or null for box-only regions. */
maskDir: z.string().nullable(),
});
export type PrivacyCheck = z.infer<typeof PrivacyCheckSchema>;
export const PrivacyRecordSchema = z.object({
version: z.literal(1),
assets: z.record(
z.string(),
z.object({
/** Unredacted file the prepared source derives from (pixel reference). */
basePath: z.string(),
preparedPath: z.string(),
width: z.number().int().positive(),
height: z.number().int().positive(),
still: z.boolean(),
/** Static crop applied for reframe_out, in base display pixels. */
crop: PixelBoxSchema.nullable(),
checks: z.array(PrivacyCheckSchema),
}),
),
/** Detection gaps over MAX_TRACKING_GAP_MS per asset (source time), for review. */
trackingGaps: z.array(
z.object({ assetId: z.string(), personId: z.string().nullable(), ...SpanSchema }),
),
/** Person-bound marks re-found with low confidence, for review. */
reviews: z.array(z.object({ markId: z.string(), assetId: z.string(), score: z.number() })),
/** Faces that stay detectable by design (synthetic identity, partial modes), source time. */
expectedFaces: z.array(
z.object({
assetId: z.string(),
personId: z.string().nullable(),
mode: AnonymizeModeSchema,
...SpanSchema,
}),
),
/** Faces in the clip ranges that belong to no person (consent cannot be shown). */
unassignedFaces: z.array(z.object({ assetId: z.string(), ...SpanSchema })),
aiAlterations: z.array(AiAlterationSchema),
/** Effective protection per person (EDL target over person default). */
protectedPeople: z.record(z.string(), AnonymizeModeSchema),
/** People whose faces appear in the clip ranges. */
visiblePeople: z.array(z.string()),
protectUnassigned: z.boolean(),
});
export type PrivacyRecord = z.infer<typeof PrivacyRecordSchema>;
export async function writePrivacyRecord(workDir: string, record: PrivacyRecord): Promise<void> {
await writeFile(path.join(workDir, PRIVACY_RECORD_FILE), JSON.stringify(record));
}
/** The record prepareSources wrote, or null when it is missing. Malformed records throw. */
export async function readPrivacyRecord(workDir: string): Promise<PrivacyRecord | null> {
let text: string;
try {
text = await readFile(path.join(workDir, PRIVACY_RECORD_FILE), "utf8");
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
throw error;
}
return PrivacyRecordSchema.parse(JSON.parse(text));
}
/** Stored face keyframe files: an array of keyframes or a tracking result with `keyframes`. */
const StoredKeyframeSchema = BoxKeyframeSchema.extend({ confidence: z.number().optional() });
const StoredKeyframesSchema = z.union([
z.array(StoredKeyframeSchema.passthrough()),
z.object({ keyframes: z.array(StoredKeyframeSchema.passthrough()) }).passthrough(),
]);
export function parseStoredKeyframes(json: unknown): TrackKeyframe[] {
const parsed = StoredKeyframesSchema.parse(json);
const list = Array.isArray(parsed) ? parsed : parsed.keyframes;
return list.map((k) => ({
tMs: k.tMs,
box: k.box,
...(k.confidence !== undefined ? { confidence: k.confidence } : {}),
}));
}

39
packages/adapters/src/media-privacy.ts Normal file
View File

@ -0,0 +1,39 @@
import { rm } from "node:fs/promises";
import path from "node:path";
import { prepareSources } from "./media-anonymize.js";
import type { MediaPrivacyService, MediaServiceDeps } from "./media-context.js";
import { runGate } from "./media-gate.js";
import { createMark, updateMark } from "./media-marks.js";
import type { PrivacyToolOptions } from "./media-privacy-io.js";
import { resolveStored } from "./media-privacy-io.js";
/*
* MediaPrivacyService: anonymization before rendering, the gate after it,
* identifying marks, and deleting a person with their biometric data. Face
* data never leaves this host: matching, tracking, and clustering run against
* the local inference provider and database only.
*/
export function createMediaPrivacyService(
deps: MediaServiceDeps,
tools: PrivacyToolOptions = {},
): MediaPrivacyService {
const withTools = { ...deps, ...tools };
return {
prepareSources: (input) => prepareSources(withTools, input),
gate: (input) => runGate(withTools, input),
createMark: (actor, input) => createMark(withTools, actor, input),
updateMark: (actor, input) => updateMark(withTools, actor, input),
async removePerson(actor, personId) {
const removed = await deps.repos.removePersonCompletely(actor.spaceId, personId);
// Rows are gone; now the files: track keyframes and each mark's directory
// (masks plus previews). A path outside the media root throws untouched.
const markDirs = removed.maskDirs.map((dir) =>
path.basename(dir) === "masks" ? path.dirname(dir) : dir,
);
for (const stored of [...removed.keyframesPaths, ...markDirs]) {
await rm(await resolveStored(deps.mediaRoot, stored), { recursive: true, force: true });
}
},
};
}