merge: W2 privacy
This commit is contained in:
Claude
commit
90f18cea52
424
packages/adapters/src/media-anonymize.test.ts
Normal file
424
packages/adapters/src/media-anonymize.test.ts
Normal file
@ -0,0 +1,424 @@
|
||||
import { copyFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
|
||||
import type { AnonymizeMode, Edl } from "@rakazo/contracts";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { prepareSources } from "./media-anonymize.js";
|
||||
import { ffmpegAvailable } from "./media-emulator-clips.js";
|
||||
import { runGate } from "./media-gate.js";
|
||||
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
|
||||
import {
|
||||
ACTOR,
|
||||
cleanProbe,
|
||||
createPrivacyHarness,
|
||||
testEdl,
|
||||
testProfile,
|
||||
} from "./media-privacy-fixtures.js";
|
||||
import { PrivacyUnavailableError } from "./media-privacy-io.js";
|
||||
import { extractLuma, residualVisibility } from "./media-privacy-pixels.js";
|
||||
import { readPrivacyRecord } from "./media-privacy-plan.js";
|
||||
|
||||
const hasFfmpeg = ffmpegAvailable();
|
||||
let harness: PrivacyHarness | null = null;
|
||||
|
||||
afterEach(async () => {
|
||||
await harness?.cleanup();
|
||||
harness = null;
|
||||
});
|
||||
|
||||
/** One asset with one face of person "alice" (stored track), one clip over it. */
|
||||
async function setup(
|
||||
mode: AnonymizeMode,
|
||||
options: {
|
||||
fileName?: string;
|
||||
edl?: (assetId: string) => Edl;
|
||||
inference?: (emulator: PrivacyHarness["inference"]) => MediaInferenceProvider | null;
|
||||
} = {},
|
||||
) {
|
||||
harness = await createPrivacyHarness(options.inference);
|
||||
const asset = await harness.addAsset(options.fileName ?? "clip__faces-1____person-alice__.mp4");
|
||||
harness.addPerson("alice", mode);
|
||||
await harness.analyseFaces(asset.id, ["alice"]);
|
||||
await harness.learnPerson("alice", asset.id);
|
||||
const edl =
|
||||
options.edl?.(asset.id) ?? testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
|
||||
const workDir = await harness.workDir(`render-${mode}`);
|
||||
return { h: harness, asset, edl, workDir };
|
||||
}
|
||||
|
||||
async function gateOn(
|
||||
h: PrivacyHarness,
|
||||
input: { edl: Edl; workDir: string; prepared: string; outputName?: string },
|
||||
profile = testProfile(),
|
||||
) {
|
||||
// The compiler is not under test: the prepared source stands in for the render output.
|
||||
const outputPath = path.join(input.workDir, input.outputName ?? "output.mp4");
|
||||
await copyFile(input.prepared, outputPath);
|
||||
return runGate(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl: input.edl,
|
||||
profile,
|
||||
outputPath,
|
||||
ffprobe: cleanProbe(input.edl),
|
||||
copy: { title: null, caption: null, tags: [] },
|
||||
bundleHasDisclosure: false,
|
||||
autoFixRound: 0,
|
||||
});
|
||||
}
|
||||
|
||||
describe.skipIf(!hasFfmpeg)("prepareSources", () => {
|
||||
it.each([
|
||||
"blur",
|
||||
"pixelate",
|
||||
"solid",
|
||||
"sticker",
|
||||
"partial_eyes",
|
||||
"partial_upper",
|
||||
"partial_lower",
|
||||
] as const)("applies %s to a protected person and the gate verifies it", async (mode) => {
|
||||
const { h, asset, edl, workDir } = await setup(mode);
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
const source = prepared.sources.get(asset.id);
|
||||
expect(source?.path).toContain(`${path.sep}privacy${path.sep}`);
|
||||
expect(source).toMatchObject({ width: 320, height: 180, rotation: 0, still: false });
|
||||
const record = await readPrivacyRecord(workDir);
|
||||
expect(record?.protectedPeople).toEqual({ alice: mode });
|
||||
const check = record?.assets[asset.id]?.checks[0];
|
||||
expect(check).toMatchObject({ kind: "face", personId: "alice" });
|
||||
|
||||
// The protected box changed, the rest of the frame did not.
|
||||
const base = await extractLuma(path.join(h.root, asset.storagePath!), {
|
||||
tMs: 1000,
|
||||
width: 320,
|
||||
height: 180,
|
||||
});
|
||||
const treated = await extractLuma(source!.path, { tMs: 1000, width: 320, height: 180 });
|
||||
const box = check!.keyframes.find((k) => k.tMs >= 1000)!.box;
|
||||
expect(residualVisibility({ source: base, treated, box })).toBeLessThan(0.05);
|
||||
expect(
|
||||
residualVisibility({ source: base, treated, box: { x: 0.6, y: 0.6, w: 0.35, h: 0.35 } }),
|
||||
).toBeGreaterThan(0.95);
|
||||
|
||||
const report = await gateOn(h, { edl, workDir, prepared: source!.path });
|
||||
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
|
||||
expect(report.passed).toBe(true);
|
||||
});
|
||||
|
||||
it("crops a reframe_out face out of frame", async () => {
|
||||
const { h, asset, workDir } = await setup("reframe_out");
|
||||
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
|
||||
width: 180,
|
||||
height: 320,
|
||||
});
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
const source = prepared.sources.get(asset.id)!;
|
||||
expect(prepared.fallbacks).toEqual([]);
|
||||
expect(source.width / source.height).toBeCloseTo(180 / 320, 1);
|
||||
const record = await readPrivacyRecord(workDir);
|
||||
expect(record?.assets[asset.id]?.crop).not.toBeNull();
|
||||
const report = await gateOn(h, { edl, workDir, prepared: source.path });
|
||||
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
|
||||
});
|
||||
|
||||
it("falls back from reframe_out to blur and says so when no crop avoids the face", async () => {
|
||||
const { h, asset, edl, workDir } = await setup("none", {
|
||||
edl: (assetId) =>
|
||||
testEdl([{ id: "c1", assetId, inMs: 0, outMs: 2000 }], {
|
||||
redactions: {
|
||||
faces: [
|
||||
{
|
||||
id: "big",
|
||||
target: { personId: "alice" },
|
||||
mode: "reframe_out",
|
||||
paddingPct: 100,
|
||||
featherPct: 0,
|
||||
minSizePct: 50,
|
||||
},
|
||||
],
|
||||
marks: [],
|
||||
regions: [],
|
||||
},
|
||||
}),
|
||||
});
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
expect(prepared.fallbacks).toEqual([
|
||||
"person alice: no crop keeps the face out of frame; blurred instead",
|
||||
]);
|
||||
const source = prepared.sources.get(asset.id)!;
|
||||
expect(source.width).toBe(320);
|
||||
const report = await gateOn(h, { edl, workDir, prepared: source.path });
|
||||
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
|
||||
});
|
||||
|
||||
it("blocks the export when tracking loses the face for more than 500 ms", async () => {
|
||||
const { h, asset, edl, workDir } = await setup("blur", {
|
||||
fileName: "clip__faces-1____person-alice____noface-gap-600-1400__.mp4",
|
||||
});
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
const record = await readPrivacyRecord(workDir);
|
||||
expect(record?.trackingGaps).toEqual([
|
||||
expect.objectContaining({ assetId: asset.id, personId: "alice" }),
|
||||
]);
|
||||
const report = await gateOn(h, {
|
||||
edl,
|
||||
workDir,
|
||||
prepared: prepared.sources.get(asset.id)!.path,
|
||||
});
|
||||
expect(report.passed).toBe(false);
|
||||
expect(report.issues).toContainEqual(
|
||||
expect.objectContaining({ code: "tracking_gap", severity: "block", personId: "alice" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to blur when synthetic identity is unavailable and never fakes it", async () => {
|
||||
const { h, asset, edl, workDir } = await setup("synthetic_identity");
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
expect(prepared.fallbacks).toEqual([
|
||||
"person alice: synthetic identity is not available; blurred instead",
|
||||
]);
|
||||
expect(prepared.aiAlterations).not.toContain("synthetic_identity");
|
||||
const report = await gateOn(h, {
|
||||
edl,
|
||||
workDir,
|
||||
prepared: prepared.sources.get(asset.id)!.path,
|
||||
});
|
||||
expect(report.passed).toBe(true);
|
||||
});
|
||||
|
||||
it("uses a licensed synthetic identity with a stable seed and no target image", async () => {
|
||||
const synthesize = vi.fn();
|
||||
const { h, asset, edl, workDir } = await setup("synthetic_identity", {
|
||||
inference: (emulator) => {
|
||||
const provider = Object.create(emulator) as MediaInferenceProvider;
|
||||
provider.capabilities = async () => {
|
||||
const caps = await emulator.capabilities();
|
||||
caps.operations.synthesizeIdentity = {
|
||||
available: true,
|
||||
experimental: true,
|
||||
model: "test-synth",
|
||||
license: "Apache-2.0",
|
||||
commercialUse: true,
|
||||
};
|
||||
return caps;
|
||||
};
|
||||
provider.synthesizeIdentity = async (request) => {
|
||||
synthesize(request);
|
||||
// Stand-in synthesis: a solid fill over the track (real models paint a face).
|
||||
return emulator.redact({
|
||||
path: request.path,
|
||||
outPath: request.outPath,
|
||||
redactions: [
|
||||
{ keyframes: request.track, mode: "solid", paddingPct: 20, featherPct: 0 },
|
||||
],
|
||||
});
|
||||
};
|
||||
return provider;
|
||||
},
|
||||
});
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
expect(prepared.aiAlterations).toContain("synthetic_identity");
|
||||
expect(synthesize).toHaveBeenCalledTimes(1);
|
||||
const request = synthesize.mock.calls[0]![0] as Record<string, unknown>;
|
||||
expect(Object.keys(request).sort()).toEqual(["outPath", "path", "seed", "signal", "track"]);
|
||||
const seed = h.store.people[0]!.syntheticSeed;
|
||||
expect(request.seed).toBe(seed);
|
||||
expect(seed).not.toBeNull();
|
||||
const record = await readPrivacyRecord(workDir);
|
||||
expect(record?.expectedFaces).toEqual([
|
||||
expect.objectContaining({ personId: "alice", mode: "synthetic_identity" }),
|
||||
]);
|
||||
const report = await gateOn(h, {
|
||||
edl,
|
||||
workDir,
|
||||
prepared: prepared.sources.get(asset.id)!.path,
|
||||
});
|
||||
expect(report.passed).toBe(true);
|
||||
});
|
||||
|
||||
it("fails closed without a media worker", async () => {
|
||||
const { h, edl, workDir } = await setup("blur", { inference: () => null });
|
||||
await expect(
|
||||
prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(PrivacyUnavailableError);
|
||||
});
|
||||
|
||||
it("returns the untouched source when nothing needs protection", async () => {
|
||||
const { h, asset, edl, workDir } = await setup("none");
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
expect(prepared.sources.get(asset.id)?.path).toBe(path.join(h.root, asset.storagePath!));
|
||||
expect(prepared.aiAlterations).toEqual([]);
|
||||
});
|
||||
|
||||
it("blurs unassigned faces next to a protected person and records voice disguise", async () => {
|
||||
const { h, asset, workDir } = await setup("blur", {
|
||||
fileName: "clip__faces-2____person-alice__.mp4",
|
||||
});
|
||||
// Track 2 stays unassigned (analysis stored it without a person).
|
||||
h.store.tracks[1]!.personId = null;
|
||||
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
|
||||
redactions: { faces: [], marks: [], regions: [], voice: { method: "pitch", semitones: 3 } },
|
||||
});
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
expect(prepared.aiAlterations).toEqual(["voice_disguise"]);
|
||||
const record = await readPrivacyRecord(workDir);
|
||||
expect(record?.protectUnassigned).toBe(true);
|
||||
expect(record?.assets[asset.id]?.checks.map((c) => c.personId)).toEqual(["alice", null]);
|
||||
const report = await gateOn(h, {
|
||||
edl,
|
||||
workDir,
|
||||
prepared: prepared.sources.get(asset.id)!.path,
|
||||
});
|
||||
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
|
||||
});
|
||||
|
||||
it("protects a photo and renders it from a redacted still", async () => {
|
||||
const { h, asset, edl, workDir } = await setup("pixelate", {
|
||||
fileName: "photo__faces-1____person-alice__.png",
|
||||
});
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
const source = prepared.sources.get(asset.id)!;
|
||||
expect(source).toMatchObject({ still: true, durationMs: 0, width: 320, height: 180 });
|
||||
expect(source.path.endsWith(".png")).toBe(true);
|
||||
const record = await readPrivacyRecord(workDir);
|
||||
const box = record!.assets[asset.id]!.checks[0]!.keyframes[0]!.box;
|
||||
const base = await extractLuma(path.join(h.root, asset.storagePath!), {
|
||||
tMs: 0,
|
||||
width: 320,
|
||||
height: 180,
|
||||
still: true,
|
||||
});
|
||||
const treated = await extractLuma(source.path, {
|
||||
tMs: 0,
|
||||
width: 320,
|
||||
height: 180,
|
||||
still: true,
|
||||
});
|
||||
expect(residualVisibility({ source: base, treated, box })).toBeLessThan(0.05);
|
||||
});
|
||||
|
||||
it("applies EDL regions and covers one that no crop can avoid", async () => {
|
||||
const { h, asset, workDir } = await setup("none");
|
||||
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
|
||||
redactions: {
|
||||
faces: [],
|
||||
marks: [],
|
||||
regions: [
|
||||
{
|
||||
id: "plate",
|
||||
clipId: "c1",
|
||||
keyframes: [{ tMs: 0, box: { x: 0.7, y: 0.7, w: 0.2, h: 0.2 } }],
|
||||
treatment: "blur",
|
||||
},
|
||||
{
|
||||
id: "everything",
|
||||
clipId: "c1",
|
||||
keyframes: [{ tMs: 0, box: { x: 0, y: 0, w: 1, h: 1 } }],
|
||||
treatment: "reframe_out",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
expect(prepared.fallbacks).toEqual([
|
||||
"region everything: no crop keeps it out of frame; covered instead",
|
||||
]);
|
||||
const record = await readPrivacyRecord(workDir);
|
||||
expect(record?.assets[asset.id]?.checks.map((c) => c.id)).toEqual(["plate", "everything"]);
|
||||
});
|
||||
|
||||
it("blocks when a provider claims to redact but leaves the face visible", async () => {
|
||||
const { h, asset, edl, workDir } = await setup("blur", {
|
||||
inference: (emulator) => {
|
||||
const provider = Object.create(emulator) as MediaInferenceProvider;
|
||||
provider.redact = async (request) => {
|
||||
await copyFile(request.path, request.outPath);
|
||||
return { outPath: request.outPath, framesProcessed: 50 };
|
||||
};
|
||||
return provider;
|
||||
},
|
||||
});
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
const report = await gateOn(h, {
|
||||
edl,
|
||||
workDir,
|
||||
prepared: prepared.sources.get(asset.id)!.path,
|
||||
});
|
||||
expect(report.passed).toBe(false);
|
||||
expect(report.issues).toContainEqual(
|
||||
expect.objectContaining({ code: "face_not_covered", personId: "alice" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
928
packages/adapters/src/media-anonymize.ts
Normal file
928
packages/adapters/src/media-anonymize.ts
Normal file
@ -0,0 +1,928 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { RedactionInstruction } from "@rakazo/adapter-kit";
|
||||
import type {
|
||||
AiAlteration,
|
||||
AnonymizeMode,
|
||||
BoxKeyframe,
|
||||
Edl,
|
||||
EdlVideoClip,
|
||||
MarkTreatment,
|
||||
MediaCapabilities,
|
||||
MediaRenderQuality,
|
||||
NormalizedBox,
|
||||
NormalizedPoint,
|
||||
} from "@rakazo/contracts";
|
||||
import type { MediaRepos } from "@rakazo/db";
|
||||
import type { CompileSource } from "@rakazo/media";
|
||||
import { MAX_TRACKING_GAP_MS, reframeExcluding } from "@rakazo/media";
|
||||
import type { MediaActor, MediaPreparedSources, MediaServiceDeps } from "./media-context.js";
|
||||
import {
|
||||
anchorFromFace,
|
||||
faceAt,
|
||||
firstConfident,
|
||||
placeAnchor,
|
||||
promptBox,
|
||||
segmentMark,
|
||||
} from "./media-marks.js";
|
||||
import type { PrivacyToolOptions, ProbedSource } from "./media-privacy-io.js";
|
||||
import {
|
||||
analysisFile,
|
||||
copyInto,
|
||||
cropFile,
|
||||
firstFrame,
|
||||
inferenceCapabilities,
|
||||
isInsideRoot,
|
||||
maskKeyframes,
|
||||
operationAvailable,
|
||||
PrivacyUnavailableError,
|
||||
probeSource,
|
||||
readTrackKeyframes,
|
||||
resolveAssetFile,
|
||||
resolveStored,
|
||||
} from "./media-privacy-io.js";
|
||||
import type {
|
||||
CandidateTrack,
|
||||
PixelBox,
|
||||
PrivacyCheck,
|
||||
PrivacyRecord,
|
||||
Protection,
|
||||
TimeWindow,
|
||||
} from "./media-privacy-plan.js";
|
||||
import {
|
||||
boxAt,
|
||||
clipWindows,
|
||||
cropPixels,
|
||||
effectiveModes,
|
||||
FACE_TRACK_FPS,
|
||||
iou,
|
||||
isPartial,
|
||||
MARK_PADDING_PCT,
|
||||
MARK_QUALITY_MIN,
|
||||
MARK_REVIEW_MIN_SCORE,
|
||||
matchPerson,
|
||||
outsideCrop,
|
||||
overlapsWindows,
|
||||
PRIVACY_RECORD_FILE,
|
||||
padBox,
|
||||
partialCheckBand,
|
||||
protectKeyframes,
|
||||
resolveProtection,
|
||||
TRACK_MATCH_MIN_IOU,
|
||||
trackOverlap,
|
||||
unionBox,
|
||||
writePrivacyRecord,
|
||||
} from "./media-privacy-plan.js";
|
||||
|
||||
/*
|
||||
* prepareSources (spec 7): resolve every clip's file and apply all face, mark,
|
||||
* and region protection before the compiler sees it. The compiler applies no
|
||||
* redactions, so anything this function cannot honor throws
|
||||
* (PrivacyUnavailableError) instead of returning an unredacted source.
|
||||
*
|
||||
* Per asset, in this order: synthetic identities, inpainted marks, one
|
||||
* redaction pass (blur, pixelate, solid, sticker, partial bands, covers) by the
|
||||
* inference provider, then a static crop for reframe_out. Faces are found from
|
||||
* the stored tracks plus a fresh tracking pass; tracking gaps over 500 ms are
|
||||
* covered by the grown last box and recorded so the gate blocks them. Voice
|
||||
* disguise is applied by the compiler on the program audio and only recorded
|
||||
* here. Everything the gate must verify goes into privacy.json in workDir.
|
||||
*/
|
||||
|
||||
export type AnonymizeDeps = Pick<
|
||||
MediaServiceDeps,
|
||||
"repos" | "inference" | "mediaRoot" | "sources"
|
||||
> &
|
||||
PrivacyToolOptions;
|
||||
|
||||
type AssetRow = Awaited<ReturnType<MediaRepos["getAssetRecord"]>>;
|
||||
type PersonRow = Awaited<ReturnType<MediaRepos["listPersonRecords"]>>[number];
|
||||
|
||||
/** reframe_out boxes are unioned per bucket of this length so the crop search stays small. */
|
||||
const REFRAME_BUCKET_MS = 1000;
|
||||
/** Landmark samples per partial-mode track. */
|
||||
const MAX_LANDMARK_SAMPLES = 120;
|
||||
|
||||
interface AssetPlan {
|
||||
asset: AssetRow;
|
||||
clips: EdlVideoClip[];
|
||||
windows: TimeWindow[];
|
||||
base: string;
|
||||
probe: ProbedSource;
|
||||
dir: string;
|
||||
candidates: CandidateTrack[];
|
||||
instructions: RedactionInstruction[];
|
||||
checks: PrivacyCheck[];
|
||||
synthetic: Array<{ personId: string | null; keyframes: BoxKeyframe[]; seed: number }>;
|
||||
inpaint: Array<{ markId: string; maskDir: string; keyframes: BoxKeyframe[] }>;
|
||||
reframe: Array<{
|
||||
check: PrivacyCheck;
|
||||
/** Boxes the crop must exclude. */
|
||||
boxes: Array<{ tMs: number; box: NormalizedBox }>;
|
||||
/** Applied when no crop can exclude the boxes. */
|
||||
fallback: () => void;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface PrepareInput {
|
||||
actor: MediaActor;
|
||||
projectId: string;
|
||||
edl: Edl;
|
||||
quality: MediaRenderQuality;
|
||||
workDir: string;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export async function prepareSources(
|
||||
deps: AnonymizeDeps,
|
||||
input: PrepareInput,
|
||||
): Promise<MediaPreparedSources> {
|
||||
const { actor, edl, signal } = input;
|
||||
const spaceId = actor.spaceId;
|
||||
const workDir = await resolveStored(deps.mediaRoot, input.workDir);
|
||||
await mkdir(workDir, { recursive: true });
|
||||
const privacyDir = path.join(workDir, "privacy");
|
||||
await rm(privacyDir, { recursive: true, force: true });
|
||||
// A record from an earlier attempt must never vouch for this one.
|
||||
await rm(path.join(workDir, PRIVACY_RECORD_FILE), { force: true });
|
||||
|
||||
const clipsByAsset = new Map<string, EdlVideoClip[]>();
|
||||
for (const clip of edl.tracks.video) {
|
||||
clipsByAsset.set(clip.assetId, [...(clipsByAsset.get(clip.assetId) ?? []), clip]);
|
||||
}
|
||||
const assetIds = [...clipsByAsset.keys()].sort();
|
||||
const assets = await deps.repos.listAssetRecords(spaceId, assetIds);
|
||||
if (assets.length !== assetIds.length)
|
||||
throw new PrivacyUnavailableError("a clip asset is missing");
|
||||
|
||||
const project = await deps.repos.getProjectRecord(spaceId, input.projectId);
|
||||
const kit = project.brandKitId
|
||||
? (await deps.repos.getBrandKit(spaceId, project.brandKitId)).kit
|
||||
: null;
|
||||
const people = await deps.repos.listPersonRecords(spaceId);
|
||||
const peopleById = new Map(people.map((person) => [person.id, person]));
|
||||
const defaults = new Map(
|
||||
people.map((person) => [person.id, person.anonymizeByDefault as AnonymizeMode]),
|
||||
);
|
||||
const modes = effectiveModes(edl.redactions.faces, defaults);
|
||||
const faceWork =
|
||||
edl.redactions.faces.length > 0 || [...modes.values()].some((mode) => mode !== "none");
|
||||
const capabilities = await inferenceCapabilities(deps.inference, signal);
|
||||
const stored = await deps.repos.faceTrackRecords(spaceId, assetIds);
|
||||
const centroids = faceWork ? await deps.repos.personCentroids(spaceId) : [];
|
||||
|
||||
const fallbacks: string[] = [];
|
||||
const alterations = new Set<AiAlteration>();
|
||||
if (edl.redactions.voice) alterations.add("voice_disguise");
|
||||
const record: PrivacyRecord = {
|
||||
version: 1,
|
||||
assets: {},
|
||||
trackingGaps: [],
|
||||
reviews: [],
|
||||
expectedFaces: [],
|
||||
unassignedFaces: [],
|
||||
aiAlterations: [],
|
||||
protectedPeople: {},
|
||||
visiblePeople: [],
|
||||
protectUnassigned: false,
|
||||
};
|
||||
|
||||
// 1. Files and face candidates per asset.
|
||||
const plans: AssetPlan[] = [];
|
||||
for (const asset of assets) {
|
||||
signal?.throwIfAborted();
|
||||
const clips = clipsByAsset.get(asset.id) ?? [];
|
||||
const base = await resolveAssetFile(deps, spaceId, asset, input.quality);
|
||||
const probe = await probeSource(base, asset.kind, { ...deps, signal });
|
||||
const windows = probe.still ? [{ startMs: 0, endMs: 0 }] : clipWindows(clips);
|
||||
const plan: AssetPlan = {
|
||||
asset,
|
||||
clips,
|
||||
windows,
|
||||
base,
|
||||
probe,
|
||||
dir: path.join(privacyDir, asset.id),
|
||||
candidates: [],
|
||||
instructions: [],
|
||||
checks: [],
|
||||
synthetic: [],
|
||||
inpaint: [],
|
||||
reframe: [],
|
||||
};
|
||||
const storedTracks = stored.filter((track) => track.assetId === asset.id);
|
||||
const fromStore: CandidateTrack[] = [];
|
||||
for (const track of storedTracks) {
|
||||
fromStore.push({
|
||||
id: track.id,
|
||||
storedId: track.id,
|
||||
personId: track.personId,
|
||||
keyframes: await readTrackKeyframes(deps.mediaRoot, track.keyframesPath),
|
||||
});
|
||||
}
|
||||
if (faceWork) {
|
||||
if (deps.inference && operationAvailable(capabilities, "trackFaces")) {
|
||||
const file = (await analysisFile(deps, spaceId, asset)) ?? (await workingCopy(deps, plan));
|
||||
const fresh = await deps.inference.trackFaces({
|
||||
path: file,
|
||||
sampleFps: FACE_TRACK_FPS,
|
||||
signal,
|
||||
});
|
||||
plan.candidates = mergeTracks(asset.id, fromStore, fresh.tracks, centroids);
|
||||
} else {
|
||||
const analysed = await deps.repos.latestAnalysis(spaceId, asset.id, "faces");
|
||||
if (!analysed) {
|
||||
throw new PrivacyUnavailableError(
|
||||
`faces of asset ${asset.id} were never detected and the media worker is unavailable`,
|
||||
);
|
||||
}
|
||||
plan.candidates = fromStore;
|
||||
}
|
||||
} else {
|
||||
plan.candidates = fromStore;
|
||||
}
|
||||
plan.candidates = plan.candidates.filter((track) => visibleIn(track, plan));
|
||||
plans.push(plan);
|
||||
}
|
||||
|
||||
// 2. Who is visible and who is protected.
|
||||
const visible = new Set<string>();
|
||||
for (const plan of plans) {
|
||||
for (const track of plan.candidates) {
|
||||
if (track.personId) visible.add(track.personId);
|
||||
else {
|
||||
const span = spanOf(track);
|
||||
if (span) record.unassignedFaces.push({ assetId: plan.asset.id, ...span });
|
||||
}
|
||||
}
|
||||
}
|
||||
record.visiblePeople = [...visible].sort();
|
||||
for (const [personId, mode] of modes) {
|
||||
if (mode !== "none" && visible.has(personId)) record.protectedPeople[personId] = mode;
|
||||
}
|
||||
record.protectUnassigned = Object.keys(record.protectedPeople).length > 0;
|
||||
const colors = {
|
||||
fill: kit?.colors.primary,
|
||||
sticker: kit?.colors.accent ?? kit?.colors.primary,
|
||||
};
|
||||
|
||||
// 3. Faces.
|
||||
const syntheticReady = operationAvailable(capabilities, "synthesizeIdentity");
|
||||
for (const plan of plans) {
|
||||
for (const track of plan.candidates) {
|
||||
const protection = resolveProtection(track, {
|
||||
faces: edl.redactions.faces,
|
||||
defaults,
|
||||
protectUnassigned: record.protectUnassigned,
|
||||
});
|
||||
if (!protection) continue;
|
||||
const protectedTrack = protectKeyframes(track.keyframes, plan.windows, protection.minSizePct);
|
||||
const keyframes = plan.probe.still
|
||||
? stillKeyframes(protectedTrack.keyframes)
|
||||
: protectedTrack.keyframes;
|
||||
if (keyframes.length === 0) continue;
|
||||
if (!plan.probe.still) {
|
||||
for (const gap of protectedTrack.gaps) {
|
||||
record.trackingGaps.push({ assetId: plan.asset.id, personId: track.personId, ...gap });
|
||||
}
|
||||
}
|
||||
await planFace(deps, plan, {
|
||||
track,
|
||||
protection,
|
||||
keyframes,
|
||||
person: track.personId ? (peopleById.get(track.personId) ?? null) : null,
|
||||
syntheticReady,
|
||||
capabilities,
|
||||
colors,
|
||||
fallbacks,
|
||||
record,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Marks (asset marks and the standing marks of people seen in the asset).
|
||||
const edlMarks = new Map(edl.redactions.marks.map((mark) => [mark.markId, mark.treatment]));
|
||||
for (const plan of plans) {
|
||||
const marks = await deps.repos.marksForAsset(spaceId, plan.asset.id);
|
||||
for (const mark of marks) {
|
||||
const treatment: MarkTreatment = edlMarks.get(mark.id) ?? mark.treatment;
|
||||
if (treatment === "none") continue;
|
||||
const located = await locateMark(deps, plan, mark, signal);
|
||||
if (!located) continue;
|
||||
if (located.review) {
|
||||
record.reviews.push({ markId: mark.id, assetId: plan.asset.id, score: located.score });
|
||||
}
|
||||
const keyframes = plan.probe.still
|
||||
? stillKeyframes(located.keyframes)
|
||||
: located.keyframes.filter((k) => nearWindows(k.tMs, plan.windows));
|
||||
if (keyframes.length === 0) continue;
|
||||
const check: PrivacyCheck = {
|
||||
kind: "mark",
|
||||
id: mark.id,
|
||||
personId: mark.personId,
|
||||
markId: mark.id,
|
||||
keyframes,
|
||||
maskDir: located.maskDir,
|
||||
};
|
||||
const cover = () => {
|
||||
plan.instructions.push({
|
||||
keyframes,
|
||||
mode: "cover",
|
||||
paddingPct: MARK_PADDING_PCT,
|
||||
featherPct: 0,
|
||||
maskDir: located.maskDir,
|
||||
...(colors.fill ? { color: colors.fill } : {}),
|
||||
});
|
||||
plan.checks.push(check);
|
||||
};
|
||||
if (treatment === "blur") {
|
||||
plan.instructions.push({
|
||||
keyframes,
|
||||
mode: "blur",
|
||||
paddingPct: MARK_PADDING_PCT,
|
||||
featherPct: 0,
|
||||
maskDir: located.maskDir,
|
||||
});
|
||||
plan.checks.push(check);
|
||||
} else if (treatment === "cover") {
|
||||
cover();
|
||||
} else if (treatment === "reframe_out") {
|
||||
plan.reframe.push({
|
||||
check,
|
||||
boxes: keyframes.map((k) => ({ tMs: k.tMs, box: padBox(k.box, MARK_PADDING_PCT) })),
|
||||
fallback: () => {
|
||||
fallbacks.push(`mark ${mark.id}: no crop keeps it out of frame; covered instead`);
|
||||
cover();
|
||||
},
|
||||
});
|
||||
} else if (operationAvailable(capabilities, "inpaintVideo")) {
|
||||
plan.inpaint.push({ markId: mark.id, maskDir: located.maskDir, keyframes });
|
||||
plan.checks.push(check);
|
||||
} else {
|
||||
fallbacks.push(`mark ${mark.id}: inpainting is unavailable; covered instead`);
|
||||
cover();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Regions drawn in the EDL.
|
||||
for (const region of edl.redactions.regions) {
|
||||
const clip = edl.tracks.video.find((candidate) => candidate.id === region.clipId);
|
||||
const plan = plans.find((candidate) => candidate.asset.id === clip?.assetId);
|
||||
if (!clip || !plan) throw new PrivacyUnavailableError(`region ${region.id} has no clip`);
|
||||
const keyframes = plan.probe.still ? stillKeyframes(region.keyframes) : region.keyframes;
|
||||
const check: PrivacyCheck = {
|
||||
kind: "region",
|
||||
id: region.id,
|
||||
personId: null,
|
||||
markId: null,
|
||||
keyframes,
|
||||
maskDir: null,
|
||||
};
|
||||
const apply = (mode: "blur" | "cover") => {
|
||||
plan.instructions.push({
|
||||
keyframes,
|
||||
mode,
|
||||
paddingPct: 0,
|
||||
featherPct: 0,
|
||||
...(mode === "cover" && colors.fill ? { color: colors.fill } : {}),
|
||||
});
|
||||
plan.checks.push(check);
|
||||
};
|
||||
if (region.treatment === "reframe_out") {
|
||||
plan.reframe.push({
|
||||
check,
|
||||
boxes: keyframes,
|
||||
fallback: () => {
|
||||
fallbacks.push(`region ${region.id}: no crop keeps it out of frame; covered instead`);
|
||||
apply("cover");
|
||||
},
|
||||
});
|
||||
} else {
|
||||
apply(region.treatment);
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Apply per asset.
|
||||
const sources = new Map<string, CompileSource>();
|
||||
for (const plan of plans) {
|
||||
signal?.throwIfAborted();
|
||||
const crop = chooseCrop(plan, edl);
|
||||
const hasWork =
|
||||
plan.synthetic.length > 0 ||
|
||||
plan.inpaint.length > 0 ||
|
||||
plan.instructions.length > 0 ||
|
||||
crop !== null;
|
||||
if (!hasWork) {
|
||||
sources.set(plan.asset.id, compileSource(plan.base, plan.probe, plan.probe.rotation));
|
||||
continue;
|
||||
}
|
||||
const prepared = await applyPlan(deps, plan, crop, {
|
||||
capabilities,
|
||||
alterations,
|
||||
fallbacks,
|
||||
signal,
|
||||
});
|
||||
const probed = await probeSource(prepared, plan.asset.kind, { ...deps, signal });
|
||||
verifyPrepared(plan, crop, probed);
|
||||
sources.set(plan.asset.id, compileSource(prepared, probed, 0));
|
||||
record.assets[plan.asset.id] = {
|
||||
basePath: plan.base,
|
||||
preparedPath: prepared,
|
||||
width: plan.probe.width,
|
||||
height: plan.probe.height,
|
||||
still: plan.probe.still,
|
||||
crop,
|
||||
checks: plan.checks,
|
||||
};
|
||||
}
|
||||
|
||||
record.aiAlterations = [...alterations].sort();
|
||||
await writePrivacyRecord(workDir, record);
|
||||
return { sources, aiAlterations: record.aiAlterations, fallbacks };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Faces
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Fresh tracks take over stored ones they overlap (keeping the stored id and
|
||||
* person); unmatched fresh tracks are matched to people by embedding; stored
|
||||
* tracks the fresh pass missed stay (fail closed).
|
||||
*/
|
||||
export function mergeTracks(
|
||||
assetId: string,
|
||||
stored: CandidateTrack[],
|
||||
fresh: Array<{
|
||||
keyframes: Array<BoxKeyframe & { confidence: number }>;
|
||||
embedding: number[] | null;
|
||||
}>,
|
||||
centroids: Array<{ personId: string; centroid: Uint8Array }>,
|
||||
): CandidateTrack[] {
|
||||
const taken = new Set<string>();
|
||||
const merged: CandidateTrack[] = [];
|
||||
fresh.forEach((track, index) => {
|
||||
let best: { candidate: CandidateTrack; score: number } | null = null;
|
||||
for (const candidate of stored) {
|
||||
if (taken.has(candidate.id)) continue;
|
||||
const score = trackOverlap(track.keyframes, candidate.keyframes);
|
||||
if (score >= TRACK_MATCH_MIN_IOU && (!best || score > best.score))
|
||||
best = { candidate, score };
|
||||
}
|
||||
if (best) {
|
||||
taken.add(best.candidate.id);
|
||||
merged.push({
|
||||
...best.candidate,
|
||||
personId: best.candidate.personId ?? matchPerson(track.embedding, centroids),
|
||||
keyframes: [...track.keyframes, ...best.candidate.keyframes].sort((a, b) => a.tMs - b.tMs),
|
||||
});
|
||||
return;
|
||||
}
|
||||
merged.push({
|
||||
id: `fresh-${assetId}-${index + 1}`,
|
||||
storedId: null,
|
||||
personId: matchPerson(track.embedding, centroids),
|
||||
keyframes: track.keyframes,
|
||||
});
|
||||
});
|
||||
for (const candidate of stored) if (!taken.has(candidate.id)) merged.push(candidate);
|
||||
return merged;
|
||||
}
|
||||
|
||||
function nearWindows(tMs: number, windows: TimeWindow[]): boolean {
|
||||
return windows.some(
|
||||
(w) => tMs >= w.startMs - MAX_TRACKING_GAP_MS && tMs <= w.endMs + MAX_TRACKING_GAP_MS,
|
||||
);
|
||||
}
|
||||
|
||||
function spanOf(track: CandidateTrack): TimeWindow | null {
|
||||
if (track.keyframes.length === 0) return null;
|
||||
const times = track.keyframes.map((k) => k.tMs);
|
||||
return { startMs: Math.min(...times), endMs: Math.max(...times) };
|
||||
}
|
||||
|
||||
function visibleIn(track: CandidateTrack, plan: AssetPlan): boolean {
|
||||
if (plan.probe.still) return track.keyframes.length > 0;
|
||||
const span = spanOf(track);
|
||||
return span !== null && overlapsWindows(span, plan.windows);
|
||||
}
|
||||
|
||||
/** A still shows one moment: every box at once, held over the whole image. */
|
||||
function stillKeyframes(keyframes: BoxKeyframe[]): BoxKeyframe[] {
|
||||
const first = keyframes[0];
|
||||
if (!first) return [];
|
||||
const box = keyframes.reduce((acc, k) => unionBox(acc, k.box), first.box);
|
||||
return [
|
||||
{ tMs: 0, box },
|
||||
{ tMs: 1000, box },
|
||||
];
|
||||
}
|
||||
|
||||
/** Stable per-person seed for a synthetic identity (stored on first use). */
|
||||
export function syntheticSeedFor(id: string): number {
|
||||
return createHash("sha256").update(`synthetic:${id}`).digest().readInt32LE(0);
|
||||
}
|
||||
|
||||
async function planFace(
|
||||
deps: AnonymizeDeps,
|
||||
plan: AssetPlan,
|
||||
input: {
|
||||
track: CandidateTrack;
|
||||
protection: Protection;
|
||||
keyframes: BoxKeyframe[];
|
||||
person: PersonRow | null;
|
||||
syntheticReady: boolean;
|
||||
capabilities: MediaCapabilities | null;
|
||||
colors: { fill?: string; sticker?: string };
|
||||
fallbacks: string[];
|
||||
record: PrivacyRecord;
|
||||
signal?: AbortSignal;
|
||||
},
|
||||
): Promise<void> {
|
||||
const { track, protection, keyframes } = input;
|
||||
const who = track.personId ? `person ${track.personId}` : `face ${track.id}`;
|
||||
let mode = protection.mode;
|
||||
const padded = keyframes.map((k) => ({ tMs: k.tMs, box: padBox(k.box, protection.paddingPct) }));
|
||||
const check: PrivacyCheck = {
|
||||
kind: "face",
|
||||
id: track.id,
|
||||
personId: track.personId,
|
||||
markId: null,
|
||||
keyframes: padded,
|
||||
maskDir: null,
|
||||
};
|
||||
const span = spanOf({ ...track, keyframes });
|
||||
if (mode === "synthetic_identity") {
|
||||
if (input.syntheticReady) {
|
||||
let seed: number;
|
||||
if (input.person) {
|
||||
seed = input.person.syntheticSeed ?? syntheticSeedFor(input.person.id);
|
||||
if (input.person.syntheticSeed === null) {
|
||||
await deps.repos.updatePerson(plan.asset.spaceId, input.person.id, {
|
||||
syntheticSeed: seed,
|
||||
});
|
||||
input.person.syntheticSeed = seed;
|
||||
}
|
||||
} else {
|
||||
seed = syntheticSeedFor(track.id);
|
||||
}
|
||||
plan.synthetic.push({ personId: track.personId, keyframes, seed });
|
||||
plan.checks.push(check);
|
||||
if (span) {
|
||||
input.record.expectedFaces.push({
|
||||
assetId: plan.asset.id,
|
||||
personId: track.personId,
|
||||
mode,
|
||||
...span,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
input.fallbacks.push(`${who}: synthetic identity is not available; blurred instead`);
|
||||
mode = "blur";
|
||||
}
|
||||
if (mode === "reframe_out") {
|
||||
plan.reframe.push({
|
||||
check,
|
||||
boxes: padded,
|
||||
fallback: () => {
|
||||
input.fallbacks.push(`${who}: no crop keeps the face out of frame; blurred instead`);
|
||||
plan.instructions.push({
|
||||
keyframes,
|
||||
mode: "blur",
|
||||
paddingPct: protection.paddingPct,
|
||||
featherPct: protection.featherPct,
|
||||
});
|
||||
plan.checks.push(check);
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
const instruction: RedactionInstruction = {
|
||||
keyframes,
|
||||
mode,
|
||||
paddingPct: protection.paddingPct,
|
||||
featherPct: protection.featherPct,
|
||||
};
|
||||
if (mode === "solid" && input.colors.fill) instruction.color = input.colors.fill;
|
||||
if (mode === "sticker" && input.colors.sticker) instruction.color = input.colors.sticker;
|
||||
if (isPartial(mode)) {
|
||||
const partial = mode;
|
||||
const landmarks = await landmarksFor(deps, plan, keyframes, input.capabilities, input.signal);
|
||||
if (landmarks.length > 0) instruction.landmarks = landmarks;
|
||||
check.keyframes = padded.map((k) => ({ tMs: k.tMs, box: partialCheckBand(k.box, partial) }));
|
||||
if (span) {
|
||||
input.record.expectedFaces.push({
|
||||
assetId: plan.asset.id,
|
||||
personId: track.personId,
|
||||
mode,
|
||||
...span,
|
||||
});
|
||||
}
|
||||
}
|
||||
plan.instructions.push(instruction);
|
||||
plan.checks.push(check);
|
||||
}
|
||||
|
||||
/** Five-point landmarks per sampled keyframe for partial modes, when the provider has them. */
|
||||
async function landmarksFor(
|
||||
deps: AnonymizeDeps,
|
||||
plan: AssetPlan,
|
||||
keyframes: BoxKeyframe[],
|
||||
capabilities: MediaCapabilities | null,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Array<{ tMs: number; points: NormalizedPoint[] }>> {
|
||||
if (!deps.inference || !operationAvailable(capabilities, "faceLandmarks")) return [];
|
||||
const step = Math.max(1, Math.ceil(keyframes.length / MAX_LANDMARK_SAMPLES));
|
||||
const timesMs = keyframes.filter((_, i) => i % step === 0).map((k) => k.tMs);
|
||||
const file =
|
||||
(await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan));
|
||||
const result = await deps.inference.faceLandmarks({ path: file, timesMs, signal });
|
||||
const out: Array<{ tMs: number; points: NormalizedPoint[] }> = [];
|
||||
for (const frame of result.frames) {
|
||||
const expected = boxAt(keyframes, frame.tMs);
|
||||
if (!expected) continue;
|
||||
const best = frame.faces
|
||||
.map((face) => ({ face, score: iou(face.box, expected) }))
|
||||
.sort((a, b) => b.score - a.score)[0];
|
||||
if (best && best.score > 0.2) out.push({ tMs: frame.tMs, points: best.face.landmarks });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Marks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type MarkRow = Awaited<ReturnType<MediaRepos["marksForAsset"]>>[number];
|
||||
|
||||
/**
|
||||
* Mask and keyframes of a mark in this asset. Marks drawn on the asset use
|
||||
* their stored mask (segmenting now when none exists); standing person marks
|
||||
* are re-prompted at the same place relative to the person's face. Low
|
||||
* segmentation confidence sends the mark to review.
|
||||
*/
|
||||
async function locateMark(
|
||||
deps: AnonymizeDeps,
|
||||
plan: AssetPlan,
|
||||
mark: MarkRow,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ maskDir: string; keyframes: BoxKeyframe[]; score: number; review: boolean } | null> {
|
||||
const outDir = path.join(plan.dir, `mark-${mark.id}`);
|
||||
if (mark.assetId === plan.asset.id) {
|
||||
if (mark.maskDir) {
|
||||
const maskDir = await resolveStored(deps.mediaRoot, mark.maskDir);
|
||||
return { maskDir, keyframes: await maskKeyframes(maskDir), score: 1, review: false };
|
||||
}
|
||||
if (!mark.prompt) throw new PrivacyUnavailableError(`mark ${mark.id} has no prompt`);
|
||||
const file =
|
||||
(await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan));
|
||||
const segmented = await segmentMark(deps, { file, prompt: mark.prompt, outDir, signal });
|
||||
return {
|
||||
maskDir: segmented.maskDir,
|
||||
keyframes: segmented.keyframes,
|
||||
score: segmented.score,
|
||||
review: false,
|
||||
};
|
||||
}
|
||||
// A standing mark of a person seen in this asset.
|
||||
if (!mark.personId) return null;
|
||||
const tracks = plan.candidates.filter((track) => track.personId === mark.personId);
|
||||
if (tracks.length === 0) return null;
|
||||
const anchorFace = await originFace(deps, plan.asset.spaceId, mark);
|
||||
const target = tracks
|
||||
.map((track) =>
|
||||
firstConfident(track.keyframes, Math.min(...plan.windows.map((w) => w.startMs))),
|
||||
)
|
||||
.filter((k) => k !== null)
|
||||
.sort((a, b) => a.tMs - b.tMs)[0];
|
||||
const placed =
|
||||
anchorFace && target && mark.prompt
|
||||
? placeAnchor(anchorFromFace(promptBox(mark.prompt), anchorFace), target.box)
|
||||
: null;
|
||||
if (!placed || !target) {
|
||||
// Cannot relate the mark to the face here: cover the person's face region and review.
|
||||
const keyframes = tracks.flatMap((track) =>
|
||||
track.keyframes.map((k) => ({ tMs: k.tMs, box: k.box })),
|
||||
);
|
||||
return { maskDir: await emptyMaskDir(outDir), keyframes, score: 0, review: true };
|
||||
}
|
||||
const file =
|
||||
(await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan));
|
||||
const segmented = await segmentMark(deps, {
|
||||
file,
|
||||
prompt: { tMs: target.tMs, box: placed },
|
||||
outDir,
|
||||
signal,
|
||||
});
|
||||
return {
|
||||
maskDir: segmented.maskDir,
|
||||
keyframes: segmented.keyframes,
|
||||
score: segmented.score,
|
||||
review: segmented.score < MARK_REVIEW_MIN_SCORE,
|
||||
};
|
||||
}
|
||||
|
||||
/** The person's face in the frame a standing mark was drawn on. */
|
||||
async function originFace(
|
||||
deps: AnonymizeDeps,
|
||||
spaceId: string,
|
||||
mark: MarkRow,
|
||||
): Promise<NormalizedBox | null> {
|
||||
if (!mark.assetId || !mark.prompt || !mark.personId) return null;
|
||||
const tracks = (await deps.repos.faceTrackRecords(spaceId, [mark.assetId])).filter(
|
||||
(t) => t.personId === mark.personId,
|
||||
);
|
||||
for (const track of tracks) {
|
||||
const face = faceAt(
|
||||
await readTrackKeyframes(deps.mediaRoot, track.keyframesPath),
|
||||
mark.prompt.tMs,
|
||||
);
|
||||
if (face) return face;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function emptyMaskDir(dir: string): Promise<string> {
|
||||
await mkdir(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Applying
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** In-root copy of the base file for providers (once per asset). */
|
||||
async function workingCopy(deps: AnonymizeDeps, plan: AssetPlan): Promise<string> {
|
||||
if (!(await isInsideRoot(deps.mediaRoot, plan.base)))
|
||||
plan.base = await copyInto(plan.base, plan.dir);
|
||||
return plan.base;
|
||||
}
|
||||
|
||||
/** Static crop that keeps every reframe_out box out of frame; applies fallbacks otherwise. */
|
||||
function chooseCrop(plan: AssetPlan, edl: Edl): PixelBox | null {
|
||||
if (plan.reframe.length === 0) return null;
|
||||
const boxes = bucketBoxes(plan.reframe.flatMap((item) => item.boxes));
|
||||
const geometry = {
|
||||
sourceWidth: plan.probe.width,
|
||||
sourceHeight: plan.probe.height,
|
||||
outWidth: edl.output.width,
|
||||
outHeight: edl.output.height,
|
||||
durationMs: plan.probe.durationMs,
|
||||
};
|
||||
const keyframe = reframeExcluding(boxes, geometry)?.[0];
|
||||
const crop = keyframe ? cropPixels(geometry, keyframe) : null;
|
||||
if (
|
||||
crop &&
|
||||
boxes.every(({ box }) => outsideCrop(box, crop, plan.probe.width, plan.probe.height))
|
||||
) {
|
||||
for (const item of plan.reframe) plan.checks.push(item.check);
|
||||
return crop;
|
||||
}
|
||||
for (const item of plan.reframe) item.fallback();
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Union boxes per second so the crop search stays small; unions only widen the excluded area. */
|
||||
function bucketBoxes(boxes: Array<{ tMs: number; box: NormalizedBox }>) {
|
||||
const buckets = new Map<number, NormalizedBox>();
|
||||
for (const { tMs, box } of boxes) {
|
||||
const key = Math.floor(tMs / REFRAME_BUCKET_MS);
|
||||
const current = buckets.get(key);
|
||||
buckets.set(key, current ? unionBox(current, box) : box);
|
||||
}
|
||||
return [...buckets.entries()].map(([key, box]) => ({ tMs: key * REFRAME_BUCKET_MS, box }));
|
||||
}
|
||||
|
||||
function stepPath(plan: AssetPlan, file: string, step: string, ext = ".mp4"): string {
|
||||
const stem = path
|
||||
.basename(file, path.extname(file))
|
||||
.replace(/\.(?:synth|inpaint|redacted|cropped)[-\w]*$/, "");
|
||||
return path.join(plan.dir, `${stem}.${step}${ext}`);
|
||||
}
|
||||
|
||||
async function applyPlan(
|
||||
deps: AnonymizeDeps,
|
||||
plan: AssetPlan,
|
||||
crop: PixelBox | null,
|
||||
context: {
|
||||
capabilities: MediaCapabilities | null;
|
||||
alterations: Set<AiAlteration>;
|
||||
fallbacks: string[];
|
||||
signal?: AbortSignal;
|
||||
},
|
||||
): Promise<string> {
|
||||
const { signal } = context;
|
||||
await mkdir(plan.dir, { recursive: true });
|
||||
const needsProvider = plan.synthetic.length + plan.inpaint.length + plan.instructions.length > 0;
|
||||
if (needsProvider && !deps.inference) {
|
||||
throw new PrivacyUnavailableError("redaction needs the media worker, which is not configured");
|
||||
}
|
||||
let current = needsProvider ? await workingCopy(deps, plan) : plan.base;
|
||||
const inference = deps.inference;
|
||||
if (inference) {
|
||||
for (const [index, synthetic] of plan.synthetic.entries()) {
|
||||
const outPath = stepPath(plan, current, `synth-${index + 1}`);
|
||||
current = (
|
||||
await inference.synthesizeIdentity({
|
||||
path: current,
|
||||
track: synthetic.keyframes,
|
||||
seed: synthetic.seed,
|
||||
outPath,
|
||||
signal,
|
||||
})
|
||||
).outPath;
|
||||
context.alterations.add("synthetic_identity");
|
||||
}
|
||||
for (const mark of plan.inpaint) {
|
||||
const outPath = stepPath(plan, current, `inpaint-${mark.markId}`);
|
||||
const result = await inference.inpaintVideo({
|
||||
path: current,
|
||||
maskDir: mark.maskDir,
|
||||
outPath,
|
||||
signal,
|
||||
});
|
||||
if (result.quality.score >= MARK_QUALITY_MIN) {
|
||||
current = result.outPath;
|
||||
context.alterations.add("inpaint");
|
||||
continue;
|
||||
}
|
||||
// Never ship artifacts silently: low quality falls back to a cover.
|
||||
context.fallbacks.push(
|
||||
`mark ${mark.markId}: inpainting scored ${result.quality.score.toFixed(2)}; covered instead`,
|
||||
);
|
||||
await rm(result.outPath, { force: true });
|
||||
plan.instructions.push({
|
||||
keyframes: mark.keyframes,
|
||||
mode: "cover",
|
||||
paddingPct: MARK_PADDING_PCT,
|
||||
featherPct: 0,
|
||||
maskDir: mark.maskDir,
|
||||
});
|
||||
}
|
||||
if (plan.instructions.length > 0) {
|
||||
if (!operationAvailable(context.capabilities, "redact")) {
|
||||
throw new PrivacyUnavailableError("the media worker cannot redact");
|
||||
}
|
||||
const outPath = stepPath(plan, current, "redacted");
|
||||
const result = await inference.redact({
|
||||
path: current,
|
||||
redactions: plan.instructions,
|
||||
outPath,
|
||||
signal,
|
||||
});
|
||||
if (result.framesProcessed <= 0)
|
||||
throw new PrivacyUnavailableError("redaction processed no frames");
|
||||
current = result.outPath;
|
||||
}
|
||||
}
|
||||
if (crop) {
|
||||
const outPath = stepPath(plan, current, "cropped", plan.probe.still ? ".png" : ".mp4");
|
||||
await cropFile(current, outPath, crop, plan.probe.still, { ...deps, signal });
|
||||
current = outPath;
|
||||
}
|
||||
if (plan.probe.still && path.extname(current).toLowerCase() !== ".png" && current !== plan.base) {
|
||||
const outPath = stepPath(plan, current, "still", ".png");
|
||||
await firstFrame(current, outPath, { ...deps, signal });
|
||||
current = outPath;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
/** The prepared file must show the whole clip range at the expected size. */
|
||||
function verifyPrepared(plan: AssetPlan, crop: PixelBox | null, probed: ProbedSource): void {
|
||||
const width = crop ? crop.w : plan.probe.width;
|
||||
const height = crop ? crop.h : plan.probe.height;
|
||||
if (probed.width !== width || probed.height !== height) {
|
||||
throw new PrivacyUnavailableError(
|
||||
`redacted asset ${plan.asset.id} is ${probed.width}x${probed.height}, expected ${width}x${height}`,
|
||||
);
|
||||
}
|
||||
if (plan.probe.still) return;
|
||||
const neededMs = Math.max(...plan.clips.map((clip) => clip.outMs));
|
||||
const frameMs = 1000 / Math.max(1, plan.probe.fps);
|
||||
if (probed.durationMs + 2 * frameMs < Math.min(neededMs, plan.probe.durationMs)) {
|
||||
throw new PrivacyUnavailableError(`redacted asset ${plan.asset.id} is shorter than its clips`);
|
||||
}
|
||||
}
|
||||
|
||||
function compileSource(
|
||||
file: string,
|
||||
probe: ProbedSource,
|
||||
rotation: CompileSource["rotation"],
|
||||
): CompileSource {
|
||||
const quarter = rotation === 90 || rotation === 270;
|
||||
return {
|
||||
path: file,
|
||||
// CompileSource wants the coded size; probe reports the display size.
|
||||
width: quarter ? probe.height : probe.width,
|
||||
height: quarter ? probe.width : probe.height,
|
||||
durationMs: probe.durationMs,
|
||||
fps: probe.fps,
|
||||
hasAudio: probe.hasAudio,
|
||||
rotation,
|
||||
still: probe.still,
|
||||
};
|
||||
}
|
||||
97
packages/adapters/src/media-consent.test.ts
Normal file
97
packages/adapters/src/media-consent.test.ts
Normal file
@ -0,0 +1,97 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertExportConsent,
|
||||
ConsentRequiredError,
|
||||
consentIssues,
|
||||
consentRequirements,
|
||||
enforceConsentFlags,
|
||||
participantsOf,
|
||||
} from "./media-consent.js";
|
||||
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
|
||||
import { ACTOR, createPrivacyHarness, testEdl, testProfile } from "./media-privacy-fixtures.js";
|
||||
|
||||
const now = new Date("2026-06-01T12:00:00Z");
|
||||
const past = new Date("2026-01-01T00:00:00Z");
|
||||
|
||||
let harness: PrivacyHarness | null = null;
|
||||
afterEach(async () => {
|
||||
await harness?.cleanup();
|
||||
harness = null;
|
||||
});
|
||||
|
||||
describe("consent rules", () => {
|
||||
it("always requires consent and age proof for paid video", () => {
|
||||
expect(consentRequirements(testProfile())).toEqual({ consent: false, age: false });
|
||||
expect(consentRequirements(testProfile({ kind: "paid_video" }))).toEqual({
|
||||
consent: true,
|
||||
age: true,
|
||||
});
|
||||
expect(enforceConsentFlags(testProfile({ kind: "paid_video" })).consent).toEqual({
|
||||
requirePerformerConsent: true,
|
||||
requireAgeVerification: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts only recorded, past proof with a verification method", () => {
|
||||
const paid = testProfile({ kind: "paid_video" });
|
||||
const ok = {
|
||||
id: "a",
|
||||
consentRecordedAt: past,
|
||||
ageVerifiedAt: past,
|
||||
ageVerificationMethod: "platform_verified",
|
||||
};
|
||||
expect(consentIssues(paid, [ok], [], now)).toEqual([]);
|
||||
const future = { ...ok, consentRecordedAt: new Date("2027-01-01T00:00:00Z") };
|
||||
const noMethod = { ...ok, id: "b", ageVerificationMethod: "none" };
|
||||
expect(consentIssues(paid, [future, noMethod], [{ atMs: 10 }], now).map((i) => i.code)).toEqual(
|
||||
["consent_missing", "age_unverified", "unknown_performer"],
|
||||
);
|
||||
expect(consentIssues(testProfile(), [noMethod], [{}], now)).toEqual([]);
|
||||
});
|
||||
|
||||
it("collects the people and unassigned faces inside the clip ranges", () => {
|
||||
const edl = testEdl([{ id: "c1", assetId: "a1", inMs: 1000, outMs: 2000 }]);
|
||||
expect(
|
||||
participantsOf(edl, [
|
||||
{ assetId: "a1", personId: "alice", startMs: 0, endMs: 1200 },
|
||||
{ assetId: "a1", personId: "bob", startMs: 2500, endMs: 3000 },
|
||||
{ assetId: "a1", personId: null, startMs: 1500, endMs: 1600 },
|
||||
{ assetId: "a2", personId: "carol", startMs: 0, endMs: 5000 },
|
||||
]),
|
||||
).toEqual({
|
||||
personIds: ["alice"],
|
||||
unassigned: [{ assetId: "a1", startMs: 1500, endMs: 1600 }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("export consent", () => {
|
||||
it("blocks a paid export until every participant has consent and age proof", async () => {
|
||||
harness = await createPrivacyHarness(() => null);
|
||||
const alice = harness.addPerson("alice", "none");
|
||||
harness.store.tracks.push({
|
||||
id: "t1",
|
||||
assetId: "a1",
|
||||
personId: "alice",
|
||||
startMs: 0,
|
||||
endMs: 1000,
|
||||
confidenceMin: 0.9,
|
||||
keyframesPath: "x.json",
|
||||
modelVersion: "v1",
|
||||
});
|
||||
const edl = testEdl([{ id: "c1", assetId: "a1", inMs: 0, outMs: 1000 }]);
|
||||
const paid = testProfile({ kind: "paid_video" });
|
||||
await expect(assertExportConsent(harness.deps, ACTOR, { edl, profile: paid })).rejects.toThrow(
|
||||
ConsentRequiredError,
|
||||
);
|
||||
await expect(
|
||||
assertExportConsent(harness.deps, ACTOR, { edl, profile: testProfile() }),
|
||||
).resolves.toBeUndefined();
|
||||
alice.consentRecordedAt = past;
|
||||
alice.ageVerifiedAt = past;
|
||||
alice.ageVerificationMethod = "document_on_file";
|
||||
await expect(
|
||||
assertExportConsent(harness.deps, ACTOR, { edl, profile: paid }),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
161
packages/adapters/src/media-consent.ts
Normal file
161
packages/adapters/src/media-consent.ts
Normal file
@ -0,0 +1,161 @@
|
||||
import type { Edl, GateIssue, PlatformProfile } from "@rakazo/contracts";
|
||||
import type { MediaActor, MediaServiceDeps } from "./media-context.js";
|
||||
import { clipWindows, overlapsWindows } from "./media-privacy-plan.js";
|
||||
|
||||
/*
|
||||
* Consent and age proof (spec 0 and 6.4). A paid-video profile always needs
|
||||
* documented consent and age verification of every recognised participant,
|
||||
* whatever its own flags say; other profiles follow their consent flags.
|
||||
* Faces that belong to no person can never show consent and block.
|
||||
*/
|
||||
|
||||
export interface ConsentRequirements {
|
||||
consent: boolean;
|
||||
age: boolean;
|
||||
}
|
||||
|
||||
export function consentRequirements(profile: PlatformProfile): ConsentRequirements {
|
||||
const paid = profile.kind === "paid_video";
|
||||
return {
|
||||
consent: paid || profile.consent.requirePerformerConsent,
|
||||
age: paid || profile.consent.requireAgeVerification,
|
||||
};
|
||||
}
|
||||
|
||||
/** The profile with its consent flags raised to what the hard rules demand. */
|
||||
export function enforceConsentFlags(profile: PlatformProfile): PlatformProfile {
|
||||
const required = consentRequirements(profile);
|
||||
return {
|
||||
...profile,
|
||||
consent: { requirePerformerConsent: required.consent, requireAgeVerification: required.age },
|
||||
};
|
||||
}
|
||||
|
||||
export interface PersonConsentState {
|
||||
id: string;
|
||||
consentRecordedAt: Date | string | null;
|
||||
ageVerifiedAt: Date | string | null;
|
||||
ageVerificationMethod: string;
|
||||
}
|
||||
|
||||
/** Valid proof: a recorded date that is not in the future. */
|
||||
function proven(value: Date | string | null, now: Date): boolean {
|
||||
if (!value) return false;
|
||||
const at = value instanceof Date ? value : new Date(value);
|
||||
return !Number.isNaN(at.getTime()) && at.getTime() <= now.getTime();
|
||||
}
|
||||
|
||||
export function hasConsent(person: PersonConsentState, now: Date): boolean {
|
||||
return proven(person.consentRecordedAt, now);
|
||||
}
|
||||
|
||||
export function hasAgeProof(person: PersonConsentState, now: Date): boolean {
|
||||
return person.ageVerificationMethod !== "none" && proven(person.ageVerifiedAt, now);
|
||||
}
|
||||
|
||||
/**
|
||||
* Consent issues for the participants of an export. `unassigned` counts faces
|
||||
* no person could be matched to; each blocks because nobody can consent for it.
|
||||
*/
|
||||
export function consentIssues(
|
||||
profile: PlatformProfile,
|
||||
people: PersonConsentState[],
|
||||
unassigned: Array<{ atMs?: number; endMs?: number }>,
|
||||
now: Date,
|
||||
): GateIssue[] {
|
||||
const required = consentRequirements(profile);
|
||||
if (!required.consent && !required.age) return [];
|
||||
const issues: GateIssue[] = [];
|
||||
for (const person of [...people].sort((a, b) => a.id.localeCompare(b.id))) {
|
||||
if (required.consent && !hasConsent(person, now)) {
|
||||
issues.push({
|
||||
check: "consent",
|
||||
severity: "block",
|
||||
code: "consent_missing",
|
||||
message: "no recorded consent",
|
||||
personId: person.id,
|
||||
});
|
||||
}
|
||||
if (required.age && !hasAgeProof(person, now)) {
|
||||
issues.push({
|
||||
check: "consent",
|
||||
severity: "block",
|
||||
code: "age_unverified",
|
||||
message: "age is not verified",
|
||||
personId: person.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const face of unassigned) {
|
||||
issues.push({
|
||||
check: "consent",
|
||||
severity: "block",
|
||||
code: "unknown_performer",
|
||||
message: "a visible face is not assigned to a person with consent",
|
||||
...(face.atMs !== undefined ? { atMs: face.atMs } : {}),
|
||||
...(face.endMs !== undefined ? { endMs: face.endMs } : {}),
|
||||
});
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
/**
|
||||
* People and unassigned faces in the clip ranges of an EDL, from the stored
|
||||
* face tracks of its assets.
|
||||
*/
|
||||
export function participantsOf(
|
||||
edl: Edl,
|
||||
tracks: Array<{ assetId: string; personId: string | null; startMs: number; endMs: number }>,
|
||||
): {
|
||||
personIds: string[];
|
||||
unassigned: Array<{ assetId: string; startMs: number; endMs: number }>;
|
||||
} {
|
||||
const personIds = new Set<string>();
|
||||
const unassigned: Array<{ assetId: string; startMs: number; endMs: number }> = [];
|
||||
for (const track of tracks) {
|
||||
const windows = clipWindows(edl.tracks.video.filter((clip) => clip.assetId === track.assetId));
|
||||
if (!overlapsWindows(track, windows)) continue;
|
||||
if (track.personId) personIds.add(track.personId);
|
||||
else unassigned.push({ assetId: track.assetId, startMs: track.startMs, endMs: track.endMs });
|
||||
}
|
||||
return { personIds: [...personIds].sort(), unassigned };
|
||||
}
|
||||
|
||||
export class ConsentRequiredError extends Error {
|
||||
constructor(readonly issues: GateIssue[]) {
|
||||
super(
|
||||
`export blocked: ${issues.length} participant${issues.length === 1 ? "" : "s"} without documented consent or age proof`,
|
||||
);
|
||||
this.name = "ConsentRequiredError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Consent issues for exporting a render to a profile, re-read at export time
|
||||
* so a withdrawn consent stops a bundle that passed the gate earlier.
|
||||
*/
|
||||
export async function exportConsentIssues(
|
||||
deps: Pick<MediaServiceDeps, "repos" | "now">,
|
||||
actor: MediaActor,
|
||||
input: { edl: Edl; profile: PlatformProfile },
|
||||
): Promise<GateIssue[]> {
|
||||
const required = consentRequirements(input.profile);
|
||||
if (!required.consent && !required.age) return [];
|
||||
const assetIds = [...new Set(input.edl.tracks.video.map((clip) => clip.assetId))];
|
||||
const tracks = await deps.repos.faceTrackRecords(actor.spaceId, assetIds);
|
||||
const { personIds, unassigned } = participantsOf(input.edl, tracks);
|
||||
const people = personIds.length
|
||||
? await deps.repos.listPersonRecords(actor.spaceId, personIds)
|
||||
: [];
|
||||
return consentIssues(input.profile, people, unassigned, deps.now());
|
||||
}
|
||||
|
||||
/** Throws ConsentRequiredError when the export may not happen. */
|
||||
export async function assertExportConsent(
|
||||
deps: Pick<MediaServiceDeps, "repos" | "now">,
|
||||
actor: MediaActor,
|
||||
input: { edl: Edl; profile: PlatformProfile },
|
||||
): Promise<void> {
|
||||
const issues = await exportConsentIssues(deps, actor, input);
|
||||
if (issues.length > 0) throw new ConsentRequiredError(issues);
|
||||
}
|
||||
188
packages/adapters/src/media-gate-autofix.test.ts
Normal file
188
packages/adapters/src/media-gate-autofix.test.ts
Normal file
@ -0,0 +1,188 @@
|
||||
import type { Edl, GateReport } from "@rakazo/contracts";
|
||||
import { MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { autoFixRender, nonCompliantChange, proposeAutoFix } from "./media-gate-autofix.js";
|
||||
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
|
||||
import { ACTOR, createPrivacyHarness, testEdl } from "./media-privacy-fixtures.js";
|
||||
|
||||
let harness: PrivacyHarness | null = null;
|
||||
afterEach(async () => {
|
||||
await harness?.cleanup();
|
||||
harness = null;
|
||||
});
|
||||
|
||||
function edlWithFace(): Edl {
|
||||
return testEdl(
|
||||
[
|
||||
{ id: "c1", assetId: "a1", inMs: 0, outMs: 4000 },
|
||||
{ id: "c2", assetId: "a2", inMs: 0, outMs: 4000 },
|
||||
],
|
||||
{
|
||||
redactions: {
|
||||
faces: [
|
||||
{
|
||||
id: "f1",
|
||||
target: { personId: "alice" },
|
||||
mode: "blur",
|
||||
paddingPct: 20,
|
||||
featherPct: 10,
|
||||
minSizePct: 2,
|
||||
},
|
||||
],
|
||||
marks: [{ markId: "m1", treatment: "inpaint" }],
|
||||
regions: [],
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function blocked(issues: GateReport["issues"], round = 0): GateReport {
|
||||
return {
|
||||
profileId: "test-profile",
|
||||
passed: false,
|
||||
checkedAt: "2026-06-01T12:00:00.000Z",
|
||||
sampledFrames: 10,
|
||||
issues,
|
||||
autoFixRound: round,
|
||||
};
|
||||
}
|
||||
|
||||
const contentHit = {
|
||||
check: "content" as const,
|
||||
severity: "block" as const,
|
||||
code: "content_threshold",
|
||||
message: "explicit_nudity 0.90 above 0.5",
|
||||
atMs: 7000,
|
||||
endMs: 7500,
|
||||
category: "explicit_nudity" as const,
|
||||
score: 0.9,
|
||||
};
|
||||
|
||||
describe("proposeAutoFix", () => {
|
||||
it("only widens protection, covers marks, and cuts offending footage", () => {
|
||||
const edl = edlWithFace();
|
||||
const proposal = proposeAutoFix({
|
||||
edl,
|
||||
report: blocked([
|
||||
contentHit,
|
||||
{
|
||||
check: "anonymization",
|
||||
severity: "block",
|
||||
code: "face_not_covered",
|
||||
message: "face covered 80 %",
|
||||
atMs: 100,
|
||||
personId: "alice",
|
||||
},
|
||||
{
|
||||
check: "anonymization",
|
||||
severity: "block",
|
||||
code: "mark_visible",
|
||||
message: "visible",
|
||||
atMs: 200,
|
||||
markId: "m1",
|
||||
},
|
||||
]),
|
||||
round: 0,
|
||||
alternatives: [],
|
||||
});
|
||||
expect("edl" in proposal).toBe(true);
|
||||
if (!("edl" in proposal)) return;
|
||||
expect(proposal.edl.redactions.faces[0]!.paddingPct).toBe(45);
|
||||
expect(proposal.edl.redactions.marks[0]!.treatment).toBe("cover");
|
||||
expect(proposal.edl.tracks.video.find((c) => c.id === "c2")!.outMs).toBe(2500);
|
||||
expect(proposal.changes.length).toBeGreaterThanOrEqual(3);
|
||||
expect(nonCompliantChange(edl, proposal.edl)).toBeNull();
|
||||
});
|
||||
|
||||
it("stops after the round limit", () => {
|
||||
const proposal = proposeAutoFix({
|
||||
edl: edlWithFace(),
|
||||
report: blocked([contentHit], MAX_AUTO_FIX_ROUNDS),
|
||||
round: MAX_AUTO_FIX_ROUNDS,
|
||||
alternatives: [],
|
||||
});
|
||||
expect(proposal).toEqual({ reason: "round_limit", changes: [] });
|
||||
});
|
||||
|
||||
it("rejects any change that weakens protection or adds content", () => {
|
||||
const edl = edlWithFace();
|
||||
const weaker = structuredClone(edl);
|
||||
weaker.redactions.faces[0]!.paddingPct = 5;
|
||||
expect(nonCompliantChange(edl, weaker)).toMatch(/shrank/);
|
||||
const removed = structuredClone(edl);
|
||||
removed.redactions.faces = [];
|
||||
expect(nonCompliantChange(edl, removed)).toMatch(/removed/);
|
||||
const unmasked = structuredClone(edl);
|
||||
unmasked.redactions.marks[0]!.treatment = "blur";
|
||||
expect(nonCompliantChange(edl, unmasked)).toMatch(/changed to blur/);
|
||||
const grown = structuredClone(edl);
|
||||
grown.tracks.video[0]!.outMs = 9000;
|
||||
expect(nonCompliantChange(edl, grown)).toMatch(/grew/);
|
||||
const overlay = structuredClone(edl);
|
||||
overlay.overlays.push({ id: "o1", kind: "text", atMs: 0, durationMs: 100, text: "x" });
|
||||
expect(nonCompliantChange(edl, overlay)).toMatch(/overlays/);
|
||||
const swapped = structuredClone(edl);
|
||||
swapped.redactions.faces[0]!.mode = "partial_eyes";
|
||||
expect(nonCompliantChange(edl, swapped)).toMatch(/mode/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("autoFixRender", () => {
|
||||
async function blockedRender(round: number) {
|
||||
harness = await createPrivacyHarness(() => null);
|
||||
const edl = edlWithFace();
|
||||
harness.store.plans.push({ id: "plan-1", projectId: "project-1", edl });
|
||||
harness.store.renders.push({
|
||||
id: "render-1",
|
||||
planId: "plan-1",
|
||||
projectId: "project-1",
|
||||
quality: "draft",
|
||||
status: "blocked",
|
||||
autoFixRound: round,
|
||||
gateReport: blocked([contentHit], round),
|
||||
});
|
||||
harness.store.renders.push({ id: "render-2", status: "queued", autoFixRound: 0 });
|
||||
const submitPlan = vi.fn(async () => ({
|
||||
ok: true as const,
|
||||
plan: { id: "plan-2" } as never,
|
||||
}));
|
||||
const start = vi.fn(async () => ({ id: "render-2" }) as never);
|
||||
return {
|
||||
services: { deps: harness.deps, marketing: { submitPlan }, render: { start } },
|
||||
submitPlan,
|
||||
start,
|
||||
};
|
||||
}
|
||||
|
||||
it("submits a new plan with the reasons and renders it as the next round", async () => {
|
||||
const { services, submitPlan, start } = await blockedRender(1);
|
||||
const result = await autoFixRender(services, ACTOR, "render-1");
|
||||
expect(result.status).toBe("rerendering");
|
||||
expect(submitPlan).toHaveBeenCalledWith(
|
||||
ACTOR,
|
||||
expect.objectContaining({
|
||||
projectId: "project-1",
|
||||
createdBy: "agent",
|
||||
rationale: expect.stringMatching(/^Auto-fix round 2 of 3:\n- clip c2: /),
|
||||
}),
|
||||
);
|
||||
expect(start).toHaveBeenCalledWith(ACTOR, { planId: "plan-2", quality: "draft" });
|
||||
expect(harness!.store.renders[1]!.autoFixRound).toBe(2);
|
||||
});
|
||||
|
||||
it("hands over to the user after three rounds", async () => {
|
||||
const { services, submitPlan } = await blockedRender(MAX_AUTO_FIX_ROUNDS);
|
||||
const result = await autoFixRender(services, ACTOR, "render-1");
|
||||
expect(result).toEqual({ status: "user_required", reason: "round_limit", changes: [] });
|
||||
expect(submitPlan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does nothing for a render that is not blocked", async () => {
|
||||
const { services } = await blockedRender(0);
|
||||
harness!.store.renders[0]!.status = "done";
|
||||
expect(await autoFixRender(services, ACTOR, "render-1")).toMatchObject({
|
||||
status: "user_required",
|
||||
reason: "not_blocked",
|
||||
});
|
||||
});
|
||||
});
|
||||
193
packages/adapters/src/media-gate-autofix.ts
Normal file
193
packages/adapters/src/media-gate-autofix.ts
Normal file
@ -0,0 +1,193 @@
|
||||
import type { Edl, EdlError, GateReport, MediaRender, MediaRenderQuality } from "@rakazo/contracts";
|
||||
import { EdlSchema, GateReportSchema, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
|
||||
import { ACTIVE_RENDER_STATUSES } from "@rakazo/db";
|
||||
import { planAutoFix } from "@rakazo/media";
|
||||
import type {
|
||||
MediaActor,
|
||||
MediaMarketingService,
|
||||
MediaRenderService,
|
||||
MediaServiceDeps,
|
||||
} from "./media-context.js";
|
||||
|
||||
/*
|
||||
* Auto-fix for blocked renders (spec 6): only compliance-preserving repairs
|
||||
* (trim or drop a clip, take another segment, change the crop, widen a
|
||||
* redaction, cover a mark) through planAutoFix, re-checked here so no round
|
||||
* can weaken protection or add anything new. Each round submits a new plan
|
||||
* version with the reasons as rationale and starts a new render, which runs
|
||||
* the gate again. After MAX_AUTO_FIX_ROUNDS the user takes over.
|
||||
*/
|
||||
|
||||
export type AutoFixResult =
|
||||
| { status: "rerendering"; round: number; render: MediaRender; changes: string[] }
|
||||
| {
|
||||
status: "user_required";
|
||||
reason: "not_blocked" | "round_limit" | "no_safe_fix" | "plan_rejected";
|
||||
changes: string[];
|
||||
errors?: EdlError[];
|
||||
};
|
||||
|
||||
export interface AutoFixDeps {
|
||||
deps: Pick<MediaServiceDeps, "repos">;
|
||||
marketing: Pick<MediaMarketingService, "submitPlan">;
|
||||
render: Pick<MediaRenderService, "start">;
|
||||
}
|
||||
|
||||
/**
|
||||
* Why a proposed fix is not a pure compliance repair, or null when it is.
|
||||
* Allowed: fewer or shorter clips, other segments, other crops, wider padding
|
||||
* on the same face redactions, added blur redactions, marks switched to cover.
|
||||
* Forbidden: anything that removes or weakens protection or adds content.
|
||||
*/
|
||||
export function nonCompliantChange(before: Edl, after: Edl): string | null {
|
||||
const facesBefore = new Map(before.redactions.faces.map((f) => [f.id, f]));
|
||||
for (const face of before.redactions.faces) {
|
||||
const next = after.redactions.faces.find((f) => f.id === face.id);
|
||||
if (!next) return `face redaction ${face.id} was removed`;
|
||||
if (JSON.stringify(next.target) !== JSON.stringify(face.target))
|
||||
return `face redaction ${face.id} changed its target`;
|
||||
if (next.mode !== face.mode) return `face redaction ${face.id} changed its mode`;
|
||||
if (next.paddingPct < face.paddingPct || next.minSizePct < face.minSizePct)
|
||||
return `face redaction ${face.id} shrank`;
|
||||
}
|
||||
for (const face of after.redactions.faces) {
|
||||
if (!facesBefore.has(face.id) && face.mode !== "blur")
|
||||
return `added face redaction ${face.id} is not a blur`;
|
||||
}
|
||||
for (const mark of before.redactions.marks) {
|
||||
const next = after.redactions.marks.find((m) => m.markId === mark.markId);
|
||||
if (!next) return `mark ${mark.markId} lost its treatment`;
|
||||
if (next.treatment !== mark.treatment && next.treatment !== "cover")
|
||||
return `mark ${mark.markId} changed to ${next.treatment}`;
|
||||
}
|
||||
if (after.redactions.marks.length !== before.redactions.marks.length)
|
||||
return "mark treatments were added";
|
||||
const clipIds = new Set(after.tracks.video.map((clip) => clip.id));
|
||||
for (const region of before.redactions.regions) {
|
||||
if (!clipIds.has(region.clipId)) continue;
|
||||
const next = after.redactions.regions.find((r) => r.id === region.id);
|
||||
if (!next || JSON.stringify(next) !== JSON.stringify(region))
|
||||
return `region ${region.id} changed`;
|
||||
}
|
||||
if (after.redactions.regions.length > before.redactions.regions.length)
|
||||
return "regions were added";
|
||||
if (JSON.stringify(after.redactions.voice) !== JSON.stringify(before.redactions.voice))
|
||||
return "voice disguise changed";
|
||||
const beforeClips = new Map(before.tracks.video.map((clip) => [clip.id, clip]));
|
||||
for (const clip of after.tracks.video) {
|
||||
const old = beforeClips.get(clip.id);
|
||||
if (!old) return `clip ${clip.id} was added`;
|
||||
if (clip.speed !== old.speed || clip.muted !== old.muted)
|
||||
return `clip ${clip.id} changed speed or sound`;
|
||||
if (JSON.stringify(clip.color) !== JSON.stringify(old.color))
|
||||
return `clip ${clip.id} changed its colors`;
|
||||
if (clip.assetId === old.assetId && (clip.inMs < old.inMs || clip.outMs > old.outMs)) {
|
||||
if (clip.segmentId === old.segmentId) return `clip ${clip.id} grew`;
|
||||
}
|
||||
}
|
||||
if (after.overlays.length > before.overlays.length) return "overlays were added";
|
||||
for (const overlay of after.overlays) {
|
||||
const old = before.overlays.find((o) => o.id === overlay.id);
|
||||
if (!old || old.text !== overlay.text || old.kind !== overlay.kind)
|
||||
return `overlay ${overlay.id} changed`;
|
||||
}
|
||||
if (
|
||||
JSON.stringify(after.tracks.audio) !== JSON.stringify(before.tracks.audio) ||
|
||||
JSON.stringify(after.captions) !== JSON.stringify(before.captions) ||
|
||||
JSON.stringify(after.intro) !== JSON.stringify(before.intro) ||
|
||||
JSON.stringify(after.outro) !== JSON.stringify(before.outro) ||
|
||||
JSON.stringify(after.output) !== JSON.stringify(before.output)
|
||||
) {
|
||||
return "audio, captions, bookends, or output settings changed";
|
||||
}
|
||||
for (const alteration of after.metadata.aiAlterations) {
|
||||
if (!before.metadata.aiAlterations.includes(alteration))
|
||||
return `AI alteration ${alteration} was added`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Plan one auto-fix round for a blocked render, or say why the user must take over. */
|
||||
export function proposeAutoFix(input: {
|
||||
edl: Edl;
|
||||
report: GateReport;
|
||||
round: number;
|
||||
alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>;
|
||||
}): { edl: Edl; changes: string[] } | { reason: "round_limit" | "no_safe_fix"; changes: string[] } {
|
||||
if (input.round >= MAX_AUTO_FIX_ROUNDS) return { reason: "round_limit", changes: [] };
|
||||
const fix = planAutoFix(
|
||||
input.edl,
|
||||
{ ...input.report, autoFixRound: input.round },
|
||||
input.alternatives,
|
||||
);
|
||||
if (!fix) return { reason: "no_safe_fix", changes: [] };
|
||||
const parsed = EdlSchema.safeParse(fix.edl);
|
||||
if (!parsed.success) return { reason: "no_safe_fix", changes: fix.changes };
|
||||
const violation = nonCompliantChange(input.edl, parsed.data);
|
||||
if (violation)
|
||||
return { reason: "no_safe_fix", changes: [...fix.changes, `rejected: ${violation}`] };
|
||||
return { edl: parsed.data, changes: fix.changes };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one auto-fix round for a blocked render: new plan version (createdBy
|
||||
* agent, rationale lists every change) and a new render carrying the next
|
||||
* round number, whose gate decides again.
|
||||
*/
|
||||
export async function autoFixRender(
|
||||
services: AutoFixDeps,
|
||||
actor: MediaActor,
|
||||
renderId: string,
|
||||
): Promise<AutoFixResult> {
|
||||
const { repos } = services.deps;
|
||||
const row = await repos.getRenderRecord(actor.spaceId, renderId);
|
||||
const report = GateReportSchema.nullable().parse(row.gateReport ?? null);
|
||||
if (row.status !== "blocked" || !report || report.passed) {
|
||||
return { status: "user_required", reason: "not_blocked", changes: [] };
|
||||
}
|
||||
const plan = await repos.getPlan(actor.spaceId, row.planId);
|
||||
const project = await repos.getProjectRecord(actor.spaceId, row.projectId);
|
||||
const assetIds = [
|
||||
...new Set([...project.assetIds, ...plan.edl.tracks.video.map((clip) => clip.assetId)]),
|
||||
];
|
||||
const segments = await repos.listSegments(actor.spaceId, assetIds);
|
||||
const alternatives = segments
|
||||
.filter((segment) => !segment.paidOnly)
|
||||
.map((segment) => ({
|
||||
segmentId: segment.id,
|
||||
assetId: segment.assetId,
|
||||
startMs: segment.startMs,
|
||||
endMs: segment.endMs,
|
||||
}));
|
||||
const proposal = proposeAutoFix({ edl: plan.edl, report, round: row.autoFixRound, alternatives });
|
||||
if ("reason" in proposal) {
|
||||
return { status: "user_required", reason: proposal.reason, changes: proposal.changes };
|
||||
}
|
||||
const round = row.autoFixRound + 1;
|
||||
const rationale = [
|
||||
`Auto-fix round ${round} of ${MAX_AUTO_FIX_ROUNDS}:`,
|
||||
...proposal.changes,
|
||||
].join("\n- ");
|
||||
const submitted = await services.marketing.submitPlan(actor, {
|
||||
projectId: row.projectId,
|
||||
edl: proposal.edl,
|
||||
rationale,
|
||||
createdBy: "agent",
|
||||
});
|
||||
if (!submitted.ok) {
|
||||
return {
|
||||
status: "user_required",
|
||||
reason: "plan_rejected",
|
||||
changes: proposal.changes,
|
||||
errors: submitted.errors,
|
||||
};
|
||||
}
|
||||
const started = await services.render.start(actor, {
|
||||
planId: submitted.plan.id,
|
||||
quality: row.quality as MediaRenderQuality,
|
||||
});
|
||||
const updated = await repos.updateRender(actor.spaceId, started.id, { autoFixRound: round }, [
|
||||
...ACTIVE_RENDER_STATUSES,
|
||||
]);
|
||||
return { status: "rerendering", round, render: updated ?? started, changes: proposal.changes };
|
||||
}
|
||||
250
packages/adapters/src/media-gate.test.ts
Normal file
250
packages/adapters/src/media-gate.test.ts
Normal file
@ -0,0 +1,250 @@
|
||||
import { copyFile, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MediaInferenceProvider, TrackFacesResult } from "@rakazo/adapter-kit";
|
||||
import type { AnonymizeMode, Edl, PlatformProfile } from "@rakazo/contracts";
|
||||
import { decodeVectors } from "@rakazo/media";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { prepareSources } from "./media-anonymize.js";
|
||||
import { ffmpegAvailable } from "./media-emulator-clips.js";
|
||||
import { cutTimes, GATE_PIXEL_STEP_MS, pixelSampleTimes, runGate } from "./media-gate.js";
|
||||
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
|
||||
import {
|
||||
ACTOR,
|
||||
cleanProbe,
|
||||
createPrivacyHarness,
|
||||
testEdl,
|
||||
testProfile,
|
||||
} from "./media-privacy-fixtures.js";
|
||||
import { PRIVACY_RECORD_FILE } from "./media-privacy-plan.js";
|
||||
|
||||
const hasFfmpeg = ffmpegAvailable();
|
||||
let harness: PrivacyHarness | null = null;
|
||||
|
||||
afterEach(async () => {
|
||||
await harness?.cleanup();
|
||||
harness = null;
|
||||
});
|
||||
|
||||
/** Faces the fake detector reports on the render output. */
|
||||
let outputFaces: TrackFacesResult = { sampleFps: 4, tracks: [] };
|
||||
let classifierAvailable = true;
|
||||
|
||||
function withOutputDetector(emulator: PrivacyHarness["inference"]): MediaInferenceProvider {
|
||||
const provider = Object.create(emulator) as MediaInferenceProvider;
|
||||
provider.trackFaces = async (request) =>
|
||||
path.basename(request.path).startsWith("output") ? outputFaces : emulator.trackFaces(request);
|
||||
provider.capabilities = async () => {
|
||||
const caps = await emulator.capabilities();
|
||||
caps.operations.classifyContent.available = classifierAvailable;
|
||||
return caps;
|
||||
};
|
||||
return provider;
|
||||
}
|
||||
|
||||
async function prepared(
|
||||
mode: AnonymizeMode,
|
||||
options: { file?: string; unassigned?: boolean } = {},
|
||||
) {
|
||||
outputFaces = { sampleFps: 4, tracks: [] };
|
||||
classifierAvailable = true;
|
||||
harness = await createPrivacyHarness(withOutputDetector);
|
||||
const asset = await harness.addAsset(options.file ?? "clip__faces-2____person-alice__.mp4");
|
||||
harness.addPerson("alice", mode);
|
||||
await harness.analyseFaces(asset.id, ["alice", options.unassigned ? null : "alice"]);
|
||||
await harness.learnPerson("alice", asset.id);
|
||||
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
|
||||
const workDir = await harness.workDir(`gate-${mode}`);
|
||||
const result = await prepareSources(harness.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
const output = path.join(workDir, "output.mp4");
|
||||
await copyFile(result.sources.get(asset.id)!.path, output);
|
||||
return { h: harness, asset, edl, workDir, output };
|
||||
}
|
||||
|
||||
function gate(
|
||||
h: PrivacyHarness,
|
||||
input: { edl: Edl; output: string },
|
||||
options: {
|
||||
profile?: PlatformProfile;
|
||||
copy?: { title: string | null; caption: string | null; tags: string[] };
|
||||
probe?: Partial<ReturnType<typeof cleanProbe>>;
|
||||
} = {},
|
||||
) {
|
||||
return runGate(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl: input.edl,
|
||||
profile: options.profile ?? testProfile(),
|
||||
outputPath: input.output,
|
||||
ffprobe: { ...cleanProbe(input.edl), ...options.probe },
|
||||
copy: options.copy ?? { title: null, caption: null, tags: [] },
|
||||
bundleHasDisclosure: false,
|
||||
autoFixRound: 0,
|
||||
});
|
||||
}
|
||||
|
||||
const box = { x: 0.4, y: 0.4, w: 0.2, h: 0.2 };
|
||||
const strangerEmbedding = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
|
||||
|
||||
describe("gate sampling", () => {
|
||||
it("samples each clip range and spreads long ones", () => {
|
||||
const edl = testEdl([
|
||||
{ id: "a", assetId: "x", inMs: 0, outMs: 1000 },
|
||||
{ id: "b", assetId: "x", inMs: 5000, outMs: 6000 },
|
||||
]);
|
||||
expect(cutTimes(edl)).toEqual([0, 999, 1000, 1999]);
|
||||
expect(pixelSampleTimes(edl.tracks.video, false)).toEqual([0, 500, 960, 5000, 5500, 5960]);
|
||||
expect(pixelSampleTimes(edl.tracks.video, true)).toEqual([0]);
|
||||
const long = pixelSampleTimes([{ ...edl.tracks.video[0]!, outMs: 600_000 }], false);
|
||||
expect(long).toHaveLength(40);
|
||||
expect(long[1]! - long[0]!).toBeGreaterThanOrEqual(GATE_PIXEL_STEP_MS);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasFfmpeg)("runGate", () => {
|
||||
it("passes a protected render with a clean classifier", async () => {
|
||||
const { h, edl, output } = await prepared("blur");
|
||||
const report = await gate(h, { edl, output });
|
||||
expect(report.issues).toEqual([]);
|
||||
expect(report.passed).toBe(true);
|
||||
expect(report.sampledFrames).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("classifies the rendered output and blocks above the threshold", async () => {
|
||||
const { h, edl, workDir, output } = await prepared("blur");
|
||||
const flagged = path.join(workDir, "output__nsfw-explicit_nudity-0.9__.mp4");
|
||||
await copyFile(output, flagged);
|
||||
const report = await gate(h, { edl, output: flagged });
|
||||
expect(report.passed).toBe(false);
|
||||
expect(report.issues).toContainEqual(
|
||||
expect.objectContaining({
|
||||
check: "content",
|
||||
code: "content_threshold",
|
||||
category: "explicit_nudity",
|
||||
score: 0.9,
|
||||
atMs: 0,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("blocks when the classifier is unavailable", async () => {
|
||||
const { h, edl, output } = await prepared("blur");
|
||||
classifierAvailable = false;
|
||||
const report = await gate(h, { edl, output });
|
||||
expect(report.issues).toContainEqual(
|
||||
expect.objectContaining({ code: "classifier_unavailable", severity: "block" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("blocks a recognisable protected face and an unknown face in the output", async () => {
|
||||
const { h, edl, output } = await prepared("blur");
|
||||
const alice = h.store.people[0]!;
|
||||
const embedding = [...decodeVectors(alice.centroid!, 16)[0]!];
|
||||
outputFaces = {
|
||||
sampleFps: 4,
|
||||
tracks: [
|
||||
{ trackId: "o1", keyframes: [{ tMs: 250, box, confidence: 0.9 }], embedding },
|
||||
{
|
||||
trackId: "o2",
|
||||
keyframes: [{ tMs: 750, box, confidence: 0.9 }],
|
||||
embedding: strangerEmbedding,
|
||||
},
|
||||
],
|
||||
};
|
||||
const report = await gate(h, { edl, output });
|
||||
expect(report.issues).toContainEqual(
|
||||
expect.objectContaining({ code: "face_not_covered", personId: "alice", atMs: 250 }),
|
||||
);
|
||||
expect(report.issues).toContainEqual(
|
||||
expect.objectContaining({ code: "unknown_face", atMs: 750 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("lets a partially covered face stay detectable without counting it as unknown", async () => {
|
||||
const { h, edl, workDir, output } = await prepared("partial_eyes");
|
||||
expect(path.basename(workDir)).toBe("gate-partial_eyes");
|
||||
outputFaces = {
|
||||
sampleFps: 4,
|
||||
tracks: [
|
||||
{
|
||||
trackId: "o1",
|
||||
keyframes: [{ tMs: 250, box, confidence: 0.9 }],
|
||||
embedding: strangerEmbedding,
|
||||
},
|
||||
],
|
||||
};
|
||||
const report = await gate(h, { edl, output });
|
||||
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]);
|
||||
});
|
||||
|
||||
it("requires consent and age proof of every participant for paid video", async () => {
|
||||
const { h, edl, output } = await prepared("none");
|
||||
// The profile's own flags are off; paid video turns them on regardless.
|
||||
const paid = testProfile({ kind: "paid_video" });
|
||||
const blocked = await gate(h, { edl, output }, { profile: paid });
|
||||
expect(blocked.passed).toBe(false);
|
||||
expect(blocked.issues.filter((i) => i.check === "consent").map((i) => i.code)).toEqual([
|
||||
"consent_missing",
|
||||
"age_unverified",
|
||||
]);
|
||||
const alice = h.store.people[0]!;
|
||||
alice.consentRecordedAt = new Date("2026-01-01T00:00:00Z");
|
||||
alice.ageVerifiedAt = new Date("2026-01-01T00:00:00Z");
|
||||
alice.ageVerificationMethod = "document_on_file";
|
||||
const allowed = await gate(h, { edl, output }, { profile: paid });
|
||||
expect(allowed.issues.filter((i) => i.check === "consent")).toEqual([]);
|
||||
expect(allowed.passed).toBe(true);
|
||||
});
|
||||
|
||||
it("blocks consent profiles when a face belongs to nobody", async () => {
|
||||
const { h, edl, output } = await prepared("none", { unassigned: true });
|
||||
const profile = testProfile({
|
||||
consent: { requirePerformerConsent: true, requireAgeVerification: true },
|
||||
});
|
||||
const alice = h.store.people[0]!;
|
||||
alice.consentRecordedAt = new Date("2026-01-01T00:00:00Z");
|
||||
alice.ageVerifiedAt = new Date("2026-01-01T00:00:00Z");
|
||||
alice.ageVerificationMethod = "platform_verified";
|
||||
const report = await gate(h, { edl, output }, { profile });
|
||||
expect(report.issues).toEqual([
|
||||
expect.objectContaining({ check: "consent", code: "unknown_performer", atMs: 0 }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("blocks banned text and a duration outside the profile", async () => {
|
||||
const { h, edl, output } = await prepared("blur");
|
||||
const profile = testProfile({
|
||||
duration: { hardMinMs: 0, hardMaxMs: 1500 },
|
||||
text: {
|
||||
captionMaxChars: 100,
|
||||
hashtagLimit: 3,
|
||||
linkPolicy: "forbidden",
|
||||
bannedPatterns: [{ pattern: "free\\s+money", reason: "spam" }],
|
||||
},
|
||||
});
|
||||
const report = await gate(
|
||||
h,
|
||||
{ edl, output },
|
||||
{ profile, copy: { title: "Free money", caption: "see example.com", tags: [] } },
|
||||
);
|
||||
const codes = report.issues.map((issue) => issue.code);
|
||||
expect(codes).toEqual(
|
||||
expect.arrayContaining(["duration_too_long", "banned_pattern", "link_forbidden"]),
|
||||
);
|
||||
expect(report.passed).toBe(false);
|
||||
});
|
||||
|
||||
it("blocks when the sources were not prepared for this render", async () => {
|
||||
const { h, edl, workDir, output } = await prepared("blur");
|
||||
await rm(path.join(workDir, PRIVACY_RECORD_FILE));
|
||||
const report = await gate(h, { edl, output });
|
||||
expect(report.issues).toContainEqual(
|
||||
expect.objectContaining({ code: "privacy_unverified", severity: "block" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
396
packages/adapters/src/media-gate.ts
Normal file
396
packages/adapters/src/media-gate.ts
Normal file
@ -0,0 +1,396 @@
|
||||
import path from "node:path";
|
||||
import type {
|
||||
ContentCategory,
|
||||
Edl,
|
||||
EdlVideoClip,
|
||||
FfprobeReport,
|
||||
GateIssue,
|
||||
GateReport,
|
||||
NormalizedBox,
|
||||
PlatformProfile,
|
||||
} from "@rakazo/contracts";
|
||||
import type { GateInput } from "@rakazo/media";
|
||||
import { clipLengthMs, evaluateGate, introMs, MAX_MARK_RESIDUAL } from "@rakazo/media";
|
||||
import { assertInside } from "@rakazo/media/node";
|
||||
import { consentIssues, enforceConsentFlags, hasAgeProof, hasConsent } from "./media-consent.js";
|
||||
import type { MediaActor, MediaServiceDeps } from "./media-context.js";
|
||||
import type { PrivacyToolOptions } from "./media-privacy-io.js";
|
||||
import { inferenceCapabilities, maskTimes, operationAvailable } from "./media-privacy-io.js";
|
||||
import type { LumaFrame } from "./media-privacy-pixels.js";
|
||||
import {
|
||||
controlAgreement,
|
||||
extractLuma,
|
||||
readPngLuma,
|
||||
residualVisibility,
|
||||
} from "./media-privacy-pixels.js";
|
||||
import type { PrivacyCheck, PrivacyRecord, TimeWindow } from "./media-privacy-plan.js";
|
||||
import {
|
||||
boxAt,
|
||||
isPartial,
|
||||
matchPerson,
|
||||
readPrivacyRecord,
|
||||
sourceToOutput,
|
||||
} from "./media-privacy-plan.js";
|
||||
|
||||
/*
|
||||
* Gate execution on the rendered output (spec 6). Content is classified on the
|
||||
* output file at the profile's sample rate plus every cut; faces are tracked on
|
||||
* the output and matched to people locally. Protection is verified pixel by
|
||||
* pixel: for every protected region prepareSources recorded, the prepared
|
||||
* frame is compared with the unredacted frame at the same moment. The
|
||||
* compiler only crops, scales, and composites prepared frames, so a region
|
||||
* altered there stays altered in the output. Anything the gate cannot check
|
||||
* blocks (fail closed).
|
||||
*/
|
||||
|
||||
export type GateDeps = Pick<MediaServiceDeps, "repos" | "inference" | "mediaRoot" | "now"> &
|
||||
PrivacyToolOptions;
|
||||
|
||||
/** Face tracking rate on the output. */
|
||||
export const GATE_FACE_FPS = 4;
|
||||
/** Pixel verification step inside each clip range. */
|
||||
export const GATE_PIXEL_STEP_MS = 500;
|
||||
/** Pixel samples per asset at most (spread evenly). */
|
||||
export const MAX_PIXEL_SAMPLES_PER_ASSET = 40;
|
||||
/** Below this agreement outside the treated regions, two frames are not comparable. */
|
||||
const MIN_CONTROL_AGREEMENT = 0.8;
|
||||
|
||||
export interface GateRunInput {
|
||||
actor: MediaActor;
|
||||
projectId: string;
|
||||
edl: Edl;
|
||||
profile: PlatformProfile;
|
||||
outputPath: string;
|
||||
ffprobe: FfprobeReport;
|
||||
copy: { title: string | null; caption: string | null; tags: string[] };
|
||||
bundleHasDisclosure: boolean;
|
||||
autoFixRound: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
type Faces = GateInput["faces"];
|
||||
type Marks = GateInput["marks"];
|
||||
|
||||
function block(issue: Omit<GateIssue, "severity">): GateIssue {
|
||||
return { severity: "block", ...issue };
|
||||
}
|
||||
|
||||
/** Output times of every cut: clip starts and ends, intro end. */
|
||||
export function cutTimes(edl: Edl): number[] {
|
||||
const intro = introMs(edl);
|
||||
const times = new Set<number>();
|
||||
if (intro > 0) times.add(intro);
|
||||
for (const clip of edl.tracks.video) {
|
||||
times.add(intro + clip.atMs);
|
||||
times.add(Math.max(0, intro + clip.atMs + clipLengthMs(clip) - 1));
|
||||
}
|
||||
return [...times].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function clipsOf(edl: Edl, assetId: string): EdlVideoClip[] {
|
||||
return edl.tracks.video.filter((clip) => clip.assetId === assetId);
|
||||
}
|
||||
|
||||
/** Output windows of a source span across every clip of its asset. */
|
||||
function toOutput(edl: Edl, assetId: string, span: TimeWindow): TimeWindow[] {
|
||||
return clipsOf(edl, assetId).flatMap((clip) => {
|
||||
const mapped = sourceToOutput(edl, clip, span);
|
||||
return mapped ? [mapped] : [];
|
||||
});
|
||||
}
|
||||
|
||||
/** Evaluate the full gate for a render output. */
|
||||
export async function runGate(deps: GateDeps, input: GateRunInput): Promise<GateReport> {
|
||||
const { actor, edl, signal } = input;
|
||||
const profile = enforceConsentFlags(input.profile);
|
||||
const outputPath = await assertInside(deps.mediaRoot, input.outputPath);
|
||||
const record = await readPrivacyRecord(path.dirname(outputPath));
|
||||
const capabilities = await inferenceCapabilities(deps.inference, signal);
|
||||
const extra: GateIssue[] = [];
|
||||
const now = deps.now();
|
||||
|
||||
if (!record) {
|
||||
extra.push(
|
||||
block({
|
||||
check: "anonymization",
|
||||
code: "privacy_unverified",
|
||||
message: "sources were not prepared for this render, so protection cannot be shown",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Content on the rendered output.
|
||||
let content: GateInput["content"] = [];
|
||||
let sampledFrames = 0;
|
||||
if (Object.keys(profile.content.thresholds).length > 0) {
|
||||
if (deps.inference && operationAvailable(capabilities, "classifyContent")) {
|
||||
const classified = await deps.inference.classifyContent({
|
||||
path: outputPath,
|
||||
sampleFps: profile.content.sampleFps,
|
||||
extraTimesMs: cutTimes(edl),
|
||||
signal,
|
||||
});
|
||||
content = classified.frames.map((frame) => ({
|
||||
tMs: frame.tMs,
|
||||
scores: frame.scores as Partial<Record<ContentCategory, number>>,
|
||||
}));
|
||||
sampledFrames = content.length;
|
||||
} else {
|
||||
extra.push(
|
||||
block({
|
||||
check: "content",
|
||||
code: "classifier_unavailable",
|
||||
message: "the content classifier is unavailable, so the output cannot be checked",
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Faces re-detected on the output.
|
||||
const faces: Faces = [];
|
||||
const matchedPeople = new Set<string>();
|
||||
const protectedPeople = record?.protectedPeople ?? {};
|
||||
const consentRequired =
|
||||
profile.consent.requirePerformerConsent || profile.consent.requireAgeVerification;
|
||||
const needFaces = Object.keys(protectedPeople).length > 0 || consentRequired;
|
||||
if (needFaces) {
|
||||
if (deps.inference && operationAvailable(capabilities, "trackFaces")) {
|
||||
const tracked = await deps.inference.trackFaces({
|
||||
path: outputPath,
|
||||
sampleFps: GATE_FACE_FPS,
|
||||
signal,
|
||||
});
|
||||
const centroids = await deps.repos.personCentroids(actor.spaceId);
|
||||
const expected = expectedOutputFaces(edl, record);
|
||||
const unmatchedAt = new Map<number, number>();
|
||||
const slot = (tMs: number) => Math.round(tMs / (1000 / GATE_FACE_FPS));
|
||||
for (const track of tracked.tracks) {
|
||||
const personId = matchPerson(track.embedding, centroids);
|
||||
if (personId) matchedPeople.add(personId);
|
||||
const mode = personId ? protectedPeople[personId] : undefined;
|
||||
for (const keyframe of track.keyframes) {
|
||||
if (personId) {
|
||||
// Partial modes and synthetic identities stay detectable by design;
|
||||
// their regions are verified pixel by pixel below.
|
||||
if (mode && (isPartial(mode) || mode === "synthetic_identity")) continue;
|
||||
faces.push({ tMs: keyframe.tMs, box: keyframe.box, personId, coverage: mode ? 0 : 1 });
|
||||
continue;
|
||||
}
|
||||
const key = slot(keyframe.tMs);
|
||||
const seen = unmatchedAt.get(key) ?? 0;
|
||||
unmatchedAt.set(key, seen + 1);
|
||||
const allowance = expected.filter(
|
||||
(w) => keyframe.tMs >= w.startMs && keyframe.tMs <= w.endMs,
|
||||
).length;
|
||||
if (seen < allowance) continue;
|
||||
faces.push({ tMs: keyframe.tMs, box: keyframe.box, personId: null, coverage: 0 });
|
||||
}
|
||||
}
|
||||
} else {
|
||||
extra.push(
|
||||
block({
|
||||
check: "anonymization",
|
||||
code: "face_check_unavailable",
|
||||
message: "face detection is unavailable, so faces in the output cannot be checked",
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Pixel verification of every recorded protection.
|
||||
const marks: Marks = [];
|
||||
if (record) {
|
||||
const verified = await verifyPixels(deps, edl, record, signal);
|
||||
for (const sample of verified) {
|
||||
const covered = sample.residual <= MAX_MARK_RESIDUAL;
|
||||
const { check } = sample;
|
||||
if (check.kind === "face") {
|
||||
if (check.personId && protectedPeople[check.personId]) {
|
||||
faces.push({
|
||||
tMs: sample.outputMs,
|
||||
box: sample.box,
|
||||
personId: check.personId,
|
||||
coverage: covered ? 1 : 1 - sample.residual,
|
||||
});
|
||||
} else if (!covered) {
|
||||
extra.push(
|
||||
block({
|
||||
check: "anonymization",
|
||||
code: "face_not_covered",
|
||||
message: `face covered ${((1 - sample.residual) * 100).toFixed(0)} %`,
|
||||
atMs: sample.outputMs,
|
||||
endMs: sample.outputMs,
|
||||
score: 1 - sample.residual,
|
||||
...(check.personId ? { personId: check.personId } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
marks.push({
|
||||
markId: check.markId ?? check.id,
|
||||
tMs: sample.outputMs,
|
||||
residualVisibility: sample.residual,
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const review of record.reviews) {
|
||||
const window = toOutput(edl, review.assetId, {
|
||||
startMs: 0,
|
||||
endMs: Number.MAX_SAFE_INTEGER,
|
||||
})[0];
|
||||
extra.push(
|
||||
block({
|
||||
check: "anonymization",
|
||||
code: "mark_needs_review",
|
||||
message: `mark found with low confidence (${review.score.toFixed(2)}); review it`,
|
||||
markId: review.markId,
|
||||
...(window ? { atMs: window.startMs, endMs: window.endMs } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const trackingGaps: GateInput["trackingGaps"] = (record?.trackingGaps ?? []).flatMap((gap) =>
|
||||
toOutput(edl, gap.assetId, gap).map((window) => ({ ...window, personId: gap.personId })),
|
||||
);
|
||||
|
||||
// People in the output and their consent state.
|
||||
const ids = [...new Set([...(record?.visiblePeople ?? []), ...matchedPeople])].sort();
|
||||
const persons = ids.length ? await deps.repos.listPersonRecords(actor.spaceId, ids) : [];
|
||||
const people: GateInput["people"] = persons.map((person) => ({
|
||||
id: person.id,
|
||||
anonymizeByDefault: protectedPeople[person.id] ?? "none",
|
||||
consentRecordedAt: hasConsent(person, now)
|
||||
? new Date(person.consentRecordedAt!).toISOString()
|
||||
: null,
|
||||
ageVerifiedAt: hasAgeProof(person, now) ? new Date(person.ageVerifiedAt!).toISOString() : null,
|
||||
}));
|
||||
if (record && consentRequired) {
|
||||
const unassigned = record.unassignedFaces.flatMap((face) =>
|
||||
toOutput(edl, face.assetId, face).map((w) => ({ atMs: w.startMs, endMs: w.endMs })),
|
||||
);
|
||||
extra.push(...consentIssues(profile, [], unassigned, now));
|
||||
}
|
||||
|
||||
const report = evaluateGate({
|
||||
profile,
|
||||
edl,
|
||||
ffprobe: input.ffprobe,
|
||||
content,
|
||||
faces,
|
||||
trackingGaps,
|
||||
marks,
|
||||
people,
|
||||
copy: input.copy,
|
||||
aiAlterations: record?.aiAlterations ?? [],
|
||||
bundleHasDisclosure: input.bundleHasDisclosure,
|
||||
watermarkBox: null,
|
||||
sampledFrames,
|
||||
autoFixRound: input.autoFixRound,
|
||||
now,
|
||||
});
|
||||
const issues = [...report.issues, ...extra];
|
||||
return { ...report, issues, passed: !issues.some((issue) => issue.severity === "block") };
|
||||
}
|
||||
|
||||
/** Output windows where a face stays detectable by design (synthetic identity, partial modes). */
|
||||
function expectedOutputFaces(edl: Edl, record: PrivacyRecord | null): TimeWindow[] {
|
||||
return (record?.expectedFaces ?? []).flatMap((face) => toOutput(edl, face.assetId, face));
|
||||
}
|
||||
|
||||
interface PixelSample {
|
||||
check: PrivacyCheck;
|
||||
outputMs: number;
|
||||
box: NormalizedBox;
|
||||
residual: number;
|
||||
}
|
||||
|
||||
/** Source times to verify per asset: every GATE_PIXEL_STEP_MS inside its clips, spread evenly. */
|
||||
export function pixelSampleTimes(clips: EdlVideoClip[], still: boolean): number[] {
|
||||
if (still) return [0];
|
||||
const times = new Set<number>();
|
||||
for (const clip of clips) {
|
||||
for (let t = clip.inMs; t < clip.outMs; t += GATE_PIXEL_STEP_MS) times.add(t);
|
||||
times.add(Math.max(clip.inMs, clip.outMs - 40));
|
||||
}
|
||||
const sorted = [...times].sort((a, b) => a - b);
|
||||
if (sorted.length <= MAX_PIXEL_SAMPLES_PER_ASSET) return sorted;
|
||||
const step = (sorted.length - 1) / (MAX_PIXEL_SAMPLES_PER_ASSET - 1);
|
||||
return Array.from(
|
||||
{ length: MAX_PIXEL_SAMPLES_PER_ASSET },
|
||||
(_, i) => sorted[Math.round(i * step)]!,
|
||||
);
|
||||
}
|
||||
|
||||
async function verifyPixels(
|
||||
deps: GateDeps,
|
||||
edl: Edl,
|
||||
record: PrivacyRecord,
|
||||
signal?: AbortSignal,
|
||||
): Promise<PixelSample[]> {
|
||||
const samples: PixelSample[] = [];
|
||||
const maskCache = new Map<string, number[]>();
|
||||
for (const [assetId, entry] of Object.entries(record.assets).sort((a, b) =>
|
||||
a[0].localeCompare(b[0]),
|
||||
)) {
|
||||
if (entry.checks.length === 0) continue;
|
||||
const clips = clipsOf(edl, assetId);
|
||||
const treatedSize = entry.crop
|
||||
? { width: entry.crop.w, height: entry.crop.h }
|
||||
: { width: entry.width, height: entry.height };
|
||||
const offset = entry.crop ? { x: entry.crop.x, y: entry.crop.y } : undefined;
|
||||
for (const tMs of pixelSampleTimes(clips, entry.still)) {
|
||||
signal?.throwIfAborted();
|
||||
const active = entry.checks.flatMap((check) => {
|
||||
const box = entry.still ? (check.keyframes[0]?.box ?? null) : boxAt(check.keyframes, tMs);
|
||||
return box ? [{ check, box }] : [];
|
||||
});
|
||||
if (active.length === 0) continue;
|
||||
const tool = { ffmpegPath: deps.ffmpegPath, signal };
|
||||
const source = await extractLuma(
|
||||
entry.basePath,
|
||||
{ tMs, width: entry.width, height: entry.height, still: entry.still },
|
||||
tool,
|
||||
);
|
||||
const treated = await extractLuma(
|
||||
entry.preparedPath,
|
||||
{ tMs, ...treatedSize, still: entry.still },
|
||||
tool,
|
||||
);
|
||||
const aligned =
|
||||
controlAgreement({ source, treated, boxes: active.map((a) => a.box), offset }) >=
|
||||
MIN_CONTROL_AGREEMENT;
|
||||
const outputTimes = entry.still
|
||||
? clips.map((clip) => introMs(edl) + clip.atMs)
|
||||
: clips.flatMap((clip) => {
|
||||
const mapped = sourceToOutput(edl, clip, { startMs: tMs, endMs: tMs });
|
||||
return mapped ? [mapped.startMs] : [];
|
||||
});
|
||||
for (const { check, box } of active) {
|
||||
let residual = 1;
|
||||
if (aligned) {
|
||||
const mask = check.maskDir ? await maskAt(check.maskDir, tMs, maskCache) : undefined;
|
||||
residual = residualVisibility({ source, treated, box, offset, mask });
|
||||
}
|
||||
for (const outputMs of outputTimes) samples.push({ check, outputMs, box, residual });
|
||||
}
|
||||
}
|
||||
}
|
||||
return samples;
|
||||
}
|
||||
|
||||
/** The mask frame nearest at or before `tMs` (or the first one), if the directory has any. */
|
||||
async function maskAt(
|
||||
maskDir: string,
|
||||
tMs: number,
|
||||
cache: Map<string, number[]>,
|
||||
): Promise<LumaFrame | undefined> {
|
||||
let times = cache.get(maskDir);
|
||||
if (!times) {
|
||||
times = await maskTimes(maskDir).catch(() => []);
|
||||
cache.set(maskDir, times);
|
||||
}
|
||||
if (times.length === 0) return undefined;
|
||||
const before = times.filter((t) => t <= tMs).at(-1) ?? times[0]!;
|
||||
return readPngLuma(path.join(maskDir, `${before}.png`));
|
||||
}
|
||||
300
packages/adapters/src/media-marks.test.ts
Normal file
300
packages/adapters/src/media-marks.test.ts
Normal file
@ -0,0 +1,300 @@
|
||||
import { copyFile, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
|
||||
import type { Edl } from "@rakazo/contracts";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { prepareSources } from "./media-anonymize.js";
|
||||
import { ffmpegAvailable } from "./media-emulator-clips.js";
|
||||
import { runGate } from "./media-gate.js";
|
||||
import { anchorFromFace, createMark, placeAnchor, promptBox, updateMark } from "./media-marks.js";
|
||||
import { createMediaPrivacyService } from "./media-privacy.js";
|
||||
import type { PrivacyHarness } from "./media-privacy-fixtures.js";
|
||||
import {
|
||||
ACTOR,
|
||||
cleanProbe,
|
||||
createPrivacyHarness,
|
||||
testEdl,
|
||||
testProfile,
|
||||
} from "./media-privacy-fixtures.js";
|
||||
import { PrivacyUnavailableError } from "./media-privacy-io.js";
|
||||
import { readPrivacyRecord } from "./media-privacy-plan.js";
|
||||
|
||||
const hasFfmpeg = ffmpegAvailable();
|
||||
let harness: PrivacyHarness | null = null;
|
||||
|
||||
afterEach(async () => {
|
||||
await harness?.cleanup();
|
||||
harness = null;
|
||||
});
|
||||
|
||||
const tattoo = { tMs: 500, box: { x: 0.6, y: 0.55, w: 0.2, h: 0.25 } };
|
||||
|
||||
async function render(h: PrivacyHarness, edl: Edl, name: string) {
|
||||
const workDir = await h.workDir(name);
|
||||
const prepared = await prepareSources(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
quality: "final",
|
||||
workDir,
|
||||
});
|
||||
return { workDir, prepared };
|
||||
}
|
||||
|
||||
async function gate(h: PrivacyHarness, edl: Edl, workDir: string, prepared: string) {
|
||||
const outputPath = path.join(workDir, "output.mp4");
|
||||
await copyFile(prepared, outputPath);
|
||||
return runGate(h.deps, {
|
||||
actor: ACTOR,
|
||||
projectId: "project-1",
|
||||
edl,
|
||||
profile: testProfile(),
|
||||
outputPath,
|
||||
ffprobe: cleanProbe(edl),
|
||||
copy: { title: null, caption: null, tags: [] },
|
||||
bundleHasDisclosure: false,
|
||||
autoFixRound: 0,
|
||||
});
|
||||
}
|
||||
|
||||
describe("mark geometry", () => {
|
||||
it("turns a point prompt into a small box inside the frame", () => {
|
||||
expect(promptBox({ tMs: 0, point: { x: 0.99, y: 0.01 } })).toEqual({
|
||||
x: 0.88,
|
||||
y: 0,
|
||||
w: 0.12,
|
||||
h: 0.12,
|
||||
});
|
||||
expect(() => promptBox({ tMs: 0 })).toThrow(PrivacyUnavailableError);
|
||||
});
|
||||
|
||||
it("keeps a mark's place relative to the face", () => {
|
||||
const anchor = anchorFromFace(
|
||||
{ x: 0.5, y: 0.5, w: 0.1, h: 0.1 },
|
||||
{ x: 0.4, y: 0.3, w: 0.2, h: 0.2 },
|
||||
);
|
||||
const placed = placeAnchor(anchor, { x: 0.1, y: 0.1, w: 0.4, h: 0.4 })!;
|
||||
expect(placed.x).toBeCloseTo(0.3);
|
||||
expect(placed.y).toBeCloseTo(0.5);
|
||||
expect(placed.w).toBeCloseTo(0.2);
|
||||
expect(placeAnchor(anchor, { x: 0.95, y: 0.95, w: 0.4, h: 0.4 })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasFfmpeg)("marks", () => {
|
||||
it("segments, propagates, scores, and stores the mask under the media root", async () => {
|
||||
harness = await createPrivacyHarness();
|
||||
const asset = await harness.addAsset("clip.mp4");
|
||||
const mark = await createMark(harness.deps, ACTOR, {
|
||||
kind: "tattoo",
|
||||
label: null,
|
||||
personId: null,
|
||||
assetId: asset.id,
|
||||
prompt: tattoo,
|
||||
treatment: "blur",
|
||||
});
|
||||
expect(mark).toMatchObject({ treatment: "blur", qualityScore: 0.9, suggestedFallback: null });
|
||||
const stored = harness.store.marks[0]!;
|
||||
expect(path.isAbsolute(stored.maskDir!)).toBe(false);
|
||||
expect((await stat(path.join(harness.root, stored.maskDir!, "0.png"))).isFile()).toBe(true);
|
||||
});
|
||||
|
||||
it("suggests a fallback when inpainting scores low and rescoring follows the treatment", async () => {
|
||||
harness = await createPrivacyHarness();
|
||||
const asset = await harness.addAsset("clip__inpaintq-0.3__.mp4");
|
||||
const mark = await createMark(harness.deps, ACTOR, {
|
||||
kind: "tattoo",
|
||||
label: null,
|
||||
personId: null,
|
||||
assetId: asset.id,
|
||||
prompt: tattoo,
|
||||
treatment: "inpaint",
|
||||
});
|
||||
expect(mark.qualityScore).toBeLessThanOrEqual(0.3);
|
||||
expect(mark.suggestedFallback).toBe("cover");
|
||||
const blurred = await updateMark(harness.deps, ACTOR, { markId: mark.id, treatment: "blur" });
|
||||
expect(blurred).toMatchObject({
|
||||
treatment: "blur",
|
||||
qualityScore: 0.9,
|
||||
suggestedFallback: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a mark without a worker and rendering it fails closed", async () => {
|
||||
harness = await createPrivacyHarness(() => null);
|
||||
const asset = await harness.addAsset("clip.mp4");
|
||||
const mark = await createMark(harness.deps, ACTOR, {
|
||||
kind: "license_plate",
|
||||
label: null,
|
||||
personId: null,
|
||||
assetId: asset.id,
|
||||
prompt: tattoo,
|
||||
treatment: "cover",
|
||||
});
|
||||
expect(mark.qualityScore).toBeNull();
|
||||
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
|
||||
await expect(render(harness, edl, "no-worker")).rejects.toBeInstanceOf(PrivacyUnavailableError);
|
||||
});
|
||||
|
||||
it("treats a marked region through the video and the gate checks what is left", async () => {
|
||||
harness = await createPrivacyHarness();
|
||||
const asset = await harness.addAsset("clip.mp4");
|
||||
const mark = await createMark(harness.deps, ACTOR, {
|
||||
kind: "tattoo",
|
||||
label: null,
|
||||
personId: null,
|
||||
assetId: asset.id,
|
||||
prompt: tattoo,
|
||||
treatment: "none",
|
||||
});
|
||||
// The plan asks for a cover even though the mark itself says none.
|
||||
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], {
|
||||
redactions: { faces: [], marks: [{ markId: mark.id, treatment: "cover" }], regions: [] },
|
||||
});
|
||||
const { workDir, prepared } = await render(harness, edl, "cover");
|
||||
const record = await readPrivacyRecord(workDir);
|
||||
expect(record?.assets[asset.id]?.checks).toEqual([
|
||||
expect.objectContaining({ kind: "mark", markId: mark.id }),
|
||||
]);
|
||||
const report = await gate(harness, edl, workDir, prepared.sources.get(asset.id)!.path);
|
||||
expect(report.passed).toBe(true);
|
||||
});
|
||||
|
||||
it("blocks when the treatment left the mark visible", async () => {
|
||||
harness = await createPrivacyHarness((emulator) => {
|
||||
const provider = Object.create(emulator) as MediaInferenceProvider;
|
||||
provider.redact = async (request) => {
|
||||
await copyFile(request.path, request.outPath);
|
||||
return { outPath: request.outPath, framesProcessed: 50 };
|
||||
};
|
||||
return provider;
|
||||
});
|
||||
const asset = await harness.addAsset("clip.mp4");
|
||||
const mark = await createMark(harness.deps, ACTOR, {
|
||||
kind: "background_detail",
|
||||
label: null,
|
||||
personId: null,
|
||||
assetId: asset.id,
|
||||
prompt: tattoo,
|
||||
treatment: "blur",
|
||||
});
|
||||
const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]);
|
||||
const { workDir, prepared } = await render(harness, edl, "leak");
|
||||
const report = await gate(harness, edl, workDir, prepared.sources.get(asset.id)!.path);
|
||||
expect(report.passed).toBe(false);
|
||||
expect(report.issues).toContainEqual(
|
||||
expect.objectContaining({ code: "mark_visible", markId: mark.id }),
|
||||
);
|
||||
});
|
||||
|
||||
it("inpaints when quality holds and covers instead when it does not", async () => {
|
||||
harness = await createPrivacyHarness();
|
||||
const good = await harness.addAsset("good.mp4");
|
||||
const poor = await harness.addAsset("poor__inpaintq-0.3__.mp4");
|
||||
for (const asset of [good, poor]) {
|
||||
await createMark(harness.deps, ACTOR, {
|
||||
kind: "scar",
|
||||
label: null,
|
||||
personId: null,
|
||||
assetId: asset.id,
|
||||
prompt: tattoo,
|
||||
treatment: "inpaint",
|
||||
});
|
||||
}
|
||||
const edl = testEdl([
|
||||
{ id: "c1", assetId: good.id, inMs: 0, outMs: 1000 },
|
||||
{ id: "c2", assetId: poor.id, inMs: 0, outMs: 1000 },
|
||||
]);
|
||||
const { prepared } = await render(harness, edl, "inpaint");
|
||||
expect(prepared.aiAlterations).toEqual(["inpaint"]);
|
||||
expect(prepared.fallbacks).toEqual([
|
||||
expect.stringMatching(/inpainting scored 0\.30; covered instead$/),
|
||||
]);
|
||||
});
|
||||
|
||||
it("re-finds a person's standing mark in another video relative to the face", async () => {
|
||||
harness = await createPrivacyHarness();
|
||||
const origin = await harness.addAsset("a__faces-1____person-alice__.mp4");
|
||||
const other = await harness.addAsset("b__faces-1____person-alice__.mp4");
|
||||
harness.addPerson("alice", "none");
|
||||
await harness.analyseFaces(origin.id, ["alice"]);
|
||||
await harness.analyseFaces(other.id, ["alice"]);
|
||||
const segment = vi.spyOn(harness.inference, "segmentPrompted");
|
||||
const mark = await createMark(harness.deps, ACTOR, {
|
||||
kind: "tattoo",
|
||||
label: "left forearm",
|
||||
personId: "alice",
|
||||
assetId: origin.id,
|
||||
prompt: tattoo,
|
||||
treatment: "blur",
|
||||
});
|
||||
const edl = testEdl([{ id: "c1", assetId: other.id, inMs: 0, outMs: 2000 }]);
|
||||
const { workDir, prepared } = await render(harness, edl, "standing");
|
||||
expect(segment).toHaveBeenCalledTimes(2);
|
||||
const reprompt = segment.mock.calls[1]![0];
|
||||
expect(reprompt.path).toContain("b__faces-1");
|
||||
expect(reprompt.prompt.box).toBeDefined();
|
||||
const record = await readPrivacyRecord(workDir);
|
||||
expect(record?.reviews).toEqual([]);
|
||||
expect(record?.assets[other.id]?.checks).toEqual([
|
||||
expect.objectContaining({ kind: "mark", markId: mark.id }),
|
||||
]);
|
||||
const report = await gate(harness, edl, workDir, prepared.sources.get(other.id)!.path);
|
||||
expect(report.passed).toBe(true);
|
||||
});
|
||||
|
||||
it("sends a low-confidence re-find to review, which blocks the export", async () => {
|
||||
harness = await createPrivacyHarness((emulator) => {
|
||||
const provider = Object.create(emulator) as MediaInferenceProvider;
|
||||
provider.segmentPrompted = async (request) => {
|
||||
const result = await emulator.segmentPrompted(request);
|
||||
return request.path.includes("b__") ? { ...result, score: 0.3 } : result;
|
||||
};
|
||||
return provider;
|
||||
});
|
||||
const origin = await harness.addAsset("a__faces-1____person-alice__.mp4");
|
||||
const other = await harness.addAsset("b__faces-1____person-alice__.mp4");
|
||||
harness.addPerson("alice", "none");
|
||||
await harness.analyseFaces(origin.id, ["alice"]);
|
||||
await harness.analyseFaces(other.id, ["alice"]);
|
||||
const mark = await createMark(harness.deps, ACTOR, {
|
||||
kind: "jewelry",
|
||||
label: null,
|
||||
personId: "alice",
|
||||
assetId: origin.id,
|
||||
prompt: tattoo,
|
||||
treatment: "blur",
|
||||
});
|
||||
const edl = testEdl([{ id: "c1", assetId: other.id, inMs: 0, outMs: 2000 }]);
|
||||
const { workDir, prepared } = await render(harness, edl, "review");
|
||||
const report = await gate(harness, edl, workDir, prepared.sources.get(other.id)!.path);
|
||||
expect(report.passed).toBe(false);
|
||||
expect(report.issues).toContainEqual(
|
||||
expect.objectContaining({ code: "mark_needs_review", markId: mark.id }),
|
||||
);
|
||||
});
|
||||
|
||||
it("removes a person with their tracks, marks, and files", async () => {
|
||||
harness = await createPrivacyHarness();
|
||||
const asset = await harness.addAsset("clip__faces-1____person-alice__.mp4");
|
||||
harness.addPerson("alice", "blur");
|
||||
await harness.analyseFaces(asset.id, ["alice"]);
|
||||
await createMark(harness.deps, ACTOR, {
|
||||
kind: "tattoo",
|
||||
label: null,
|
||||
personId: "alice",
|
||||
assetId: asset.id,
|
||||
prompt: tattoo,
|
||||
treatment: "blur",
|
||||
});
|
||||
const files = [
|
||||
path.join(harness.root, harness.store.tracks[0]!.keyframesPath),
|
||||
path.join(harness.root, harness.store.marks[0]!.maskDir!),
|
||||
];
|
||||
const service = createMediaPrivacyService(harness.deps);
|
||||
await service.removePerson(ACTOR, "alice");
|
||||
expect(harness.store.people).toEqual([]);
|
||||
expect(harness.store.tracks).toEqual([]);
|
||||
for (const file of files) await expect(stat(file)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
322
packages/adapters/src/media-marks.ts
Normal file
322
packages/adapters/src/media-marks.ts
Normal file
@ -0,0 +1,322 @@
|
||||
import { mkdir, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { SegmentPromptedResult } from "@rakazo/adapter-kit";
|
||||
import type {
|
||||
BoxKeyframe,
|
||||
MarkTreatment,
|
||||
MediaMark,
|
||||
MediaMarkPrompt,
|
||||
NormalizedBox,
|
||||
} from "@rakazo/contracts";
|
||||
import type { CreateMarkInputType, MediaActor, MediaServiceDeps } from "./media-context.js";
|
||||
import type { PrivacyToolOptions } from "./media-privacy-io.js";
|
||||
import {
|
||||
analysisFile,
|
||||
inferenceCapabilities,
|
||||
operationAvailable,
|
||||
PrivacyUnavailableError,
|
||||
probeSource,
|
||||
resolveStored,
|
||||
toStored,
|
||||
} from "./media-privacy-io.js";
|
||||
import { extractLuma, temporalStability } from "./media-privacy-pixels.js";
|
||||
import type { TrackKeyframe } from "./media-privacy-plan.js";
|
||||
import { boxAt, FACE_MIN_CONFIDENCE, MARK_QUALITY_MIN, unionBox } from "./media-privacy-plan.js";
|
||||
|
||||
/*
|
||||
* Identifying marks (spec 7): the user marks once by click or box, the
|
||||
* segmenter (SAM2 in the sidecar) propagates the mask through the video, and
|
||||
* the treatment is scored. A score under MARK_QUALITY_MIN never ships
|
||||
* silently: the mark carries a suggested fallback and rendering applies it.
|
||||
*/
|
||||
|
||||
/** Side of the box a point prompt stands for, normalized. */
|
||||
const POINT_PROMPT_SIZE = 0.12;
|
||||
/** Frames sampled for the local flicker measurement of an inpaint preview. */
|
||||
const STABILITY_FRAMES = 6;
|
||||
const STABILITY_STEP_MS = 200;
|
||||
|
||||
export type MarkDeps = Pick<MediaServiceDeps, "repos" | "inference" | "mediaRoot" | "sources"> &
|
||||
PrivacyToolOptions;
|
||||
|
||||
/** Directory of a mark's masks and previews, relative to the media root. */
|
||||
export function markDir(spaceId: string, markId: string): string {
|
||||
return path.join(spaceId, "marks", markId);
|
||||
}
|
||||
|
||||
/** The prompt as a box: a point prompt stands for a small box around it. */
|
||||
export function promptBox(prompt: MediaMarkPrompt): NormalizedBox {
|
||||
if (prompt.box) return prompt.box;
|
||||
if (!prompt.point) throw new PrivacyUnavailableError("a mark needs a box or a point");
|
||||
const x = Math.min(Math.max(0, prompt.point.x - POINT_PROMPT_SIZE / 2), 1 - POINT_PROMPT_SIZE);
|
||||
const y = Math.min(Math.max(0, prompt.point.y - POINT_PROMPT_SIZE / 2), 1 - POINT_PROMPT_SIZE);
|
||||
return { x, y, w: POINT_PROMPT_SIZE, h: POINT_PROMPT_SIZE };
|
||||
}
|
||||
|
||||
/** Fallback to propose when a treatment scores too low. */
|
||||
export function fallbackFor(treatment: MarkTreatment): MarkTreatment {
|
||||
return treatment === "cover" ? "reframe_out" : "cover";
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a treatment. Segmentation confidence bounds every treatment (a mask
|
||||
* that does not follow the mark cannot hide it); inpainting also needs the
|
||||
* provider's temporal and edge quality and a local flicker check of its
|
||||
* preview. Returns the score and, under MARK_QUALITY_MIN, the fallback.
|
||||
*/
|
||||
export async function scoreTreatment(
|
||||
deps: MarkDeps,
|
||||
input: {
|
||||
treatment: MarkTreatment;
|
||||
segmentationScore: number;
|
||||
file: string;
|
||||
maskDir: string;
|
||||
keyframes: BoxKeyframe[];
|
||||
previewDir: string;
|
||||
signal?: AbortSignal;
|
||||
},
|
||||
): Promise<{ qualityScore: number; suggestedFallback: MarkTreatment | null }> {
|
||||
let score = input.segmentationScore;
|
||||
if (input.treatment === "inpaint") {
|
||||
const capabilities = await inferenceCapabilities(deps.inference, input.signal);
|
||||
if (!deps.inference || !operationAvailable(capabilities, "inpaintVideo")) {
|
||||
return { qualityScore: 0, suggestedFallback: "cover" };
|
||||
}
|
||||
await mkdir(input.previewDir, { recursive: true });
|
||||
const preview = await deps.inference.inpaintVideo({
|
||||
path: input.file,
|
||||
maskDir: input.maskDir,
|
||||
outPath: path.join(input.previewDir, "inpaint-preview.mp4"),
|
||||
signal: input.signal,
|
||||
});
|
||||
const local = await previewStability(deps, preview.outPath, input.keyframes, input.signal);
|
||||
score = Math.min(score, preview.quality.score, preview.quality.temporalStability, local);
|
||||
}
|
||||
const qualityScore = Math.round(score * 1000) / 1000;
|
||||
return {
|
||||
qualityScore,
|
||||
suggestedFallback:
|
||||
qualityScore < MARK_QUALITY_MIN && input.treatment !== "none"
|
||||
? fallbackFor(input.treatment)
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Flicker inside the mark region of an inpainted preview around the first keyframe. */
|
||||
async function previewStability(
|
||||
deps: MarkDeps,
|
||||
file: string,
|
||||
keyframes: BoxKeyframe[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<number> {
|
||||
const first = keyframes[0];
|
||||
if (!first) return 1;
|
||||
const { width, height } = await frameSize(file, deps, signal);
|
||||
const frames = [];
|
||||
let region: NormalizedBox = first.box;
|
||||
for (let i = 0; i < STABILITY_FRAMES; i++) {
|
||||
const tMs = first.tMs + i * STABILITY_STEP_MS;
|
||||
const box = boxAt(keyframes, tMs, STABILITY_STEP_MS);
|
||||
if (box) region = unionBox(region, box);
|
||||
try {
|
||||
frames.push(
|
||||
await extractLuma(file, { tMs, width, height }, { ffmpegPath: deps.ffmpegPath, signal }),
|
||||
);
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return temporalStability(frames, region);
|
||||
}
|
||||
|
||||
async function frameSize(file: string, deps: PrivacyToolOptions, signal?: AbortSignal) {
|
||||
const probed = await probeSource(file, "video", { ffprobePath: deps.ffprobePath, signal });
|
||||
return { width: probed.width, height: probed.height };
|
||||
}
|
||||
|
||||
/** Segment a prompt and propagate it through the file into `outDir`. */
|
||||
export async function segmentMark(
|
||||
deps: MarkDeps,
|
||||
input: { file: string; prompt: MediaMarkPrompt; outDir: string; signal?: AbortSignal },
|
||||
): Promise<SegmentPromptedResult> {
|
||||
const capabilities = await inferenceCapabilities(deps.inference, input.signal);
|
||||
if (!deps.inference || !operationAvailable(capabilities, "segmentPrompted")) {
|
||||
throw new PrivacyUnavailableError("mark segmentation needs the media worker");
|
||||
}
|
||||
await rm(input.outDir, { recursive: true, force: true });
|
||||
await mkdir(input.outDir, { recursive: true });
|
||||
return deps.inference.segmentPrompted({
|
||||
path: input.file,
|
||||
prompt: input.prompt,
|
||||
propagate: true,
|
||||
outDir: input.outDir,
|
||||
signal: input.signal,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a person-bound mark sits relative to the person's face, from the
|
||||
* frame it was drawn in: offsets and size in face-box units.
|
||||
*/
|
||||
export interface MarkAnchor {
|
||||
dx: number;
|
||||
dy: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export function anchorFromFace(mark: NormalizedBox, face: NormalizedBox): MarkAnchor {
|
||||
return {
|
||||
dx: (mark.x - face.x) / face.w,
|
||||
dy: (mark.y - face.y) / face.h,
|
||||
w: mark.w / face.w,
|
||||
h: mark.h / face.h,
|
||||
};
|
||||
}
|
||||
|
||||
/** Place an anchor at a face, clamped to the frame. Null when it falls outside. */
|
||||
export function placeAnchor(anchor: MarkAnchor, face: NormalizedBox): NormalizedBox | null {
|
||||
const x0 = face.x + anchor.dx * face.w;
|
||||
const y0 = face.y + anchor.dy * face.h;
|
||||
const x1 = x0 + anchor.w * face.w;
|
||||
const y1 = y0 + anchor.h * face.h;
|
||||
const x = Math.max(0, x0);
|
||||
const y = Math.max(0, y0);
|
||||
const w = Math.min(1, x1) - x;
|
||||
const h = Math.min(1, y1) - y;
|
||||
return w > 0.001 && h > 0.001 ? { x, y, w, h } : null;
|
||||
}
|
||||
|
||||
/** First confident face keyframe of a track at or after `fromMs`. */
|
||||
export function firstConfident(keyframes: TrackKeyframe[], fromMs = 0): TrackKeyframe | null {
|
||||
return (
|
||||
[...keyframes]
|
||||
.filter((k) => k.tMs >= fromMs && (k.confidence ?? 1) >= FACE_MIN_CONFIDENCE)
|
||||
.sort((a, b) => a.tMs - b.tMs)[0] ?? null
|
||||
);
|
||||
}
|
||||
|
||||
/** Face box of a track nearest to `tMs` (within 250 ms). */
|
||||
export function faceAt(keyframes: TrackKeyframe[], tMs: number): NormalizedBox | null {
|
||||
let best: TrackKeyframe | null = null;
|
||||
for (const k of keyframes) {
|
||||
if (Math.abs(k.tMs - tMs) > 250) continue;
|
||||
if (!best || Math.abs(k.tMs - tMs) < Math.abs(best.tMs - tMs)) best = k;
|
||||
}
|
||||
return best?.box ?? null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Service operations
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Create a mark: store it, segment from the prompt, propagate through the
|
||||
* asset, score the treatment, and store a fallback suggestion when the score
|
||||
* is low. Without the media worker the mark is kept unsegmented; rendering a
|
||||
* clip of that asset then fails closed until segmentation is possible.
|
||||
*/
|
||||
export async function createMark(
|
||||
deps: MarkDeps,
|
||||
actor: MediaActor,
|
||||
input: CreateMarkInputType,
|
||||
signal?: AbortSignal,
|
||||
): Promise<MediaMark> {
|
||||
const asset = await deps.repos.getAssetRecord(actor.spaceId, input.assetId);
|
||||
const mark = await deps.repos.createMark(actor.spaceId, {
|
||||
kind: input.kind,
|
||||
label: input.label,
|
||||
personId: input.personId,
|
||||
assetId: input.assetId,
|
||||
prompt: input.prompt,
|
||||
treatment: input.treatment,
|
||||
});
|
||||
const file = await analysisFile(deps, actor.spaceId, asset);
|
||||
const capabilities = await inferenceCapabilities(deps.inference, signal);
|
||||
if (!file || !operationAvailable(capabilities, "segmentPrompted")) return mark;
|
||||
const relative = markDir(actor.spaceId, mark.id);
|
||||
const outDir = await resolveStored(deps.mediaRoot, path.join(relative, "masks"));
|
||||
try {
|
||||
const segmented = await segmentMark(deps, { file, prompt: input.prompt, outDir, signal });
|
||||
const scored = await scoreTreatment(deps, {
|
||||
treatment: input.treatment,
|
||||
segmentationScore: segmented.score,
|
||||
file,
|
||||
maskDir: segmented.maskDir,
|
||||
keyframes: segmented.keyframes,
|
||||
previewDir: await resolveStored(deps.mediaRoot, relative),
|
||||
signal,
|
||||
});
|
||||
const record = await deps.repos.recordMarkResult(actor.spaceId, mark.id, {
|
||||
...scored,
|
||||
maskDir: await toStored(deps.mediaRoot, segmented.maskDir),
|
||||
});
|
||||
const { maskDir: _maskDir, ...dto } = record;
|
||||
return dto;
|
||||
} catch (error) {
|
||||
// The mark stays without a mask (rendering fails closed); the caller sees why.
|
||||
await deps.repos.removeMark(actor.spaceId, mark.id);
|
||||
await rm(await resolveStored(deps.mediaRoot, relative), { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update treatment, label, or person. A new treatment is segmented and scored
|
||||
* again so its suggestion reflects that treatment.
|
||||
*/
|
||||
export async function updateMark(
|
||||
deps: MarkDeps,
|
||||
actor: MediaActor,
|
||||
input: {
|
||||
markId: string;
|
||||
treatment?: MarkTreatment;
|
||||
label?: string | null;
|
||||
personId?: string | null;
|
||||
},
|
||||
signal?: AbortSignal,
|
||||
): Promise<MediaMark> {
|
||||
const before = await deps.repos.getMark(actor.spaceId, input.markId);
|
||||
const updated = await deps.repos.updateMark(actor.spaceId, input.markId, {
|
||||
...(input.treatment !== undefined ? { treatment: input.treatment } : {}),
|
||||
...(input.label !== undefined ? { label: input.label } : {}),
|
||||
...(input.personId !== undefined ? { personId: input.personId } : {}),
|
||||
});
|
||||
if (input.treatment === undefined || input.treatment === before.treatment) return updated;
|
||||
if (!before.assetId || !before.prompt) return updated;
|
||||
const asset = await deps.repos.getAssetRecord(actor.spaceId, before.assetId);
|
||||
const file = await analysisFile(deps, actor.spaceId, asset);
|
||||
const capabilities = await inferenceCapabilities(deps.inference, signal);
|
||||
if (!file || !operationAvailable(capabilities, "segmentPrompted")) {
|
||||
// Without the worker the old score says nothing about the new treatment.
|
||||
const record = await deps.repos.recordMarkResult(actor.spaceId, input.markId, {
|
||||
qualityScore: null,
|
||||
suggestedFallback: null,
|
||||
maskDir: before.maskDir,
|
||||
});
|
||||
const { maskDir: _maskDir, ...dto } = record;
|
||||
return dto;
|
||||
}
|
||||
const relative = markDir(actor.spaceId, input.markId);
|
||||
const segmented = await segmentMark(deps, {
|
||||
file,
|
||||
prompt: before.prompt,
|
||||
outDir: await resolveStored(deps.mediaRoot, path.join(relative, "masks")),
|
||||
signal,
|
||||
});
|
||||
const scored = await scoreTreatment(deps, {
|
||||
treatment: input.treatment,
|
||||
segmentationScore: segmented.score,
|
||||
file,
|
||||
maskDir: segmented.maskDir,
|
||||
keyframes: segmented.keyframes,
|
||||
previewDir: await resolveStored(deps.mediaRoot, relative),
|
||||
signal,
|
||||
});
|
||||
const record = await deps.repos.recordMarkResult(actor.spaceId, input.markId, {
|
||||
...scored,
|
||||
maskDir: await toStored(deps.mediaRoot, segmented.maskDir),
|
||||
});
|
||||
const { maskDir: _maskDir, ...dto } = record;
|
||||
return dto;
|
||||
}
|
||||
437
packages/adapters/src/media-privacy-fixtures.ts
Normal file
437
packages/adapters/src/media-privacy-fixtures.ts
Normal file
@ -0,0 +1,437 @@
|
||||
import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
|
||||
import type {
|
||||
AnonymizeMode,
|
||||
Edl,
|
||||
FfprobeReport,
|
||||
MarkTreatment,
|
||||
MediaMarkKind,
|
||||
MediaMarkPrompt,
|
||||
PlatformProfile,
|
||||
} from "@rakazo/contracts";
|
||||
import { EdlSchema, PlatformProfileSchema } from "@rakazo/contracts";
|
||||
import type { MediaRepos } from "@rakazo/db";
|
||||
import { edlDurationMs, encodeVectors } from "@rakazo/media";
|
||||
import type { MediaServiceDeps } from "./media-context.js";
|
||||
import { EmulatorMediaInferenceProvider } from "./media-emulator.js";
|
||||
import { renderTestClip } from "./media-emulator-clips.js";
|
||||
import { runMediaTool } from "./media-folder-source-probe.js";
|
||||
|
||||
/*
|
||||
* In-memory repositories and synthetic ffmpeg clips for the privacy tests.
|
||||
* Clip names carry media-emulator markers (faces, person seeds, detection
|
||||
* gaps, classifier scores); nothing here shows a real person.
|
||||
*/
|
||||
|
||||
export const SPACE = "space-1";
|
||||
export const USER = "user-1";
|
||||
export const ACTOR = { spaceId: SPACE, userId: USER };
|
||||
|
||||
interface AssetRow {
|
||||
id: string;
|
||||
spaceId: string;
|
||||
sourceId: string;
|
||||
externalId: string;
|
||||
kind: string;
|
||||
storage: string;
|
||||
storagePath: string | null;
|
||||
proxyPath: string | null;
|
||||
}
|
||||
|
||||
interface PersonRow {
|
||||
id: string;
|
||||
spaceId: string;
|
||||
displayName: string;
|
||||
anonymizeByDefault: string;
|
||||
consentRecordedAt: Date | null;
|
||||
ageVerifiedAt: Date | null;
|
||||
ageVerificationMethod: string;
|
||||
syntheticSeed: number | null;
|
||||
centroid: Uint8Array | null;
|
||||
}
|
||||
|
||||
interface TrackRow {
|
||||
id: string;
|
||||
assetId: string;
|
||||
personId: string | null;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
confidenceMin: number;
|
||||
keyframesPath: string;
|
||||
modelVersion: string;
|
||||
}
|
||||
|
||||
interface MarkRow {
|
||||
id: string;
|
||||
kind: MediaMarkKind;
|
||||
label: string | null;
|
||||
personId: string | null;
|
||||
assetId: string | null;
|
||||
prompt: MediaMarkPrompt | null;
|
||||
treatment: MarkTreatment;
|
||||
qualityScore: number | null;
|
||||
suggestedFallback: MarkTreatment | null;
|
||||
maskDir: string | null;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface PrivacyHarness {
|
||||
root: string;
|
||||
inference: EmulatorMediaInferenceProvider;
|
||||
deps: MediaServiceDeps;
|
||||
store: {
|
||||
assets: AssetRow[];
|
||||
people: PersonRow[];
|
||||
tracks: TrackRow[];
|
||||
marks: MarkRow[];
|
||||
renders: Array<Record<string, unknown>>;
|
||||
plans: Array<Record<string, unknown>>;
|
||||
segments: Array<Record<string, unknown>>;
|
||||
};
|
||||
/** A synthetic clip, or a still (PNG) when the name ends in .png. */
|
||||
addAsset(fileName: string, options?: { seconds?: number; id?: string }): Promise<AssetRow>;
|
||||
addPerson(
|
||||
id: string,
|
||||
mode: AnonymizeMode,
|
||||
consent?: { consent?: boolean; age?: boolean },
|
||||
): PersonRow;
|
||||
/** Run emulator tracking on the asset and store the tracks (the analysis result). */
|
||||
analyseFaces(assetId: string, personIds: Array<string | null>, sampleFps?: number): Promise<void>;
|
||||
/** Store the emulator identity embedding of a person seed as the person's centroid. */
|
||||
learnPerson(personId: string, assetId: string, trackIndex?: number): Promise<void>;
|
||||
workDir(name: string): Promise<string>;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
export function testEdl(
|
||||
clips: Array<{ id: string; assetId: string; inMs: number; outMs: number; atMs?: number }>,
|
||||
extra: Partial<Pick<Edl, "redactions">> & { width?: number; height?: number } = {},
|
||||
): Edl {
|
||||
let cursor = 0;
|
||||
const video = clips.map((clip) => {
|
||||
const atMs = clip.atMs ?? cursor;
|
||||
cursor = atMs + clip.outMs - clip.inMs;
|
||||
return { ...clip, atMs, reframe: "none" as const };
|
||||
});
|
||||
return EdlSchema.parse({
|
||||
edlVersion: 1,
|
||||
output: {
|
||||
profileId: "test-profile",
|
||||
width: extra.width ?? 320,
|
||||
height: extra.height ?? 180,
|
||||
fps: 25,
|
||||
durationTargetMs: cursor,
|
||||
durationHardMinMs: 0,
|
||||
durationHardMaxMs: 600_000,
|
||||
},
|
||||
tracks: { video },
|
||||
redactions: extra.redactions ?? {},
|
||||
metadata: {},
|
||||
});
|
||||
}
|
||||
|
||||
export function testProfile(overrides: Record<string, unknown> = {}): PlatformProfile {
|
||||
return PlatformProfileSchema.parse({
|
||||
id: "test-profile",
|
||||
name: "Test profile",
|
||||
kind: "free_social",
|
||||
reviewedAt: "2026-01-01",
|
||||
video: {
|
||||
aspectRatios: ["16:9", "9:16"],
|
||||
width: 1920,
|
||||
height: 1920,
|
||||
fps: 60,
|
||||
codec: "h264",
|
||||
maxBitrateKbps: 20_000,
|
||||
maxFileSizeMb: 500,
|
||||
},
|
||||
duration: { hardMinMs: 0, hardMaxMs: 600_000 },
|
||||
safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 },
|
||||
captions: { recommended: false },
|
||||
audio: {},
|
||||
text: { captionMaxChars: 2200, hashtagLimit: 30, linkPolicy: "allowed" },
|
||||
content: { thresholds: { explicit_nudity: 0.5 } },
|
||||
disclosure: { aiAlteredLabelRequired: false },
|
||||
consent: { requirePerformerConsent: false, requireAgeVerification: false },
|
||||
export: {},
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
/** An ffprobe report that passes the technical check for an EDL. */
|
||||
export function cleanProbe(edl: Edl): FfprobeReport {
|
||||
return {
|
||||
durationMs: edlDurationMs(edl),
|
||||
width: edl.output.width,
|
||||
height: edl.output.height,
|
||||
fps: edl.output.fps,
|
||||
videoCodec: "h264",
|
||||
audioCodec: null,
|
||||
sizeBytes: 10_000,
|
||||
integratedLufs: null,
|
||||
truePeakDb: null,
|
||||
tags: {},
|
||||
};
|
||||
}
|
||||
|
||||
function isolation(): Error {
|
||||
return new Error("not found in space");
|
||||
}
|
||||
|
||||
export async function createPrivacyHarness(
|
||||
inferenceOverride?: (emulator: EmulatorMediaInferenceProvider) => MediaInferenceProvider | null,
|
||||
): Promise<PrivacyHarness> {
|
||||
const root = await realpath(await mkdtemp(path.join(tmpdir(), "media-privacy-")));
|
||||
const emulator = new EmulatorMediaInferenceProvider({ root });
|
||||
const store: PrivacyHarness["store"] = {
|
||||
assets: [],
|
||||
people: [],
|
||||
tracks: [],
|
||||
marks: [],
|
||||
renders: [],
|
||||
plans: [],
|
||||
segments: [],
|
||||
};
|
||||
let counter = 0;
|
||||
const nextId = (prefix: string) => `${prefix}${++counter}`;
|
||||
const markDto = (row: MarkRow) => {
|
||||
const { maskDir: _maskDir, ...dto } = row;
|
||||
return dto;
|
||||
};
|
||||
const findMark = (id: string) => {
|
||||
const row = store.marks.find((mark) => mark.id === id);
|
||||
if (!row) throw isolation();
|
||||
return row;
|
||||
};
|
||||
const repos = {
|
||||
async listAssetRecords(_spaceId: string, ids: string[]) {
|
||||
return store.assets.filter((asset) => ids.includes(asset.id));
|
||||
},
|
||||
async getAssetRecord(_spaceId: string, id: string) {
|
||||
const asset = store.assets.find((row) => row.id === id);
|
||||
if (!asset) throw isolation();
|
||||
return asset;
|
||||
},
|
||||
async getSource() {
|
||||
return { kind: "folder", config: {} };
|
||||
},
|
||||
async getProjectRecord(_spaceId: string, projectId: string) {
|
||||
return {
|
||||
id: projectId,
|
||||
brandKitId: null,
|
||||
assetIds: store.assets.map((a) => a.id),
|
||||
};
|
||||
},
|
||||
async getBrandKit() {
|
||||
throw isolation();
|
||||
},
|
||||
async listPersonRecords(_spaceId: string, ids?: string[]) {
|
||||
return store.people.filter((person) => !ids || ids.includes(person.id));
|
||||
},
|
||||
async personCentroids() {
|
||||
return store.people.flatMap((person) =>
|
||||
person.centroid ? [{ personId: person.id, centroid: person.centroid }] : [],
|
||||
);
|
||||
},
|
||||
async updatePerson(_spaceId: string, personId: string, patch: { syntheticSeed?: number }) {
|
||||
const person = store.people.find((row) => row.id === personId);
|
||||
if (!person) throw isolation();
|
||||
if (patch.syntheticSeed !== undefined) person.syntheticSeed = patch.syntheticSeed;
|
||||
return person;
|
||||
},
|
||||
async faceTrackRecords(_spaceId: string, assetIds: string[]) {
|
||||
return store.tracks.filter((track) => assetIds.includes(track.assetId));
|
||||
},
|
||||
async listFaceTracks() {
|
||||
return store.tracks;
|
||||
},
|
||||
async latestAnalysis(_spaceId: string, assetId: string, kind: string) {
|
||||
return kind === "faces" && store.tracks.some((t) => t.assetId === assetId) ? { kind } : null;
|
||||
},
|
||||
async marksForAsset(_spaceId: string, assetId: string) {
|
||||
const peopleHere = new Set(
|
||||
store.tracks.filter((t) => t.assetId === assetId && t.personId).map((t) => t.personId),
|
||||
);
|
||||
return store.marks.filter(
|
||||
(mark) => mark.assetId === assetId || (mark.personId && peopleHere.has(mark.personId)),
|
||||
);
|
||||
},
|
||||
async createMark(
|
||||
_spaceId: string,
|
||||
input: Omit<MarkRow, "id" | "qualityScore" | "suggestedFallback" | "maskDir" | "createdAt">,
|
||||
) {
|
||||
const row: MarkRow = {
|
||||
...input,
|
||||
id: nextId("mark"),
|
||||
qualityScore: null,
|
||||
suggestedFallback: null,
|
||||
maskDir: null,
|
||||
createdAt: new Date(0).toISOString(),
|
||||
};
|
||||
store.marks.push(row);
|
||||
return markDto(row);
|
||||
},
|
||||
async getMark(_spaceId: string, id: string) {
|
||||
return { ...findMark(id) };
|
||||
},
|
||||
async updateMark(_spaceId: string, id: string, patch: Partial<MarkRow>) {
|
||||
Object.assign(findMark(id), patch);
|
||||
return markDto(findMark(id));
|
||||
},
|
||||
async recordMarkResult(
|
||||
_spaceId: string,
|
||||
id: string,
|
||||
outcome: Pick<MarkRow, "qualityScore" | "suggestedFallback" | "maskDir">,
|
||||
) {
|
||||
Object.assign(findMark(id), outcome);
|
||||
return { ...findMark(id) };
|
||||
},
|
||||
async removeMark(_spaceId: string, id: string) {
|
||||
const row = findMark(id);
|
||||
store.marks = store.marks.filter((mark) => mark.id !== id);
|
||||
return { maskDir: row.maskDir };
|
||||
},
|
||||
async removePersonCompletely(_spaceId: string, personId: string) {
|
||||
const tracks = store.tracks.filter((t) => t.personId === personId);
|
||||
const marks = store.marks.filter((m) => m.personId === personId);
|
||||
store.tracks = store.tracks.filter((t) => t.personId !== personId);
|
||||
store.marks = store.marks.filter((m) => m.personId !== personId);
|
||||
store.people = store.people.filter((p) => p.id !== personId);
|
||||
return {
|
||||
deletedFaceTracks: tracks.length,
|
||||
deletedMarks: marks.length,
|
||||
keyframesPaths: tracks.map((t) => t.keyframesPath),
|
||||
maskDirs: marks.flatMap((m) => (m.maskDir ? [m.maskDir] : [])),
|
||||
};
|
||||
},
|
||||
async getRenderRecord(_spaceId: string, id: string) {
|
||||
const render = store.renders.find((row) => row.id === id);
|
||||
if (!render) throw isolation();
|
||||
return render;
|
||||
},
|
||||
async getPlan(_spaceId: string, id: string) {
|
||||
const plan = store.plans.find((row) => row.id === id);
|
||||
if (!plan) throw isolation();
|
||||
return plan;
|
||||
},
|
||||
async listSegments() {
|
||||
return store.segments;
|
||||
},
|
||||
async updateRender(_spaceId: string, id: string, patch: Record<string, unknown>) {
|
||||
const render = store.renders.find((row) => row.id === id);
|
||||
if (!render) return null;
|
||||
Object.assign(render, patch);
|
||||
return render;
|
||||
},
|
||||
};
|
||||
|
||||
const libraryDir = path.join(root, SPACE, "library");
|
||||
await mkdir(libraryDir, { recursive: true });
|
||||
const deps = {
|
||||
repos: repos as unknown as MediaRepos,
|
||||
inference: inferenceOverride ? inferenceOverride(emulator) : emulator,
|
||||
mediaRoot: root,
|
||||
sources: {},
|
||||
now: () => new Date("2026-06-01T12:00:00Z"),
|
||||
} as unknown as MediaServiceDeps;
|
||||
|
||||
const harness: PrivacyHarness = {
|
||||
root,
|
||||
inference: emulator,
|
||||
deps,
|
||||
store,
|
||||
async addAsset(fileName, options = {}) {
|
||||
const file = path.join(libraryDir, fileName);
|
||||
const still = fileName.endsWith(".png");
|
||||
if (still) {
|
||||
await runMediaTool("ffmpeg", [
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=s=320x180:r=1:d=1",
|
||||
"-frames:v",
|
||||
"1",
|
||||
file,
|
||||
]);
|
||||
} else {
|
||||
await renderTestClip(file, { seconds: options.seconds ?? 2 });
|
||||
}
|
||||
const row: AssetRow = {
|
||||
id: options.id ?? nextId("asset"),
|
||||
spaceId: SPACE,
|
||||
sourceId: "source-1",
|
||||
externalId: fileName,
|
||||
kind: still ? "photo" : "video",
|
||||
storage: "uploaded",
|
||||
storagePath: path.relative(root, file),
|
||||
proxyPath: null,
|
||||
};
|
||||
store.assets.push(row);
|
||||
return row;
|
||||
},
|
||||
addPerson(id, mode, consent = {}) {
|
||||
const at = new Date("2026-01-01T00:00:00Z");
|
||||
const row: PersonRow = {
|
||||
id,
|
||||
spaceId: SPACE,
|
||||
displayName: id,
|
||||
anonymizeByDefault: mode,
|
||||
consentRecordedAt: consent.consent ? at : null,
|
||||
ageVerifiedAt: consent.age ? at : null,
|
||||
ageVerificationMethod: consent.age ? "document_on_file" : "none",
|
||||
syntheticSeed: null,
|
||||
centroid: null,
|
||||
};
|
||||
store.people.push(row);
|
||||
return row;
|
||||
},
|
||||
async analyseFaces(assetId, personIds, sampleFps = 10) {
|
||||
const asset = store.assets.find((row) => row.id === assetId)!;
|
||||
const tracked = await emulator.trackFaces({
|
||||
path: path.join(root, asset.storagePath!),
|
||||
sampleFps,
|
||||
});
|
||||
const dir = path.join(root, SPACE, "faces", assetId);
|
||||
await mkdir(dir, { recursive: true });
|
||||
for (const [index, track] of tracked.tracks.entries()) {
|
||||
const file = path.join(dir, `${track.trackId}.json`);
|
||||
await writeFile(file, JSON.stringify({ keyframes: track.keyframes }));
|
||||
const times = track.keyframes.map((k) => k.tMs);
|
||||
store.tracks.push({
|
||||
id: nextId("track"),
|
||||
assetId,
|
||||
personId: personIds[index] ?? null,
|
||||
startMs: Math.min(...times),
|
||||
endMs: Math.max(...times),
|
||||
confidenceMin: 0.9,
|
||||
keyframesPath: path.relative(root, file),
|
||||
modelVersion: "emulator",
|
||||
});
|
||||
}
|
||||
},
|
||||
async learnPerson(personId, assetId, trackIndex = 0) {
|
||||
const asset = store.assets.find((row) => row.id === assetId)!;
|
||||
const tracked = await emulator.trackFaces({
|
||||
path: path.join(root, asset.storagePath!),
|
||||
sampleFps: 2,
|
||||
});
|
||||
const embedding = tracked.tracks[trackIndex]?.embedding;
|
||||
const person = store.people.find((row) => row.id === personId)!;
|
||||
person.centroid = embedding ? encodeVectors([embedding]) : null;
|
||||
},
|
||||
async workDir(name) {
|
||||
const dir = path.join(root, SPACE, "renders", name);
|
||||
await mkdir(dir, { recursive: true });
|
||||
return dir;
|
||||
},
|
||||
async cleanup() {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
return harness;
|
||||
}
|
||||
264
packages/adapters/src/media-privacy-io.ts
Normal file
264
packages/adapters/src/media-privacy-io.ts
Normal file
@ -0,0 +1,264 @@
|
||||
import { constants } from "node:fs";
|
||||
import { copyFile, mkdir, readdir, readFile, realpath } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
|
||||
import type {
|
||||
BoxKeyframe,
|
||||
MediaCapabilities,
|
||||
MediaInferenceOperation,
|
||||
MediaRenderQuality,
|
||||
} from "@rakazo/contracts";
|
||||
import type { MediaRepos } from "@rakazo/db";
|
||||
import { assertInside, resolveInside } from "@rakazo/media/node";
|
||||
import type { MediaServiceDeps } from "./media-context.js";
|
||||
import { probeMedia, runMediaTool } from "./media-folder-source-probe.js";
|
||||
import { maskBounds, readPngLuma } from "./media-privacy-pixels.js";
|
||||
import type { PixelBox, TrackKeyframe } from "./media-privacy-plan.js";
|
||||
import { parseStoredKeyframes } from "./media-privacy-plan.js";
|
||||
|
||||
/*
|
||||
* File and tool helpers shared by anonymization, marks, and the gate. Stored
|
||||
* media paths are relative to the media root; every resolution goes through
|
||||
* the root checks of @rakazo/media/node. ffmpeg runs as an argv array only.
|
||||
*/
|
||||
|
||||
/** A redaction or check that cannot be carried out. Rendering stops instead of exporting. */
|
||||
export class PrivacyUnavailableError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "PrivacyUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
export interface PrivacyToolOptions {
|
||||
ffmpegPath?: string;
|
||||
ffprobePath?: string;
|
||||
}
|
||||
|
||||
/** Absolute path of a stored (root-relative) or already absolute path inside the media root. */
|
||||
export async function resolveStored(root: string, stored: string): Promise<string> {
|
||||
return path.isAbsolute(stored) ? assertInside(root, stored) : resolveInside(root, stored);
|
||||
}
|
||||
|
||||
export async function isInsideRoot(root: string, file: string): Promise<boolean> {
|
||||
try {
|
||||
await assertInside(root, file);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** The path of a file relative to the media root, for storing. */
|
||||
export async function toStored(root: string, absolute: string): Promise<string> {
|
||||
const inside = await assertInside(root, absolute);
|
||||
return path.relative(await realpath(root), inside);
|
||||
}
|
||||
|
||||
type AssetRow = Awaited<ReturnType<MediaRepos["getAssetRecord"]>>;
|
||||
|
||||
/**
|
||||
* The file a render reads for an asset: the 720p proxy for drafts (when one
|
||||
* exists), otherwise the original. Originals still on a device cannot render.
|
||||
*/
|
||||
export async function resolveAssetFile(
|
||||
deps: Pick<MediaServiceDeps, "repos" | "sources" | "mediaRoot">,
|
||||
spaceId: string,
|
||||
asset: AssetRow,
|
||||
quality: MediaRenderQuality,
|
||||
): Promise<string> {
|
||||
if (quality === "draft" && asset.proxyPath) return resolveStored(deps.mediaRoot, asset.proxyPath);
|
||||
return resolveOriginal(deps, spaceId, asset);
|
||||
}
|
||||
|
||||
export async function resolveOriginal(
|
||||
deps: Pick<MediaServiceDeps, "repos" | "sources" | "mediaRoot">,
|
||||
spaceId: string,
|
||||
asset: AssetRow,
|
||||
): Promise<string> {
|
||||
if (asset.storage === "uploaded") {
|
||||
if (!asset.storagePath) throw new PrivacyUnavailableError(`asset ${asset.id} has no file yet`);
|
||||
return resolveStored(deps.mediaRoot, asset.storagePath);
|
||||
}
|
||||
if (asset.storage === "referenced") {
|
||||
const source = await deps.repos.getSource(spaceId, asset.sourceId);
|
||||
const provider = deps.sources[source.kind as "folder" | "emulator"];
|
||||
if (!provider) throw new PrivacyUnavailableError(`source of asset ${asset.id} is unavailable`);
|
||||
return provider.resolve(source.config, asset.externalId);
|
||||
}
|
||||
throw new PrivacyUnavailableError(`asset ${asset.id} is still on a device; upload it first`);
|
||||
}
|
||||
|
||||
/**
|
||||
* A file inference can read: the proxy when it exists (normalized coordinates
|
||||
* and the timeline match the original), else the original when it lies inside
|
||||
* the media root. Null when neither does.
|
||||
*/
|
||||
export async function analysisFile(
|
||||
deps: Pick<MediaServiceDeps, "repos" | "sources" | "mediaRoot">,
|
||||
spaceId: string,
|
||||
asset: AssetRow,
|
||||
): Promise<string | null> {
|
||||
if (asset.proxyPath) return resolveStored(deps.mediaRoot, asset.proxyPath);
|
||||
try {
|
||||
const original = await resolveOriginal(deps, spaceId, asset);
|
||||
return (await isInsideRoot(deps.mediaRoot, original)) ? original : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface ProbedSource {
|
||||
width: number;
|
||||
height: number;
|
||||
durationMs: number;
|
||||
fps: number;
|
||||
hasAudio: boolean;
|
||||
rotation: 0 | 90 | 180 | 270;
|
||||
still: boolean;
|
||||
}
|
||||
|
||||
/** Display size (rotation applied), duration, and audio of a source file. */
|
||||
export async function probeSource(
|
||||
file: string,
|
||||
kind: string,
|
||||
options: PrivacyToolOptions & { signal?: AbortSignal } = {},
|
||||
): Promise<ProbedSource> {
|
||||
const probed = await probeMedia(file, {
|
||||
ffprobePath: options.ffprobePath,
|
||||
signal: options.signal,
|
||||
});
|
||||
if (!probed.hasVideo || !probed.width || !probed.height)
|
||||
throw new PrivacyUnavailableError("source has no readable video stream");
|
||||
const still = kind === "photo" || !probed.durationMs;
|
||||
const rotation = (((Math.round(probed.rotation / 90) * 90) % 360) + 360) % 360;
|
||||
return {
|
||||
width: probed.width,
|
||||
height: probed.height,
|
||||
durationMs: still ? 0 : (probed.durationMs ?? 0),
|
||||
fps: probed.fps ?? 30,
|
||||
hasAudio: probed.hasAudio,
|
||||
rotation: rotation as ProbedSource["rotation"],
|
||||
still,
|
||||
};
|
||||
}
|
||||
|
||||
/** Copy a file into `dir` under its own name (reflink when the filesystem allows). */
|
||||
export async function copyInto(file: string, dir: string): Promise<string> {
|
||||
await mkdir(dir, { recursive: true });
|
||||
const target = path.join(dir, path.basename(file));
|
||||
await copyFile(file, target, constants.COPYFILE_FICLONE);
|
||||
return target;
|
||||
}
|
||||
|
||||
/** Keyframes of a stored face track file (relative to the media root). */
|
||||
export async function readTrackKeyframes(root: string, stored: string): Promise<TrackKeyframe[]> {
|
||||
const file = await resolveStored(root, stored);
|
||||
return parseStoredKeyframes(JSON.parse(await readFile(file, "utf8")));
|
||||
}
|
||||
|
||||
/** Mask frame times of a mask directory, ascending. */
|
||||
export async function maskTimes(maskDir: string): Promise<number[]> {
|
||||
const names = await readdir(maskDir);
|
||||
return names
|
||||
.filter((name) => /^\d+\.png$/.test(name))
|
||||
.map((name) => Number.parseInt(name, 10))
|
||||
.sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
/** Bounding box per mask frame. Empty masks are skipped. */
|
||||
export async function maskKeyframes(maskDir: string): Promise<BoxKeyframe[]> {
|
||||
const keyframes: BoxKeyframe[] = [];
|
||||
for (const tMs of await maskTimes(maskDir)) {
|
||||
const box = maskBounds(await readPngLuma(path.join(maskDir, `${tMs}.png`)));
|
||||
if (box) keyframes.push({ tMs, box });
|
||||
}
|
||||
return keyframes;
|
||||
}
|
||||
|
||||
/** Whether an inference operation is available; synthetic identity also needs a commercial license. */
|
||||
export function operationAvailable(
|
||||
capabilities: MediaCapabilities | null,
|
||||
operation: MediaInferenceOperation,
|
||||
): boolean {
|
||||
const entry = capabilities?.operations[operation];
|
||||
if (!entry?.available) return false;
|
||||
if (operation === "synthesizeIdentity") return entry.commercialUse === true;
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function inferenceCapabilities(
|
||||
inference: MediaInferenceProvider | null,
|
||||
signal?: AbortSignal,
|
||||
): Promise<MediaCapabilities | null> {
|
||||
return inference ? inference.capabilities(signal) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lossless static crop of a video (or a still to PNG). Audio is copied; all
|
||||
* container metadata is dropped.
|
||||
*/
|
||||
export async function cropFile(
|
||||
input: string,
|
||||
output: string,
|
||||
crop: PixelBox,
|
||||
still: boolean,
|
||||
options: PrivacyToolOptions & { signal?: AbortSignal } = {},
|
||||
): Promise<void> {
|
||||
const filter = `crop=${crop.w}:${crop.h}:${crop.x}:${crop.y}`;
|
||||
const args = still
|
||||
? [
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-i",
|
||||
input,
|
||||
"-vf",
|
||||
filter,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-map_metadata",
|
||||
"-1",
|
||||
output,
|
||||
]
|
||||
: [
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-i",
|
||||
input,
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"0:a?",
|
||||
"-vf",
|
||||
filter,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-qp",
|
||||
"0",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"copy",
|
||||
"-map_metadata",
|
||||
"-1",
|
||||
output,
|
||||
];
|
||||
await runMediaTool(options.ffmpegPath ?? "ffmpeg", args, { signal: options.signal });
|
||||
}
|
||||
|
||||
/** First frame of a video as PNG (redaction providers write videos, stills render from images). */
|
||||
export async function firstFrame(
|
||||
input: string,
|
||||
output: string,
|
||||
options: PrivacyToolOptions & { signal?: AbortSignal } = {},
|
||||
): Promise<void> {
|
||||
await runMediaTool(
|
||||
options.ffmpegPath ?? "ffmpeg",
|
||||
["-v", "error", "-y", "-i", input, "-frames:v", "1", "-map_metadata", "-1", output],
|
||||
{ signal: options.signal },
|
||||
);
|
||||
}
|
||||
104
packages/adapters/src/media-privacy-pixels.test.ts
Normal file
104
packages/adapters/src/media-privacy-pixels.test.ts
Normal file
@ -0,0 +1,104 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { encodeMaskPng } from "./media-emulator.js";
|
||||
import type { LumaFrame } from "./media-privacy-pixels.js";
|
||||
import {
|
||||
controlAgreement,
|
||||
decodePngLuma,
|
||||
maskBounds,
|
||||
residualVisibility,
|
||||
temporalStability,
|
||||
} from "./media-privacy-pixels.js";
|
||||
|
||||
/** A 64x64 checkerboard of 4 px squares: detail everywhere. */
|
||||
function checker(width = 64, height = 64, shift = 0): LumaFrame {
|
||||
const luma = new Uint8Array(width * height);
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
luma[y * width + x] = (Math.floor((x + shift) / 4) + Math.floor(y / 4)) % 2 ? 220 : 30;
|
||||
}
|
||||
}
|
||||
return { width, height, luma };
|
||||
}
|
||||
|
||||
function fill(
|
||||
frame: LumaFrame,
|
||||
box: { x: number; y: number; w: number; h: number },
|
||||
value: number,
|
||||
) {
|
||||
const out = { ...frame, luma: frame.luma.slice() };
|
||||
for (
|
||||
let y = Math.floor(box.y * frame.height);
|
||||
y < Math.ceil((box.y + box.h) * frame.height);
|
||||
y++
|
||||
) {
|
||||
for (
|
||||
let x = Math.floor(box.x * frame.width);
|
||||
x < Math.ceil((box.x + box.w) * frame.width);
|
||||
x++
|
||||
) {
|
||||
out.luma[y * frame.width + x] = value;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const region = { x: 0.25, y: 0.25, w: 0.5, h: 0.5 };
|
||||
|
||||
describe("PNG masks", () => {
|
||||
it("decodes the masks the segmenter writes", () => {
|
||||
const mask = decodePngLuma(encodeMaskPng(40, 20, { x: 0.5, y: 0.25, w: 0.25, h: 0.5 }));
|
||||
expect(mask).toMatchObject({ width: 40, height: 20 });
|
||||
expect(maskBounds(mask)).toEqual({ x: 0.5, y: 0.25, w: 0.25, h: 0.5 });
|
||||
expect(maskBounds(decodePngLuma(encodeMaskPng(8, 8, { x: 0, y: 0, w: 0, h: 0 })))).toBeNull();
|
||||
expect(() => decodePngLuma(Buffer.from("nope"))).toThrow("not a PNG");
|
||||
});
|
||||
});
|
||||
|
||||
describe("residual visibility", () => {
|
||||
it("is 1 for untouched detail and 0 for a filled or flat region", () => {
|
||||
const source = checker();
|
||||
expect(residualVisibility({ source, treated: source, box: region })).toBe(1);
|
||||
expect(residualVisibility({ source, treated: fill(source, region, 128), box: region })).toBe(0);
|
||||
const flat = fill(source, region, 90);
|
||||
expect(
|
||||
residualVisibility({ source: flat, treated: flat, box: { x: 0.3, y: 0.3, w: 0.4, h: 0.4 } }),
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it("counts only the part a crop still shows and only mask pixels", () => {
|
||||
const source = checker();
|
||||
// The treated frame is the right half of the source, untouched.
|
||||
const half: LumaFrame = { width: 32, height: 64, luma: new Uint8Array(32 * 64) };
|
||||
for (let y = 0; y < 64; y++)
|
||||
half.luma.set(source.luma.subarray(y * 64 + 32, y * 64 + 64), y * 32);
|
||||
const left = { x: 0, y: 0.25, w: 0.4, h: 0.5 };
|
||||
expect(residualVisibility({ source, treated: half, box: left, offset: { x: 32, y: 0 } })).toBe(
|
||||
0,
|
||||
);
|
||||
expect(
|
||||
residualVisibility({ source, treated: half, box: region, offset: { x: 32, y: 0 } }),
|
||||
).toBe(1);
|
||||
const mask = decodePngLuma(encodeMaskPng(64, 64, { x: 0.5, y: 0.25, w: 0.25, h: 0.5 }));
|
||||
const treated = fill(source, { x: 0.25, y: 0.25, w: 0.25, h: 0.5 }, 128);
|
||||
expect(residualVisibility({ source, treated, box: region, mask })).toBe(1);
|
||||
});
|
||||
|
||||
it("detects frames that are not the same moment", () => {
|
||||
const source = checker();
|
||||
const treated = fill(source, region, 128);
|
||||
expect(controlAgreement({ source, treated, boxes: [region] })).toBe(1);
|
||||
expect(controlAgreement({ source, treated: checker(64, 64, 2), boxes: [region] })).toBeLessThan(
|
||||
0.8,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("temporal stability", () => {
|
||||
it("drops when the treated region flickers more than the frame", () => {
|
||||
const a = checker();
|
||||
expect(temporalStability([a, a, a], region)).toBe(1);
|
||||
const flicker = [fill(a, region, 0), fill(a, region, 255), fill(a, region, 0)];
|
||||
expect(temporalStability(flicker, region)).toBe(0);
|
||||
expect(temporalStability([a], region)).toBe(1);
|
||||
});
|
||||
});
|
||||
310
packages/adapters/src/media-privacy-pixels.ts
Normal file
310
packages/adapters/src/media-privacy-pixels.ts
Normal file
@ -0,0 +1,310 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { inflateSync } from "node:zlib";
|
||||
import type { NormalizedBox } from "@rakazo/contracts";
|
||||
import { runMediaTool } from "./media-folder-source-probe.js";
|
||||
|
||||
/*
|
||||
* Pixel checks for the privacy gate and mark quality. Video frames are decoded
|
||||
* by ffmpeg straight to 8-bit luma; mask PNGs from the segmenter go through the
|
||||
* decoder below (8-bit gray, gray+alpha, RGB, RGBA; not interlaced, not palette).
|
||||
*/
|
||||
|
||||
export interface LumaFrame {
|
||||
width: number;
|
||||
height: number;
|
||||
/** One luma byte per pixel, row-major. */
|
||||
luma: Uint8Array;
|
||||
}
|
||||
|
||||
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
const CHANNELS: Record<number, number> = { 0: 1, 2: 3, 4: 2, 6: 4 };
|
||||
|
||||
function paeth(a: number, b: number, c: number): number {
|
||||
const p = a + b - c;
|
||||
const pa = Math.abs(p - a);
|
||||
const pb = Math.abs(p - b);
|
||||
const pc = Math.abs(p - c);
|
||||
if (pa <= pb && pa <= pc) return a;
|
||||
return pb <= pc ? b : c;
|
||||
}
|
||||
|
||||
/** Decode a PNG to luma (Rec. 601 weights). Throws on anything it cannot read exactly. */
|
||||
export function decodePngLuma(png: Buffer): LumaFrame {
|
||||
if (png.length < 8 || !png.subarray(0, 8).equals(PNG_SIGNATURE)) throw new Error("not a PNG");
|
||||
let offset = 8;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
let channels = 0;
|
||||
const data: Buffer[] = [];
|
||||
while (offset + 8 <= png.length) {
|
||||
const length = png.readUInt32BE(offset);
|
||||
const type = png.toString("ascii", offset + 4, offset + 8);
|
||||
const body = png.subarray(offset + 8, offset + 8 + length);
|
||||
offset += 12 + length;
|
||||
if (type === "IHDR") {
|
||||
width = body.readUInt32BE(0);
|
||||
height = body.readUInt32BE(4);
|
||||
const depth = body[8];
|
||||
const colorType = body[9] ?? -1;
|
||||
const interlace = body[12];
|
||||
channels = CHANNELS[colorType] ?? 0;
|
||||
if (depth !== 8 || channels === 0 || interlace !== 0)
|
||||
throw new Error("unsupported PNG layout");
|
||||
} else if (type === "IDAT") {
|
||||
data.push(body);
|
||||
} else if (type === "IEND") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (width === 0 || height === 0 || channels === 0) throw new Error("PNG has no image header");
|
||||
const raw = inflateSync(Buffer.concat(data));
|
||||
const stride = width * channels;
|
||||
if (raw.length < (stride + 1) * height) throw new Error("PNG data is truncated");
|
||||
const pixels = Buffer.alloc(stride * height);
|
||||
for (let y = 0; y < height; y++) {
|
||||
const filter = raw[y * (stride + 1)];
|
||||
const line = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1));
|
||||
const row = y * stride;
|
||||
for (let x = 0; x < stride; x++) {
|
||||
const left = x >= channels ? (pixels[row + x - channels] ?? 0) : 0;
|
||||
const up = y > 0 ? (pixels[row - stride + x] ?? 0) : 0;
|
||||
const upLeft = y > 0 && x >= channels ? (pixels[row - stride + x - channels] ?? 0) : 0;
|
||||
const value = line[x] ?? 0;
|
||||
let out: number;
|
||||
if (filter === 0) out = value;
|
||||
else if (filter === 1) out = value + left;
|
||||
else if (filter === 2) out = value + up;
|
||||
else if (filter === 3) out = value + ((left + up) >> 1);
|
||||
else if (filter === 4) out = value + paeth(left, up, upLeft);
|
||||
else throw new Error("unknown PNG filter");
|
||||
pixels[row + x] = out & 0xff;
|
||||
}
|
||||
}
|
||||
const luma = new Uint8Array(width * height);
|
||||
for (let i = 0; i < width * height; i++) {
|
||||
const base = i * channels;
|
||||
if (channels <= 2) {
|
||||
luma[i] = pixels[base] ?? 0;
|
||||
} else {
|
||||
const r = pixels[base] ?? 0;
|
||||
const g = pixels[base + 1] ?? 0;
|
||||
const b = pixels[base + 2] ?? 0;
|
||||
luma[i] = Math.round(0.299 * r + 0.587 * g + 0.114 * b);
|
||||
}
|
||||
}
|
||||
return { width, height, luma };
|
||||
}
|
||||
|
||||
export async function readPngLuma(file: string): Promise<LumaFrame> {
|
||||
return decodePngLuma(await readFile(file));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode the frame shown at `tMs` as luma, scaled to `width` x `height` with
|
||||
* nearest-neighbour sampling (a no-op at the display size). Throws unless
|
||||
* ffmpeg returns a full frame.
|
||||
*/
|
||||
export async function extractLuma(
|
||||
file: string,
|
||||
input: { tMs: number; width: number; height: number; still?: boolean },
|
||||
options: { ffmpegPath?: string; signal?: AbortSignal } = {},
|
||||
): Promise<LumaFrame> {
|
||||
const seek = input.still ? [] : ["-ss", (Math.max(0, input.tMs) / 1000).toFixed(3)];
|
||||
const { stdout } = await runMediaTool(
|
||||
options.ffmpegPath ?? "ffmpeg",
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
...seek,
|
||||
"-i",
|
||||
file,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-vf",
|
||||
`scale=${input.width}:${input.height}:flags=neighbor`,
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"gray",
|
||||
"-",
|
||||
],
|
||||
{ signal: options.signal },
|
||||
);
|
||||
const size = input.width * input.height;
|
||||
if (stdout.length < size) throw new Error(`no frame at ${input.tMs} ms`);
|
||||
return {
|
||||
width: input.width,
|
||||
height: input.height,
|
||||
luma: new Uint8Array(stdout.subarray(0, size)),
|
||||
};
|
||||
}
|
||||
|
||||
/** Bounding box of the bright (>= 128) pixels of a mask, or null when the mask is empty. */
|
||||
export function maskBounds(mask: LumaFrame): NormalizedBox | null {
|
||||
let minX = mask.width;
|
||||
let minY = mask.height;
|
||||
let maxX = -1;
|
||||
let maxY = -1;
|
||||
for (let y = 0; y < mask.height; y++) {
|
||||
for (let x = 0; x < mask.width; x++) {
|
||||
if ((mask.luma[y * mask.width + x] ?? 0) < 128) continue;
|
||||
minX = Math.min(minX, x);
|
||||
minY = Math.min(minY, y);
|
||||
maxX = Math.max(maxX, x);
|
||||
maxY = Math.max(maxY, y);
|
||||
}
|
||||
}
|
||||
if (maxX < 0) return null;
|
||||
return {
|
||||
x: minX / mask.width,
|
||||
y: minY / mask.height,
|
||||
w: (maxX + 1 - minX) / mask.width,
|
||||
h: (maxY + 1 - minY) / mask.height,
|
||||
};
|
||||
}
|
||||
|
||||
/** A luma step at least this large marks a detail pixel (an edge someone could recognise). */
|
||||
export const DETAIL_MIN_STEP = 12;
|
||||
/** A detail pixel within this many luma levels of the source counts as unchanged. */
|
||||
export const UNCHANGED_MAX_DIFF = 2;
|
||||
/** Looser tolerance for the control area outside treated regions (codec noise). */
|
||||
const CONTROL_MAX_DIFF = 6;
|
||||
/** Regions with fewer detail pixels show nothing identifying. */
|
||||
const MIN_DETAIL_PIXELS = 16;
|
||||
|
||||
function pixelRange(box: NormalizedBox, width: number, height: number) {
|
||||
return {
|
||||
x0: Math.max(0, Math.floor(box.x * width)),
|
||||
y0: Math.max(0, Math.floor(box.y * height)),
|
||||
x1: Math.min(width, Math.ceil((box.x + box.w) * width)),
|
||||
y1: Math.min(height, Math.ceil((box.y + box.h) * height)),
|
||||
};
|
||||
}
|
||||
|
||||
function detailStep(frame: LumaFrame, x: number, y: number): number {
|
||||
const at = y * frame.width + x;
|
||||
const value = frame.luma[at] ?? 0;
|
||||
const right = x + 1 < frame.width ? (frame.luma[at + 1] ?? value) : value;
|
||||
const down = y + 1 < frame.height ? (frame.luma[at + frame.width] ?? value) : value;
|
||||
return Math.abs(value - right) + Math.abs(value - down);
|
||||
}
|
||||
|
||||
/**
|
||||
* Residual visibility of a region (0..1): the share of the source's detail
|
||||
* pixels (edges) that the treated frame left unchanged, value and edge
|
||||
* strength alike. Blur, pixelation, fills, and inpainting all rewrite edges,
|
||||
* so a treated region scores near 0, an untouched one 1. A flat region
|
||||
* without detail scores 0: there is nothing to recognise.
|
||||
*
|
||||
* `source` and `treated` are pixel-aligned at full resolution; when `treated`
|
||||
* is a crop of the source, `offset` is the crop origin in source pixels and
|
||||
* detail outside the crop is invisible. Optional `mask` (normalized over the
|
||||
* source frame) limits the region to mask pixels.
|
||||
*/
|
||||
export function residualVisibility(input: {
|
||||
source: LumaFrame;
|
||||
treated: LumaFrame;
|
||||
box: NormalizedBox;
|
||||
offset?: { x: number; y: number };
|
||||
mask?: LumaFrame;
|
||||
}): number {
|
||||
const { source, treated, box, mask } = input;
|
||||
const offset = input.offset ?? { x: 0, y: 0 };
|
||||
const { x0, y0, x1, y1 } = pixelRange(box, source.width, source.height);
|
||||
let detail = 0;
|
||||
let unchanged = 0;
|
||||
for (let y = y0; y < y1; y++) {
|
||||
for (let x = x0; x < x1; x++) {
|
||||
if (mask) {
|
||||
const mx = Math.min(mask.width - 1, Math.floor((x / source.width) * mask.width));
|
||||
const my = Math.min(mask.height - 1, Math.floor((y / source.height) * mask.height));
|
||||
if ((mask.luma[my * mask.width + mx] ?? 0) < 128) continue;
|
||||
}
|
||||
const step = detailStep(source, x, y);
|
||||
if (step < DETAIL_MIN_STEP) continue;
|
||||
const tx = x - offset.x;
|
||||
const ty = y - offset.y;
|
||||
if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue;
|
||||
detail += 1;
|
||||
const value = source.luma[y * source.width + x] ?? 0;
|
||||
// Visible means the pixel and its edge survived, not a value that happens to match.
|
||||
if (
|
||||
Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= UNCHANGED_MAX_DIFF &&
|
||||
Math.abs(step - detailStep(treated, tx, ty)) <= 2 * UNCHANGED_MAX_DIFF
|
||||
)
|
||||
unchanged += 1;
|
||||
}
|
||||
}
|
||||
return detail < MIN_DETAIL_PIXELS ? 0 : unchanged / detail;
|
||||
}
|
||||
|
||||
/**
|
||||
* Share of detail pixels outside every treated box that the treated frame kept
|
||||
* (within codec tolerance). A low value means the two frames are not the same
|
||||
* moment or framing, so a residual measured between them proves nothing.
|
||||
* Returns 1 when the control area has too little detail to judge.
|
||||
*/
|
||||
export function controlAgreement(input: {
|
||||
source: LumaFrame;
|
||||
treated: LumaFrame;
|
||||
boxes: NormalizedBox[];
|
||||
offset?: { x: number; y: number };
|
||||
}): number {
|
||||
const { source, treated, boxes } = input;
|
||||
const offset = input.offset ?? { x: 0, y: 0 };
|
||||
const ranges = boxes.map((box) => pixelRange(box, source.width, source.height));
|
||||
let detail = 0;
|
||||
let kept = 0;
|
||||
for (let y = 0; y < source.height; y++) {
|
||||
for (let x = 0; x < source.width; x++) {
|
||||
const tx = x - offset.x;
|
||||
const ty = y - offset.y;
|
||||
if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue;
|
||||
if (ranges.some((r) => x >= r.x0 - 2 && x < r.x1 + 2 && y >= r.y0 - 2 && y < r.y1 + 2))
|
||||
continue;
|
||||
if (detailStep(source, x, y) < DETAIL_MIN_STEP) continue;
|
||||
detail += 1;
|
||||
const value = source.luma[y * source.width + x] ?? 0;
|
||||
if (Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= CONTROL_MAX_DIFF)
|
||||
kept += 1;
|
||||
}
|
||||
}
|
||||
return detail < MIN_DETAIL_PIXELS ? 1 : kept / detail;
|
||||
}
|
||||
|
||||
/**
|
||||
* Temporal stability of a treated region over consecutive frames (0..1):
|
||||
* 1 when the region changes no more between neighbours than the rest of the
|
||||
* frame, falling to 0 as it flickers 24+ luma levels more on average.
|
||||
*/
|
||||
export function temporalStability(frames: LumaFrame[], box: NormalizedBox): number {
|
||||
if (frames.length < 2) return 1;
|
||||
let excess = 0;
|
||||
for (let i = 1; i < frames.length; i++) {
|
||||
const a = frames[i - 1]!;
|
||||
const b = frames[i]!;
|
||||
if (a.width !== b.width || a.height !== b.height) throw new Error("frame sizes differ");
|
||||
const r = pixelRange(box, a.width, a.height);
|
||||
let inside = 0;
|
||||
let insideCount = 0;
|
||||
let outside = 0;
|
||||
let outsideCount = 0;
|
||||
for (let y = 0; y < a.height; y++) {
|
||||
for (let x = 0; x < a.width; x++) {
|
||||
const at = y * a.width + x;
|
||||
const diff = Math.abs((a.luma[at] ?? 0) - (b.luma[at] ?? 0));
|
||||
if (x >= r.x0 && x < r.x1 && y >= r.y0 && y < r.y1) {
|
||||
inside += diff;
|
||||
insideCount += 1;
|
||||
} else {
|
||||
outside += diff;
|
||||
outsideCount += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
const meanInside = insideCount ? inside / insideCount : 0;
|
||||
const meanOutside = outsideCount ? outside / outsideCount : 0;
|
||||
excess += Math.max(0, meanInside - meanOutside);
|
||||
}
|
||||
return Math.max(0, Math.min(1, 1 - excess / (frames.length - 1) / 24));
|
||||
}
|
||||
210
packages/adapters/src/media-privacy-plan.test.ts
Normal file
210
packages/adapters/src/media-privacy-plan.test.ts
Normal file
@ -0,0 +1,210 @@
|
||||
import type { FaceRedaction } from "@rakazo/contracts";
|
||||
import { encodeVectors } from "@rakazo/media";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { mergeTracks, syntheticSeedFor } from "./media-anonymize.js";
|
||||
import { testEdl } from "./media-privacy-fixtures.js";
|
||||
import {
|
||||
boxAt,
|
||||
cropPixels,
|
||||
effectiveModes,
|
||||
ensureMinSize,
|
||||
GAP_HOLD_GROWTH,
|
||||
matchPerson,
|
||||
outsideCrop,
|
||||
padBox,
|
||||
parseStoredKeyframes,
|
||||
protectKeyframes,
|
||||
resolveProtection,
|
||||
sourceToOutput,
|
||||
} from "./media-privacy-plan.js";
|
||||
|
||||
const box = { x: 0.4, y: 0.4, w: 0.1, h: 0.1 };
|
||||
|
||||
function face(
|
||||
id: string,
|
||||
target: FaceRedaction["target"],
|
||||
mode: FaceRedaction["mode"],
|
||||
): FaceRedaction {
|
||||
return { id, target, mode, paddingPct: 20, featherPct: 10, minSizePct: 2 };
|
||||
}
|
||||
|
||||
describe("protectKeyframes", () => {
|
||||
it("interpolates short gaps and holds a grown box over long ones for review", () => {
|
||||
const result = protectKeyframes(
|
||||
[
|
||||
{ tMs: 0, box },
|
||||
{ tMs: 400, box: { ...box, x: 0.44 } },
|
||||
{ tMs: 1400, box: { ...box, x: 0.5 } },
|
||||
],
|
||||
[{ startMs: 0, endMs: 2000 }],
|
||||
2,
|
||||
);
|
||||
expect(result.gaps).toEqual([{ startMs: 400, endMs: 1400 }]);
|
||||
expect(result.keyframes.map((k) => k.tMs)).toEqual([0, 100, 200, 300, 400, 401, 1399, 1400]);
|
||||
const held = result.keyframes.find((k) => k.tMs === 401)!.box;
|
||||
expect(held.w).toBeCloseTo(0.1 * (1 + 2 * GAP_HOLD_GROWTH), 3);
|
||||
});
|
||||
|
||||
it("treats weak detections as lost but still covers them", () => {
|
||||
const result = protectKeyframes(
|
||||
[
|
||||
{ tMs: 0, box, confidence: 0.9 },
|
||||
{ tMs: 400, box: { ...box, x: 0.8 }, confidence: 0.2 },
|
||||
{ tMs: 800, box, confidence: 0.9 },
|
||||
],
|
||||
[{ startMs: 0, endMs: 1000 }],
|
||||
2,
|
||||
);
|
||||
expect(result.gaps).toEqual([{ startMs: 0, endMs: 800 }]);
|
||||
const held = result.keyframes.find((k) => k.tMs === 1)!.box;
|
||||
expect(held.x + held.w).toBeGreaterThanOrEqual(0.9);
|
||||
});
|
||||
|
||||
it("ignores detections far outside the clip ranges and enforces a minimum size", () => {
|
||||
const result = protectKeyframes(
|
||||
[
|
||||
{ tMs: 0, box: { x: 0.5, y: 0.5, w: 0.001, h: 0.001 } },
|
||||
{ tMs: 5000, box },
|
||||
],
|
||||
[{ startMs: 0, endMs: 100 }],
|
||||
5,
|
||||
);
|
||||
expect(result.keyframes).toHaveLength(1);
|
||||
expect(result.keyframes[0]!.box.w).toBeGreaterThanOrEqual(0.05);
|
||||
expect(ensureMinSize({ x: 0.99, y: 0.99, w: 0.01, h: 0.01 }, 10)).toEqual({
|
||||
x: 0.9,
|
||||
y: 0.9,
|
||||
w: 0.1,
|
||||
h: 0.1,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("protection", () => {
|
||||
const defaults = new Map([
|
||||
["alice", "pixelate" as const],
|
||||
["bob", "none" as const],
|
||||
]);
|
||||
const faces = [
|
||||
face("by-track", { trackId: "t1" }, "solid"),
|
||||
face("by-person", { personId: "bob" }, "partial_eyes"),
|
||||
face("strangers", { unassignedFaces: true }, "pixelate"),
|
||||
];
|
||||
|
||||
it("prefers track over person over default over unassigned", () => {
|
||||
const resolve = (storedId: string | null, personId: string | null, protectUnassigned = false) =>
|
||||
resolveProtection({ storedId, personId }, { faces, defaults, protectUnassigned })?.mode;
|
||||
expect(resolve("t1", "alice")).toBe("solid");
|
||||
expect(resolve("t2", "bob")).toBe("partial_eyes");
|
||||
expect(resolve("t2", "alice")).toBe("pixelate");
|
||||
expect(resolve(null, null)).toBe("pixelate");
|
||||
expect(
|
||||
resolveProtection(
|
||||
{ storedId: null, personId: null },
|
||||
{ faces: [], defaults, protectUnassigned: true },
|
||||
),
|
||||
).toMatchObject({ mode: "blur", origin: "unassigned" });
|
||||
expect(
|
||||
resolveProtection(
|
||||
{ storedId: null, personId: null },
|
||||
{ faces: [], defaults, protectUnassigned: false },
|
||||
),
|
||||
).toBeNull();
|
||||
expect(effectiveModes(faces, defaults).get("bob")).toBe("partial_eyes");
|
||||
});
|
||||
|
||||
it("matches faces to people only above the similarity threshold", () => {
|
||||
const centroids = [
|
||||
{ personId: "alice", centroid: encodeVectors([[1, 0, 0]]) },
|
||||
{ personId: "bob", centroid: encodeVectors([[0, 1, 0]]) },
|
||||
];
|
||||
expect(matchPerson([0.9, 0.1, 0], centroids)).toBe("alice");
|
||||
expect(matchPerson([0.5, 0.5, 0.7], centroids)).toBeNull();
|
||||
expect(matchPerson(null, centroids)).toBeNull();
|
||||
});
|
||||
|
||||
it("merges fresh tracks into stored ones and keeps what the fresh pass missed", () => {
|
||||
const stored = [
|
||||
{ id: "s1", storedId: "s1", personId: "alice", keyframes: [{ tMs: 0, box }] },
|
||||
{ id: "s2", storedId: "s2", personId: null, keyframes: [{ tMs: 0, box: { ...box, x: 0 } }] },
|
||||
];
|
||||
const merged = mergeTracks(
|
||||
"a1",
|
||||
stored,
|
||||
[
|
||||
{
|
||||
keyframes: [
|
||||
{ tMs: 0, box, confidence: 0.9 },
|
||||
{ tMs: 100, box, confidence: 0.9 },
|
||||
],
|
||||
embedding: null,
|
||||
},
|
||||
{ keyframes: [{ tMs: 0, box: { ...box, y: 0.8 }, confidence: 0.9 }], embedding: [1, 0, 0] },
|
||||
],
|
||||
[{ personId: "carol", centroid: encodeVectors([[1, 0, 0]]) }],
|
||||
);
|
||||
expect(merged.map((t) => [t.id, t.personId, t.keyframes.length])).toEqual([
|
||||
["s1", "alice", 3],
|
||||
["fresh-a1-2", "carol", 1],
|
||||
["s2", null, 1],
|
||||
]);
|
||||
expect(syntheticSeedFor("alice")).toBe(syntheticSeedFor("alice"));
|
||||
expect(syntheticSeedFor("alice")).not.toBe(syntheticSeedFor("bob"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("geometry", () => {
|
||||
it("crops with the output aspect and knows what lies outside", () => {
|
||||
const geometry = { sourceWidth: 1920, sourceHeight: 1080, outWidth: 1080, outHeight: 1920 };
|
||||
const crop = cropPixels(geometry, { cx: 0.8, cy: 0.5, scale: 1 });
|
||||
expect(crop).toEqual({ x: 1232, y: 0, w: 608, h: 1080 });
|
||||
expect(outsideCrop({ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }, crop, 1920, 1080)).toBe(true);
|
||||
expect(outsideCrop({ x: 0.6, y: 0.1, w: 0.2, h: 0.2 }, crop, 1920, 1080)).toBe(false);
|
||||
expect(padBox(box, 20)).toEqual({
|
||||
x: 0.39,
|
||||
y: 0.39,
|
||||
w: expect.closeTo(0.12),
|
||||
h: expect.closeTo(0.12),
|
||||
});
|
||||
});
|
||||
|
||||
it("maps source spans onto the output timeline", () => {
|
||||
const edl = testEdl([
|
||||
{ id: "a", assetId: "x", inMs: 1000, outMs: 3000 },
|
||||
{ id: "b", assetId: "y", inMs: 0, outMs: 1000 },
|
||||
]);
|
||||
const [a, b] = edl.tracks.video;
|
||||
expect(sourceToOutput(edl, a!, { startMs: 1500, endMs: 5000 })).toEqual({
|
||||
startMs: 500,
|
||||
endMs: 2000,
|
||||
});
|
||||
expect(sourceToOutput(edl, b!, { startMs: 200, endMs: 300 })).toEqual({
|
||||
startMs: 2200,
|
||||
endMs: 2300,
|
||||
});
|
||||
expect(sourceToOutput(edl, a!, { startMs: 0, endMs: 500 })).toBeNull();
|
||||
});
|
||||
|
||||
it("finds the box between keyframes", () => {
|
||||
const keyframes = [
|
||||
{ tMs: 0, box },
|
||||
{ tMs: 100, box: { ...box, x: 0.5 } },
|
||||
];
|
||||
expect(boxAt(keyframes, 50)).toEqual({
|
||||
x: 0.4,
|
||||
y: 0.4,
|
||||
w: expect.closeTo(0.2),
|
||||
h: expect.closeTo(0.1),
|
||||
});
|
||||
expect(boxAt(keyframes, 150)).toEqual({ ...box, x: 0.5 });
|
||||
expect(boxAt(keyframes, 500)).toBeNull();
|
||||
});
|
||||
|
||||
it("reads stored keyframe files in both shapes", () => {
|
||||
expect(parseStoredKeyframes([{ tMs: 0, box, extra: 1 }])).toEqual([{ tMs: 0, box }]);
|
||||
expect(parseStoredKeyframes({ keyframes: [{ tMs: 5, box, confidence: 0.4 }] })).toEqual([
|
||||
{ tMs: 5, box, confidence: 0.4 },
|
||||
]);
|
||||
expect(() => parseStoredKeyframes({ frames: [] })).toThrow();
|
||||
});
|
||||
});
|
||||
499
packages/adapters/src/media-privacy-plan.ts
Normal file
499
packages/adapters/src/media-privacy-plan.ts
Normal file
@ -0,0 +1,499 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type {
|
||||
AnonymizeMode,
|
||||
BoxKeyframe,
|
||||
Edl,
|
||||
EdlVideoClip,
|
||||
FaceRedaction,
|
||||
NormalizedBox,
|
||||
} from "@rakazo/contracts";
|
||||
import { AiAlterationSchema, AnonymizeModeSchema, BoxKeyframeSchema } from "@rakazo/contracts";
|
||||
import { cosineSimilarity, decodeVectors, introMs, MAX_TRACKING_GAP_MS } from "@rakazo/media";
|
||||
import { z } from "zod";
|
||||
|
||||
/*
|
||||
* Pure planning helpers for face, mark, and region protection, plus the
|
||||
* privacy record that prepareSources leaves next to the render output for the
|
||||
* gate. Times are source times of the asset unless a name says otherwise;
|
||||
* boxes are normalized to the displayed (rotation-applied) frame.
|
||||
*/
|
||||
|
||||
/** Tracking rate of the fresh detection pass on clip sources. */
|
||||
export const FACE_TRACK_FPS = 10;
|
||||
/**
|
||||
* Minimum cosine similarity between a face embedding and a person centroid to
|
||||
* count as that person. Below it the face is unassigned, which fails closed:
|
||||
* unassigned faces are redacted and block the gate while a protected person
|
||||
* appears in the project.
|
||||
*/
|
||||
export const FACE_MATCH_MIN_COSINE = 0.6;
|
||||
/** Detections below this confidence count as lost (spec 7: hold and grow the last mask). */
|
||||
export const FACE_MIN_CONFIDENCE = 0.5;
|
||||
/** A fresh track inherits a stored track's person when their boxes overlap this much on average. */
|
||||
export const TRACK_MATCH_MIN_IOU = 0.3;
|
||||
/** Step for linear interpolation across short detection gaps (<= MAX_TRACKING_GAP_MS). */
|
||||
export const INTERPOLATE_STEP_MS = 100;
|
||||
/** Lost detections hold the last box, enlarged by this share on every side (spec 7). */
|
||||
export const GAP_HOLD_GROWTH = 0.25;
|
||||
/** Padding around marks and regions, in percent of the box. */
|
||||
export const MARK_PADDING_PCT = 10;
|
||||
/** Person-bound marks re-found below this segmentation score go to review and block the gate. */
|
||||
export const MARK_REVIEW_MIN_SCORE = 0.6;
|
||||
/** Treatment quality below this suggests (and, while rendering, applies) a fallback. */
|
||||
export const MARK_QUALITY_MIN = 0.7;
|
||||
|
||||
/** Name of the privacy record prepareSources writes into the render work directory. */
|
||||
export const PRIVACY_RECORD_FILE = "privacy.json";
|
||||
|
||||
export type ProtectMode = Exclude<AnonymizeMode, "none">;
|
||||
export type PartialMode = "partial_eyes" | "partial_upper" | "partial_lower";
|
||||
|
||||
export interface Protection {
|
||||
mode: ProtectMode;
|
||||
paddingPct: number;
|
||||
featherPct: number;
|
||||
minSizePct: number;
|
||||
/** Redaction id from the EDL, "default" (person setting), or "unassigned". */
|
||||
origin: string;
|
||||
}
|
||||
|
||||
export type TrackKeyframe = BoxKeyframe & { confidence?: number };
|
||||
|
||||
export interface CandidateTrack {
|
||||
/** Stable id: the stored track id, or `fresh-<n>` for faces only the fresh pass found. */
|
||||
id: string;
|
||||
/** Stored track id when the face matches an analysed track, for EDL track targets. */
|
||||
storedId: string | null;
|
||||
personId: string | null;
|
||||
keyframes: TrackKeyframe[];
|
||||
}
|
||||
|
||||
export interface TimeWindow {
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
}
|
||||
|
||||
const DEFAULT_PROTECTION = { paddingPct: 20, featherPct: 10, minSizePct: 2 };
|
||||
|
||||
export function isPartial(mode: AnonymizeMode): mode is PartialMode {
|
||||
return mode === "partial_eyes" || mode === "partial_upper" || mode === "partial_lower";
|
||||
}
|
||||
|
||||
function fromRedaction(redaction: FaceRedaction): Protection {
|
||||
return {
|
||||
mode: redaction.mode,
|
||||
paddingPct: redaction.paddingPct,
|
||||
featherPct: redaction.featherPct,
|
||||
minSizePct: redaction.minSizePct,
|
||||
origin: redaction.id,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Protection for one face track. Most specific wins: EDL track target, EDL
|
||||
* person target, the person's default, then the EDL unassigned target, and
|
||||
* finally a blur for unassigned faces while any protected person appears in
|
||||
* the project.
|
||||
*/
|
||||
export function resolveProtection(
|
||||
track: Pick<CandidateTrack, "storedId" | "personId">,
|
||||
context: {
|
||||
faces: FaceRedaction[];
|
||||
defaults: Map<string, AnonymizeMode>;
|
||||
protectUnassigned: boolean;
|
||||
},
|
||||
): Protection | null {
|
||||
if (track.storedId) {
|
||||
const byTrack = context.faces.find(
|
||||
(face) => "trackId" in face.target && face.target.trackId === track.storedId,
|
||||
);
|
||||
if (byTrack) return fromRedaction(byTrack);
|
||||
}
|
||||
if (track.personId) {
|
||||
const byPerson = context.faces.find(
|
||||
(face) => "personId" in face.target && face.target.personId === track.personId,
|
||||
);
|
||||
if (byPerson) return fromRedaction(byPerson);
|
||||
const mode = context.defaults.get(track.personId);
|
||||
if (mode && mode !== "none") return { mode, ...DEFAULT_PROTECTION, origin: "default" };
|
||||
return null;
|
||||
}
|
||||
const unassigned = context.faces.find((face) => "unassignedFaces" in face.target);
|
||||
if (unassigned) return fromRedaction(unassigned);
|
||||
if (context.protectUnassigned)
|
||||
return { mode: "blur", ...DEFAULT_PROTECTION, origin: "unassigned" };
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Effective mode per person: an EDL person target wins over the person's default. */
|
||||
export function effectiveModes(
|
||||
faces: FaceRedaction[],
|
||||
defaults: Map<string, AnonymizeMode>,
|
||||
): Map<string, AnonymizeMode> {
|
||||
const modes = new Map(defaults);
|
||||
for (const face of faces) {
|
||||
if ("personId" in face.target) modes.set(face.target.personId, face.mode);
|
||||
}
|
||||
return modes;
|
||||
}
|
||||
|
||||
/** Best person for an embedding by cosine to the centroids, or null below the threshold. */
|
||||
export function matchPerson(
|
||||
embedding: ArrayLike<number> | null,
|
||||
centroids: Array<{ personId: string; centroid: Uint8Array }>,
|
||||
): string | null {
|
||||
if (!embedding || embedding.length === 0) return null;
|
||||
let best: { personId: string; score: number } | null = null;
|
||||
for (const { personId, centroid } of centroids) {
|
||||
if (centroid.byteLength !== embedding.length * 4) continue;
|
||||
const vector = decodeVectors(centroid, embedding.length)[0];
|
||||
if (!vector) continue;
|
||||
const score = cosineSimilarity(embedding, vector);
|
||||
if (score >= FACE_MATCH_MIN_COSINE && (!best || score > best.score)) best = { personId, score };
|
||||
}
|
||||
return best?.personId ?? null;
|
||||
}
|
||||
|
||||
const clampUnit = (value: number) => Math.min(1, Math.max(0, value));
|
||||
|
||||
/** Grow a box by `share` of its size on every side, clamped to the frame. */
|
||||
export function growBox(box: NormalizedBox, share: number): NormalizedBox {
|
||||
const x = clampUnit(box.x - box.w * share);
|
||||
const y = clampUnit(box.y - box.h * share);
|
||||
return {
|
||||
x,
|
||||
y,
|
||||
w: clampUnit(box.x + box.w + box.w * share) - x,
|
||||
h: clampUnit(box.y + box.h + box.h * share) - y,
|
||||
};
|
||||
}
|
||||
|
||||
/** Padding as the redaction providers apply it: paddingPct / 2 of the size on each side. */
|
||||
export function padBox(box: NormalizedBox, paddingPct: number): NormalizedBox {
|
||||
return growBox(box, Math.max(0, paddingPct) / 200);
|
||||
}
|
||||
|
||||
/** Enlarge a box around its centre until both sides reach `minSizePct` of the frame. */
|
||||
export function ensureMinSize(box: NormalizedBox, minSizePct: number): NormalizedBox {
|
||||
const min = minSizePct / 100;
|
||||
if (box.w >= min && box.h >= min) return box;
|
||||
const w = Math.min(1, Math.max(box.w, min));
|
||||
const h = Math.min(1, Math.max(box.h, min));
|
||||
const x = clampUnit(Math.min(box.x + box.w / 2 - w / 2, 1 - w));
|
||||
const y = clampUnit(Math.min(box.y + box.h / 2 - h / 2, 1 - h));
|
||||
return { x, y, w, h };
|
||||
}
|
||||
|
||||
export function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox {
|
||||
const x = Math.min(a.x, b.x);
|
||||
const y = Math.min(a.y, b.y);
|
||||
return {
|
||||
x,
|
||||
y,
|
||||
w: Math.max(a.x + a.w, b.x + b.w) - x,
|
||||
h: Math.max(a.y + a.h, b.y + b.h) - y,
|
||||
};
|
||||
}
|
||||
|
||||
export function iou(a: NormalizedBox, b: NormalizedBox): number {
|
||||
const w = Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x);
|
||||
const h = Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y);
|
||||
if (w <= 0 || h <= 0) return 0;
|
||||
const inter = w * h;
|
||||
return inter / (a.w * a.h + b.w * b.h - inter);
|
||||
}
|
||||
|
||||
function lerpBox(a: NormalizedBox, b: NormalizedBox, f: number): NormalizedBox {
|
||||
const mix = (p: number, q: number) => p + (q - p) * f;
|
||||
return { x: mix(a.x, b.x), y: mix(a.y, b.y), w: mix(a.w, b.w), h: mix(a.h, b.h) };
|
||||
}
|
||||
|
||||
/** Round outward so a rounded box never covers less than the exact one. */
|
||||
export function roundBoxOut(box: NormalizedBox): NormalizedBox {
|
||||
const x = Math.floor(box.x * 10_000) / 10_000;
|
||||
const y = Math.floor(box.y * 10_000) / 10_000;
|
||||
return {
|
||||
x,
|
||||
y,
|
||||
w: Math.min(1 - x, Math.ceil((box.x + box.w - x) * 10_000) / 10_000),
|
||||
h: Math.min(1 - y, Math.ceil((box.y + box.h - y) * 10_000) / 10_000),
|
||||
};
|
||||
}
|
||||
|
||||
function nearWindows(tMs: number, windows: TimeWindow[]): boolean {
|
||||
return windows.some(
|
||||
(w) => tMs >= w.startMs - MAX_TRACKING_GAP_MS && tMs <= w.endMs + MAX_TRACKING_GAP_MS,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyframes for redaction near the windows (clip ranges plus the bridging
|
||||
* limit on both sides). Detections below FACE_MIN_CONFIDENCE count as lost.
|
||||
* Gaps up to MAX_TRACKING_GAP_MS between confident detections are
|
||||
* interpolated; longer ones hold the last confident box grown by
|
||||
* GAP_HOLD_GROWTH (unioned with any weak detections) until the face is found
|
||||
* again, and are returned so the gate blocks them as needing review.
|
||||
*/
|
||||
export function protectKeyframes(
|
||||
keyframes: TrackKeyframe[],
|
||||
windows: TimeWindow[],
|
||||
minSizePct: number,
|
||||
): { keyframes: BoxKeyframe[]; gaps: TimeWindow[] } {
|
||||
const inside = [...keyframes]
|
||||
.filter((k) => nearWindows(k.tMs, windows))
|
||||
.sort((a, b) => a.tMs - b.tMs)
|
||||
.map((k) => ({
|
||||
tMs: k.tMs,
|
||||
box: ensureMinSize(k.box, minSizePct),
|
||||
confident: (k.confidence ?? 1) >= FACE_MIN_CONFIDENCE,
|
||||
}));
|
||||
const out: BoxKeyframe[] = [];
|
||||
const gaps: TimeWindow[] = [];
|
||||
const confident = inside.filter((k) => k.confident);
|
||||
const weak = inside.filter((k) => !k.confident);
|
||||
if (confident.length === 0) {
|
||||
// Only weak detections: cover them grown and send the whole span to review.
|
||||
for (const k of weak)
|
||||
out.push({ tMs: k.tMs, box: roundBoxOut(growBox(k.box, GAP_HOLD_GROWTH)) });
|
||||
const first = weak[0];
|
||||
const last = weak.at(-1);
|
||||
if (first && last) gaps.push({ startMs: first.tMs, endMs: Math.max(last.tMs, first.tMs + 1) });
|
||||
return { keyframes: out, gaps };
|
||||
}
|
||||
confident.forEach((current, index) => {
|
||||
out.push({ tMs: current.tMs, box: roundBoxOut(current.box) });
|
||||
const next = confident[index + 1];
|
||||
if (!next) return;
|
||||
const span = next.tMs - current.tMs;
|
||||
const between = weak.filter((k) => k.tMs > current.tMs && k.tMs < next.tMs);
|
||||
if (span <= MAX_TRACKING_GAP_MS) {
|
||||
for (let t = current.tMs + INTERPOLATE_STEP_MS; t < next.tMs; t += INTERPOLATE_STEP_MS) {
|
||||
let box = lerpBox(current.box, next.box, (t - current.tMs) / span);
|
||||
for (const w of between)
|
||||
if (Math.abs(w.tMs - t) < INTERPOLATE_STEP_MS) box = unionBox(box, w.box);
|
||||
out.push({ tMs: t, box: roundBoxOut(box) });
|
||||
}
|
||||
return;
|
||||
}
|
||||
let held = growBox(current.box, GAP_HOLD_GROWTH);
|
||||
for (const w of between) held = unionBox(held, w.box);
|
||||
const box = roundBoxOut(held);
|
||||
out.push({ tMs: current.tMs + 1, box }, { tMs: next.tMs - 1, box });
|
||||
gaps.push({ startMs: current.tMs, endMs: next.tMs });
|
||||
});
|
||||
return { keyframes: out, gaps };
|
||||
}
|
||||
|
||||
/** Box a keyframe list covers at `tMs`: the union of the bracketing keyframes, or null outside. */
|
||||
export function boxAt(keyframes: BoxKeyframe[], tMs: number, holdMs = 100): NormalizedBox | null {
|
||||
let before: BoxKeyframe | null = null;
|
||||
let after: BoxKeyframe | null = null;
|
||||
for (const k of keyframes) {
|
||||
if (k.tMs <= tMs && (!before || k.tMs > before.tMs)) before = k;
|
||||
if (k.tMs >= tMs && (!after || k.tMs < after.tMs)) after = k;
|
||||
}
|
||||
if (before && after) return unionBox(before.box, after.box);
|
||||
if (before && tMs - before.tMs <= holdMs) return before.box;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Mean IoU of two tracks over the times both observe (nearest keyframe within 60 ms). */
|
||||
export function trackOverlap(a: BoxKeyframe[], b: BoxKeyframe[]): number {
|
||||
const scores: number[] = [];
|
||||
for (const k of a) {
|
||||
let nearest: BoxKeyframe | null = null;
|
||||
for (const other of b) {
|
||||
if (Math.abs(other.tMs - k.tMs) > 60) continue;
|
||||
if (!nearest || Math.abs(other.tMs - k.tMs) < Math.abs(nearest.tMs - k.tMs)) nearest = other;
|
||||
}
|
||||
if (nearest) scores.push(iou(k.box, nearest.box));
|
||||
}
|
||||
return scores.length === 0 ? 0 : scores.reduce((sum, s) => sum + s, 0) / scores.length;
|
||||
}
|
||||
|
||||
/** Source windows each clip of an asset shows. */
|
||||
export function clipWindows(clips: EdlVideoClip[]): TimeWindow[] {
|
||||
return clips.map((clip) => ({ startMs: clip.inMs, endMs: clip.outMs }));
|
||||
}
|
||||
|
||||
export function overlapsWindows(span: TimeWindow, windows: TimeWindow[]): boolean {
|
||||
return windows.some((w) => span.startMs <= w.endMs && w.startMs <= span.endMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inner part of a face box that a partial mode must alter. It sits inside the
|
||||
* band every provider covers (eyes 22-55 %, upper 0-55 %, lower 50-100 % of
|
||||
* the padded box), so the gate checks only pixels the mode promised to change.
|
||||
*/
|
||||
export function partialCheckBand(box: NormalizedBox, mode: PartialMode): NormalizedBox {
|
||||
const [top, bottom] =
|
||||
mode === "partial_eyes" ? [0.3, 0.48] : mode === "partial_upper" ? [0.05, 0.5] : [0.55, 0.95];
|
||||
return {
|
||||
x: box.x + box.w * 0.1,
|
||||
y: box.y + box.h * top,
|
||||
w: box.w * 0.8,
|
||||
h: box.h * (bottom - top),
|
||||
};
|
||||
}
|
||||
|
||||
export interface PixelBox {
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Static crop window in display pixels with the output aspect, as the
|
||||
* compiler sizes it for a reframe keyframe. Even sizes and offsets for yuv420p.
|
||||
*/
|
||||
export function cropPixels(
|
||||
geometry: { sourceWidth: number; sourceHeight: number; outWidth: number; outHeight: number },
|
||||
keyframe: { cx: number; cy: number; scale: number },
|
||||
): PixelBox {
|
||||
const outAspect = geometry.outWidth / geometry.outHeight;
|
||||
let w: number;
|
||||
let h: number;
|
||||
if (geometry.sourceWidth / geometry.sourceHeight > outAspect) {
|
||||
h = geometry.sourceHeight;
|
||||
w = h * outAspect;
|
||||
} else {
|
||||
w = geometry.sourceWidth;
|
||||
h = w / outAspect;
|
||||
}
|
||||
const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2);
|
||||
const even = (v: number) => Math.max(0, Math.floor(v / 2) * 2);
|
||||
const maxW = geometry.sourceWidth - (geometry.sourceWidth % 2);
|
||||
const maxH = geometry.sourceHeight - (geometry.sourceHeight % 2);
|
||||
const cw = Math.min(maxW, roundEven(w / keyframe.scale));
|
||||
const ch = Math.min(maxH, roundEven(h / keyframe.scale));
|
||||
const x = even(
|
||||
Math.min(Math.max(0, keyframe.cx * geometry.sourceWidth - cw / 2), geometry.sourceWidth - cw),
|
||||
);
|
||||
const y = even(
|
||||
Math.min(Math.max(0, keyframe.cy * geometry.sourceHeight - ch / 2), geometry.sourceHeight - ch),
|
||||
);
|
||||
return { x, y, w: cw, h: ch };
|
||||
}
|
||||
|
||||
/** Whether a normalized box lies completely outside a pixel crop. */
|
||||
export function outsideCrop(box: NormalizedBox, crop: PixelBox, width: number, height: number) {
|
||||
const x0 = box.x * width;
|
||||
const y0 = box.y * height;
|
||||
const x1 = (box.x + box.w) * width;
|
||||
const y1 = (box.y + box.h) * height;
|
||||
return x1 <= crop.x || x0 >= crop.x + crop.w || y1 <= crop.y || y0 >= crop.y + crop.h;
|
||||
}
|
||||
|
||||
/** Map a source window of one clip onto the output timeline (intro included), or null. */
|
||||
export function sourceToOutput(edl: Edl, clip: EdlVideoClip, span: TimeWindow): TimeWindow | null {
|
||||
const start = Math.max(span.startMs, clip.inMs);
|
||||
const end = Math.min(span.endMs, clip.outMs);
|
||||
if (end < start) return null;
|
||||
const toOutput = (t: number) =>
|
||||
Math.round(introMs(edl) + clip.atMs + (t - clip.inMs) / clip.speed);
|
||||
return { startMs: toOutput(start), endMs: toOutput(end) };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Privacy record (prepareSources → gate)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const PixelBoxSchema = z.object({
|
||||
x: z.number().int().min(0),
|
||||
y: z.number().int().min(0),
|
||||
w: z.number().int().min(1),
|
||||
h: z.number().int().min(1),
|
||||
});
|
||||
|
||||
const SpanSchema = { startMs: z.number(), endMs: z.number() };
|
||||
|
||||
/** One protected region the gate verifies pixel by pixel. */
|
||||
export const PrivacyCheckSchema = z.object({
|
||||
kind: z.enum(["face", "mark", "region"]),
|
||||
/** Track, mark, or region id. */
|
||||
id: z.string(),
|
||||
personId: z.string().nullable(),
|
||||
markId: z.string().nullable(),
|
||||
/** Boxes the treatment promised to alter, normalized to the base frame. */
|
||||
keyframes: z.array(BoxKeyframeSchema),
|
||||
/** Absolute mask directory (`<ms>.png` per frame) or null for box-only regions. */
|
||||
maskDir: z.string().nullable(),
|
||||
});
|
||||
export type PrivacyCheck = z.infer<typeof PrivacyCheckSchema>;
|
||||
|
||||
export const PrivacyRecordSchema = z.object({
|
||||
version: z.literal(1),
|
||||
assets: z.record(
|
||||
z.string(),
|
||||
z.object({
|
||||
/** Unredacted file the prepared source derives from (pixel reference). */
|
||||
basePath: z.string(),
|
||||
preparedPath: z.string(),
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
still: z.boolean(),
|
||||
/** Static crop applied for reframe_out, in base display pixels. */
|
||||
crop: PixelBoxSchema.nullable(),
|
||||
checks: z.array(PrivacyCheckSchema),
|
||||
}),
|
||||
),
|
||||
/** Detection gaps over MAX_TRACKING_GAP_MS per asset (source time), for review. */
|
||||
trackingGaps: z.array(
|
||||
z.object({ assetId: z.string(), personId: z.string().nullable(), ...SpanSchema }),
|
||||
),
|
||||
/** Person-bound marks re-found with low confidence, for review. */
|
||||
reviews: z.array(z.object({ markId: z.string(), assetId: z.string(), score: z.number() })),
|
||||
/** Faces that stay detectable by design (synthetic identity, partial modes), source time. */
|
||||
expectedFaces: z.array(
|
||||
z.object({
|
||||
assetId: z.string(),
|
||||
personId: z.string().nullable(),
|
||||
mode: AnonymizeModeSchema,
|
||||
...SpanSchema,
|
||||
}),
|
||||
),
|
||||
/** Faces in the clip ranges that belong to no person (consent cannot be shown). */
|
||||
unassignedFaces: z.array(z.object({ assetId: z.string(), ...SpanSchema })),
|
||||
aiAlterations: z.array(AiAlterationSchema),
|
||||
/** Effective protection per person (EDL target over person default). */
|
||||
protectedPeople: z.record(z.string(), AnonymizeModeSchema),
|
||||
/** People whose faces appear in the clip ranges. */
|
||||
visiblePeople: z.array(z.string()),
|
||||
protectUnassigned: z.boolean(),
|
||||
});
|
||||
export type PrivacyRecord = z.infer<typeof PrivacyRecordSchema>;
|
||||
|
||||
export async function writePrivacyRecord(workDir: string, record: PrivacyRecord): Promise<void> {
|
||||
await writeFile(path.join(workDir, PRIVACY_RECORD_FILE), JSON.stringify(record));
|
||||
}
|
||||
|
||||
/** The record prepareSources wrote, or null when it is missing. Malformed records throw. */
|
||||
export async function readPrivacyRecord(workDir: string): Promise<PrivacyRecord | null> {
|
||||
let text: string;
|
||||
try {
|
||||
text = await readFile(path.join(workDir, PRIVACY_RECORD_FILE), "utf8");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
||||
throw error;
|
||||
}
|
||||
return PrivacyRecordSchema.parse(JSON.parse(text));
|
||||
}
|
||||
|
||||
/** Stored face keyframe files: an array of keyframes or a tracking result with `keyframes`. */
|
||||
const StoredKeyframeSchema = BoxKeyframeSchema.extend({ confidence: z.number().optional() });
|
||||
const StoredKeyframesSchema = z.union([
|
||||
z.array(StoredKeyframeSchema.passthrough()),
|
||||
z.object({ keyframes: z.array(StoredKeyframeSchema.passthrough()) }).passthrough(),
|
||||
]);
|
||||
|
||||
export function parseStoredKeyframes(json: unknown): TrackKeyframe[] {
|
||||
const parsed = StoredKeyframesSchema.parse(json);
|
||||
const list = Array.isArray(parsed) ? parsed : parsed.keyframes;
|
||||
return list.map((k) => ({
|
||||
tMs: k.tMs,
|
||||
box: k.box,
|
||||
...(k.confidence !== undefined ? { confidence: k.confidence } : {}),
|
||||
}));
|
||||
}
|
||||
39
packages/adapters/src/media-privacy.ts
Normal file
39
packages/adapters/src/media-privacy.ts
Normal file
@ -0,0 +1,39 @@
|
||||
import { rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { prepareSources } from "./media-anonymize.js";
|
||||
import type { MediaPrivacyService, MediaServiceDeps } from "./media-context.js";
|
||||
import { runGate } from "./media-gate.js";
|
||||
import { createMark, updateMark } from "./media-marks.js";
|
||||
import type { PrivacyToolOptions } from "./media-privacy-io.js";
|
||||
import { resolveStored } from "./media-privacy-io.js";
|
||||
|
||||
/*
|
||||
* MediaPrivacyService: anonymization before rendering, the gate after it,
|
||||
* identifying marks, and deleting a person with their biometric data. Face
|
||||
* data never leaves this host: matching, tracking, and clustering run against
|
||||
* the local inference provider and database only.
|
||||
*/
|
||||
|
||||
export function createMediaPrivacyService(
|
||||
deps: MediaServiceDeps,
|
||||
tools: PrivacyToolOptions = {},
|
||||
): MediaPrivacyService {
|
||||
const withTools = { ...deps, ...tools };
|
||||
return {
|
||||
prepareSources: (input) => prepareSources(withTools, input),
|
||||
gate: (input) => runGate(withTools, input),
|
||||
createMark: (actor, input) => createMark(withTools, actor, input),
|
||||
updateMark: (actor, input) => updateMark(withTools, actor, input),
|
||||
async removePerson(actor, personId) {
|
||||
const removed = await deps.repos.removePersonCompletely(actor.spaceId, personId);
|
||||
// Rows are gone; now the files: track keyframes and each mark's directory
|
||||
// (masks plus previews). A path outside the media root throws untouched.
|
||||
const markDirs = removed.maskDirs.map((dir) =>
|
||||
path.basename(dir) === "masks" ? path.dirname(dir) : dir,
|
||||
);
|
||||
for (const stored of [...removed.keyframesPaths, ...markDirs]) {
|
||||
await rm(await resolveStored(deps.mediaRoot, stored), { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user