diff --git a/packages/adapters/src/media-anonymize.test.ts b/packages/adapters/src/media-anonymize.test.ts new file mode 100644 index 0000000..1a3604b --- /dev/null +++ b/packages/adapters/src/media-anonymize.test.ts @@ -0,0 +1,424 @@ +import { copyFile } from "node:fs/promises"; +import path from "node:path"; +import type { MediaInferenceProvider } from "@rakazo/adapter-kit"; +import type { AnonymizeMode, Edl } from "@rakazo/contracts"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { prepareSources } from "./media-anonymize.js"; +import { ffmpegAvailable } from "./media-emulator-clips.js"; +import { runGate } from "./media-gate.js"; +import type { PrivacyHarness } from "./media-privacy-fixtures.js"; +import { + ACTOR, + cleanProbe, + createPrivacyHarness, + testEdl, + testProfile, +} from "./media-privacy-fixtures.js"; +import { PrivacyUnavailableError } from "./media-privacy-io.js"; +import { extractLuma, residualVisibility } from "./media-privacy-pixels.js"; +import { readPrivacyRecord } from "./media-privacy-plan.js"; + +const hasFfmpeg = ffmpegAvailable(); +let harness: PrivacyHarness | null = null; + +afterEach(async () => { + await harness?.cleanup(); + harness = null; +}); + +/** One asset with one face of person "alice" (stored track), one clip over it. */ +async function setup( + mode: AnonymizeMode, + options: { + fileName?: string; + edl?: (assetId: string) => Edl; + inference?: (emulator: PrivacyHarness["inference"]) => MediaInferenceProvider | null; + } = {}, +) { + harness = await createPrivacyHarness(options.inference); + const asset = await harness.addAsset(options.fileName ?? "clip__faces-1____person-alice__.mp4"); + harness.addPerson("alice", mode); + await harness.analyseFaces(asset.id, ["alice"]); + await harness.learnPerson("alice", asset.id); + const edl = + options.edl?.(asset.id) ?? testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]); + const workDir = await harness.workDir(`render-${mode}`); + return { h: harness, asset, edl, workDir }; +} + +async function gateOn( + h: PrivacyHarness, + input: { edl: Edl; workDir: string; prepared: string; outputName?: string }, + profile = testProfile(), +) { + // The compiler is not under test: the prepared source stands in for the render output. + const outputPath = path.join(input.workDir, input.outputName ?? "output.mp4"); + await copyFile(input.prepared, outputPath); + return runGate(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl: input.edl, + profile, + outputPath, + ffprobe: cleanProbe(input.edl), + copy: { title: null, caption: null, tags: [] }, + bundleHasDisclosure: false, + autoFixRound: 0, + }); +} + +describe.skipIf(!hasFfmpeg)("prepareSources", () => { + it.each([ + "blur", + "pixelate", + "solid", + "sticker", + "partial_eyes", + "partial_upper", + "partial_lower", + ] as const)("applies %s to a protected person and the gate verifies it", async (mode) => { + const { h, asset, edl, workDir } = await setup(mode); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + const source = prepared.sources.get(asset.id); + expect(source?.path).toContain(`${path.sep}privacy${path.sep}`); + expect(source).toMatchObject({ width: 320, height: 180, rotation: 0, still: false }); + const record = await readPrivacyRecord(workDir); + expect(record?.protectedPeople).toEqual({ alice: mode }); + const check = record?.assets[asset.id]?.checks[0]; + expect(check).toMatchObject({ kind: "face", personId: "alice" }); + + // The protected box changed, the rest of the frame did not. + const base = await extractLuma(path.join(h.root, asset.storagePath!), { + tMs: 1000, + width: 320, + height: 180, + }); + const treated = await extractLuma(source!.path, { tMs: 1000, width: 320, height: 180 }); + const box = check!.keyframes.find((k) => k.tMs >= 1000)!.box; + expect(residualVisibility({ source: base, treated, box })).toBeLessThan(0.05); + expect( + residualVisibility({ source: base, treated, box: { x: 0.6, y: 0.6, w: 0.35, h: 0.35 } }), + ).toBeGreaterThan(0.95); + + const report = await gateOn(h, { edl, workDir, prepared: source!.path }); + expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]); + expect(report.passed).toBe(true); + }); + + it("crops a reframe_out face out of frame", async () => { + const { h, asset, workDir } = await setup("reframe_out"); + const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], { + width: 180, + height: 320, + }); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + const source = prepared.sources.get(asset.id)!; + expect(prepared.fallbacks).toEqual([]); + expect(source.width / source.height).toBeCloseTo(180 / 320, 1); + const record = await readPrivacyRecord(workDir); + expect(record?.assets[asset.id]?.crop).not.toBeNull(); + const report = await gateOn(h, { edl, workDir, prepared: source.path }); + expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]); + }); + + it("falls back from reframe_out to blur and says so when no crop avoids the face", async () => { + const { h, asset, edl, workDir } = await setup("none", { + edl: (assetId) => + testEdl([{ id: "c1", assetId, inMs: 0, outMs: 2000 }], { + redactions: { + faces: [ + { + id: "big", + target: { personId: "alice" }, + mode: "reframe_out", + paddingPct: 100, + featherPct: 0, + minSizePct: 50, + }, + ], + marks: [], + regions: [], + }, + }), + }); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + expect(prepared.fallbacks).toEqual([ + "person alice: no crop keeps the face out of frame; blurred instead", + ]); + const source = prepared.sources.get(asset.id)!; + expect(source.width).toBe(320); + const report = await gateOn(h, { edl, workDir, prepared: source.path }); + expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]); + }); + + it("blocks the export when tracking loses the face for more than 500 ms", async () => { + const { h, asset, edl, workDir } = await setup("blur", { + fileName: "clip__faces-1____person-alice____noface-gap-600-1400__.mp4", + }); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + const record = await readPrivacyRecord(workDir); + expect(record?.trackingGaps).toEqual([ + expect.objectContaining({ assetId: asset.id, personId: "alice" }), + ]); + const report = await gateOn(h, { + edl, + workDir, + prepared: prepared.sources.get(asset.id)!.path, + }); + expect(report.passed).toBe(false); + expect(report.issues).toContainEqual( + expect.objectContaining({ code: "tracking_gap", severity: "block", personId: "alice" }), + ); + }); + + it("falls back to blur when synthetic identity is unavailable and never fakes it", async () => { + const { h, asset, edl, workDir } = await setup("synthetic_identity"); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + expect(prepared.fallbacks).toEqual([ + "person alice: synthetic identity is not available; blurred instead", + ]); + expect(prepared.aiAlterations).not.toContain("synthetic_identity"); + const report = await gateOn(h, { + edl, + workDir, + prepared: prepared.sources.get(asset.id)!.path, + }); + expect(report.passed).toBe(true); + }); + + it("uses a licensed synthetic identity with a stable seed and no target image", async () => { + const synthesize = vi.fn(); + const { h, asset, edl, workDir } = await setup("synthetic_identity", { + inference: (emulator) => { + const provider = Object.create(emulator) as MediaInferenceProvider; + provider.capabilities = async () => { + const caps = await emulator.capabilities(); + caps.operations.synthesizeIdentity = { + available: true, + experimental: true, + model: "test-synth", + license: "Apache-2.0", + commercialUse: true, + }; + return caps; + }; + provider.synthesizeIdentity = async (request) => { + synthesize(request); + // Stand-in synthesis: a solid fill over the track (real models paint a face). + return emulator.redact({ + path: request.path, + outPath: request.outPath, + redactions: [ + { keyframes: request.track, mode: "solid", paddingPct: 20, featherPct: 0 }, + ], + }); + }; + return provider; + }, + }); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + expect(prepared.aiAlterations).toContain("synthetic_identity"); + expect(synthesize).toHaveBeenCalledTimes(1); + const request = synthesize.mock.calls[0]![0] as Record; + expect(Object.keys(request).sort()).toEqual(["outPath", "path", "seed", "signal", "track"]); + const seed = h.store.people[0]!.syntheticSeed; + expect(request.seed).toBe(seed); + expect(seed).not.toBeNull(); + const record = await readPrivacyRecord(workDir); + expect(record?.expectedFaces).toEqual([ + expect.objectContaining({ personId: "alice", mode: "synthetic_identity" }), + ]); + const report = await gateOn(h, { + edl, + workDir, + prepared: prepared.sources.get(asset.id)!.path, + }); + expect(report.passed).toBe(true); + }); + + it("fails closed without a media worker", async () => { + const { h, edl, workDir } = await setup("blur", { inference: () => null }); + await expect( + prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }), + ).rejects.toBeInstanceOf(PrivacyUnavailableError); + }); + + it("returns the untouched source when nothing needs protection", async () => { + const { h, asset, edl, workDir } = await setup("none"); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + expect(prepared.sources.get(asset.id)?.path).toBe(path.join(h.root, asset.storagePath!)); + expect(prepared.aiAlterations).toEqual([]); + }); + + it("blurs unassigned faces next to a protected person and records voice disguise", async () => { + const { h, asset, workDir } = await setup("blur", { + fileName: "clip__faces-2____person-alice__.mp4", + }); + // Track 2 stays unassigned (analysis stored it without a person). + h.store.tracks[1]!.personId = null; + const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], { + redactions: { faces: [], marks: [], regions: [], voice: { method: "pitch", semitones: 3 } }, + }); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + expect(prepared.aiAlterations).toEqual(["voice_disguise"]); + const record = await readPrivacyRecord(workDir); + expect(record?.protectUnassigned).toBe(true); + expect(record?.assets[asset.id]?.checks.map((c) => c.personId)).toEqual(["alice", null]); + const report = await gateOn(h, { + edl, + workDir, + prepared: prepared.sources.get(asset.id)!.path, + }); + expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]); + }); + + it("protects a photo and renders it from a redacted still", async () => { + const { h, asset, edl, workDir } = await setup("pixelate", { + fileName: "photo__faces-1____person-alice__.png", + }); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + const source = prepared.sources.get(asset.id)!; + expect(source).toMatchObject({ still: true, durationMs: 0, width: 320, height: 180 }); + expect(source.path.endsWith(".png")).toBe(true); + const record = await readPrivacyRecord(workDir); + const box = record!.assets[asset.id]!.checks[0]!.keyframes[0]!.box; + const base = await extractLuma(path.join(h.root, asset.storagePath!), { + tMs: 0, + width: 320, + height: 180, + still: true, + }); + const treated = await extractLuma(source.path, { + tMs: 0, + width: 320, + height: 180, + still: true, + }); + expect(residualVisibility({ source: base, treated, box })).toBeLessThan(0.05); + }); + + it("applies EDL regions and covers one that no crop can avoid", async () => { + const { h, asset, workDir } = await setup("none"); + const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], { + redactions: { + faces: [], + marks: [], + regions: [ + { + id: "plate", + clipId: "c1", + keyframes: [{ tMs: 0, box: { x: 0.7, y: 0.7, w: 0.2, h: 0.2 } }], + treatment: "blur", + }, + { + id: "everything", + clipId: "c1", + keyframes: [{ tMs: 0, box: { x: 0, y: 0, w: 1, h: 1 } }], + treatment: "reframe_out", + }, + ], + }, + }); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + expect(prepared.fallbacks).toEqual([ + "region everything: no crop keeps it out of frame; covered instead", + ]); + const record = await readPrivacyRecord(workDir); + expect(record?.assets[asset.id]?.checks.map((c) => c.id)).toEqual(["plate", "everything"]); + }); + + it("blocks when a provider claims to redact but leaves the face visible", async () => { + const { h, asset, edl, workDir } = await setup("blur", { + inference: (emulator) => { + const provider = Object.create(emulator) as MediaInferenceProvider; + provider.redact = async (request) => { + await copyFile(request.path, request.outPath); + return { outPath: request.outPath, framesProcessed: 50 }; + }; + return provider; + }, + }); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + const report = await gateOn(h, { + edl, + workDir, + prepared: prepared.sources.get(asset.id)!.path, + }); + expect(report.passed).toBe(false); + expect(report.issues).toContainEqual( + expect.objectContaining({ code: "face_not_covered", personId: "alice" }), + ); + }); +}); diff --git a/packages/adapters/src/media-anonymize.ts b/packages/adapters/src/media-anonymize.ts new file mode 100644 index 0000000..b335993 --- /dev/null +++ b/packages/adapters/src/media-anonymize.ts @@ -0,0 +1,928 @@ +import { createHash } from "node:crypto"; +import { mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import type { RedactionInstruction } from "@rakazo/adapter-kit"; +import type { + AiAlteration, + AnonymizeMode, + BoxKeyframe, + Edl, + EdlVideoClip, + MarkTreatment, + MediaCapabilities, + MediaRenderQuality, + NormalizedBox, + NormalizedPoint, +} from "@rakazo/contracts"; +import type { MediaRepos } from "@rakazo/db"; +import type { CompileSource } from "@rakazo/media"; +import { MAX_TRACKING_GAP_MS, reframeExcluding } from "@rakazo/media"; +import type { MediaActor, MediaPreparedSources, MediaServiceDeps } from "./media-context.js"; +import { + anchorFromFace, + faceAt, + firstConfident, + placeAnchor, + promptBox, + segmentMark, +} from "./media-marks.js"; +import type { PrivacyToolOptions, ProbedSource } from "./media-privacy-io.js"; +import { + analysisFile, + copyInto, + cropFile, + firstFrame, + inferenceCapabilities, + isInsideRoot, + maskKeyframes, + operationAvailable, + PrivacyUnavailableError, + probeSource, + readTrackKeyframes, + resolveAssetFile, + resolveStored, +} from "./media-privacy-io.js"; +import type { + CandidateTrack, + PixelBox, + PrivacyCheck, + PrivacyRecord, + Protection, + TimeWindow, +} from "./media-privacy-plan.js"; +import { + boxAt, + clipWindows, + cropPixels, + effectiveModes, + FACE_TRACK_FPS, + iou, + isPartial, + MARK_PADDING_PCT, + MARK_QUALITY_MIN, + MARK_REVIEW_MIN_SCORE, + matchPerson, + outsideCrop, + overlapsWindows, + PRIVACY_RECORD_FILE, + padBox, + partialCheckBand, + protectKeyframes, + resolveProtection, + TRACK_MATCH_MIN_IOU, + trackOverlap, + unionBox, + writePrivacyRecord, +} from "./media-privacy-plan.js"; + +/* + * prepareSources (spec 7): resolve every clip's file and apply all face, mark, + * and region protection before the compiler sees it. The compiler applies no + * redactions, so anything this function cannot honor throws + * (PrivacyUnavailableError) instead of returning an unredacted source. + * + * Per asset, in this order: synthetic identities, inpainted marks, one + * redaction pass (blur, pixelate, solid, sticker, partial bands, covers) by the + * inference provider, then a static crop for reframe_out. Faces are found from + * the stored tracks plus a fresh tracking pass; tracking gaps over 500 ms are + * covered by the grown last box and recorded so the gate blocks them. Voice + * disguise is applied by the compiler on the program audio and only recorded + * here. Everything the gate must verify goes into privacy.json in workDir. + */ + +export type AnonymizeDeps = Pick< + MediaServiceDeps, + "repos" | "inference" | "mediaRoot" | "sources" +> & + PrivacyToolOptions; + +type AssetRow = Awaited>; +type PersonRow = Awaited>[number]; + +/** reframe_out boxes are unioned per bucket of this length so the crop search stays small. */ +const REFRAME_BUCKET_MS = 1000; +/** Landmark samples per partial-mode track. */ +const MAX_LANDMARK_SAMPLES = 120; + +interface AssetPlan { + asset: AssetRow; + clips: EdlVideoClip[]; + windows: TimeWindow[]; + base: string; + probe: ProbedSource; + dir: string; + candidates: CandidateTrack[]; + instructions: RedactionInstruction[]; + checks: PrivacyCheck[]; + synthetic: Array<{ personId: string | null; keyframes: BoxKeyframe[]; seed: number }>; + inpaint: Array<{ markId: string; maskDir: string; keyframes: BoxKeyframe[] }>; + reframe: Array<{ + check: PrivacyCheck; + /** Boxes the crop must exclude. */ + boxes: Array<{ tMs: number; box: NormalizedBox }>; + /** Applied when no crop can exclude the boxes. */ + fallback: () => void; + }>; +} + +export interface PrepareInput { + actor: MediaActor; + projectId: string; + edl: Edl; + quality: MediaRenderQuality; + workDir: string; + signal?: AbortSignal; +} + +export async function prepareSources( + deps: AnonymizeDeps, + input: PrepareInput, +): Promise { + const { actor, edl, signal } = input; + const spaceId = actor.spaceId; + const workDir = await resolveStored(deps.mediaRoot, input.workDir); + await mkdir(workDir, { recursive: true }); + const privacyDir = path.join(workDir, "privacy"); + await rm(privacyDir, { recursive: true, force: true }); + // A record from an earlier attempt must never vouch for this one. + await rm(path.join(workDir, PRIVACY_RECORD_FILE), { force: true }); + + const clipsByAsset = new Map(); + for (const clip of edl.tracks.video) { + clipsByAsset.set(clip.assetId, [...(clipsByAsset.get(clip.assetId) ?? []), clip]); + } + const assetIds = [...clipsByAsset.keys()].sort(); + const assets = await deps.repos.listAssetRecords(spaceId, assetIds); + if (assets.length !== assetIds.length) + throw new PrivacyUnavailableError("a clip asset is missing"); + + const project = await deps.repos.getProjectRecord(spaceId, input.projectId); + const kit = project.brandKitId + ? (await deps.repos.getBrandKit(spaceId, project.brandKitId)).kit + : null; + const people = await deps.repos.listPersonRecords(spaceId); + const peopleById = new Map(people.map((person) => [person.id, person])); + const defaults = new Map( + people.map((person) => [person.id, person.anonymizeByDefault as AnonymizeMode]), + ); + const modes = effectiveModes(edl.redactions.faces, defaults); + const faceWork = + edl.redactions.faces.length > 0 || [...modes.values()].some((mode) => mode !== "none"); + const capabilities = await inferenceCapabilities(deps.inference, signal); + const stored = await deps.repos.faceTrackRecords(spaceId, assetIds); + const centroids = faceWork ? await deps.repos.personCentroids(spaceId) : []; + + const fallbacks: string[] = []; + const alterations = new Set(); + if (edl.redactions.voice) alterations.add("voice_disguise"); + const record: PrivacyRecord = { + version: 1, + assets: {}, + trackingGaps: [], + reviews: [], + expectedFaces: [], + unassignedFaces: [], + aiAlterations: [], + protectedPeople: {}, + visiblePeople: [], + protectUnassigned: false, + }; + + // 1. Files and face candidates per asset. + const plans: AssetPlan[] = []; + for (const asset of assets) { + signal?.throwIfAborted(); + const clips = clipsByAsset.get(asset.id) ?? []; + const base = await resolveAssetFile(deps, spaceId, asset, input.quality); + const probe = await probeSource(base, asset.kind, { ...deps, signal }); + const windows = probe.still ? [{ startMs: 0, endMs: 0 }] : clipWindows(clips); + const plan: AssetPlan = { + asset, + clips, + windows, + base, + probe, + dir: path.join(privacyDir, asset.id), + candidates: [], + instructions: [], + checks: [], + synthetic: [], + inpaint: [], + reframe: [], + }; + const storedTracks = stored.filter((track) => track.assetId === asset.id); + const fromStore: CandidateTrack[] = []; + for (const track of storedTracks) { + fromStore.push({ + id: track.id, + storedId: track.id, + personId: track.personId, + keyframes: await readTrackKeyframes(deps.mediaRoot, track.keyframesPath), + }); + } + if (faceWork) { + if (deps.inference && operationAvailable(capabilities, "trackFaces")) { + const file = (await analysisFile(deps, spaceId, asset)) ?? (await workingCopy(deps, plan)); + const fresh = await deps.inference.trackFaces({ + path: file, + sampleFps: FACE_TRACK_FPS, + signal, + }); + plan.candidates = mergeTracks(asset.id, fromStore, fresh.tracks, centroids); + } else { + const analysed = await deps.repos.latestAnalysis(spaceId, asset.id, "faces"); + if (!analysed) { + throw new PrivacyUnavailableError( + `faces of asset ${asset.id} were never detected and the media worker is unavailable`, + ); + } + plan.candidates = fromStore; + } + } else { + plan.candidates = fromStore; + } + plan.candidates = plan.candidates.filter((track) => visibleIn(track, plan)); + plans.push(plan); + } + + // 2. Who is visible and who is protected. + const visible = new Set(); + for (const plan of plans) { + for (const track of plan.candidates) { + if (track.personId) visible.add(track.personId); + else { + const span = spanOf(track); + if (span) record.unassignedFaces.push({ assetId: plan.asset.id, ...span }); + } + } + } + record.visiblePeople = [...visible].sort(); + for (const [personId, mode] of modes) { + if (mode !== "none" && visible.has(personId)) record.protectedPeople[personId] = mode; + } + record.protectUnassigned = Object.keys(record.protectedPeople).length > 0; + const colors = { + fill: kit?.colors.primary, + sticker: kit?.colors.accent ?? kit?.colors.primary, + }; + + // 3. Faces. + const syntheticReady = operationAvailable(capabilities, "synthesizeIdentity"); + for (const plan of plans) { + for (const track of plan.candidates) { + const protection = resolveProtection(track, { + faces: edl.redactions.faces, + defaults, + protectUnassigned: record.protectUnassigned, + }); + if (!protection) continue; + const protectedTrack = protectKeyframes(track.keyframes, plan.windows, protection.minSizePct); + const keyframes = plan.probe.still + ? stillKeyframes(protectedTrack.keyframes) + : protectedTrack.keyframes; + if (keyframes.length === 0) continue; + if (!plan.probe.still) { + for (const gap of protectedTrack.gaps) { + record.trackingGaps.push({ assetId: plan.asset.id, personId: track.personId, ...gap }); + } + } + await planFace(deps, plan, { + track, + protection, + keyframes, + person: track.personId ? (peopleById.get(track.personId) ?? null) : null, + syntheticReady, + capabilities, + colors, + fallbacks, + record, + signal, + }); + } + } + + // 4. Marks (asset marks and the standing marks of people seen in the asset). + const edlMarks = new Map(edl.redactions.marks.map((mark) => [mark.markId, mark.treatment])); + for (const plan of plans) { + const marks = await deps.repos.marksForAsset(spaceId, plan.asset.id); + for (const mark of marks) { + const treatment: MarkTreatment = edlMarks.get(mark.id) ?? mark.treatment; + if (treatment === "none") continue; + const located = await locateMark(deps, plan, mark, signal); + if (!located) continue; + if (located.review) { + record.reviews.push({ markId: mark.id, assetId: plan.asset.id, score: located.score }); + } + const keyframes = plan.probe.still + ? stillKeyframes(located.keyframes) + : located.keyframes.filter((k) => nearWindows(k.tMs, plan.windows)); + if (keyframes.length === 0) continue; + const check: PrivacyCheck = { + kind: "mark", + id: mark.id, + personId: mark.personId, + markId: mark.id, + keyframes, + maskDir: located.maskDir, + }; + const cover = () => { + plan.instructions.push({ + keyframes, + mode: "cover", + paddingPct: MARK_PADDING_PCT, + featherPct: 0, + maskDir: located.maskDir, + ...(colors.fill ? { color: colors.fill } : {}), + }); + plan.checks.push(check); + }; + if (treatment === "blur") { + plan.instructions.push({ + keyframes, + mode: "blur", + paddingPct: MARK_PADDING_PCT, + featherPct: 0, + maskDir: located.maskDir, + }); + plan.checks.push(check); + } else if (treatment === "cover") { + cover(); + } else if (treatment === "reframe_out") { + plan.reframe.push({ + check, + boxes: keyframes.map((k) => ({ tMs: k.tMs, box: padBox(k.box, MARK_PADDING_PCT) })), + fallback: () => { + fallbacks.push(`mark ${mark.id}: no crop keeps it out of frame; covered instead`); + cover(); + }, + }); + } else if (operationAvailable(capabilities, "inpaintVideo")) { + plan.inpaint.push({ markId: mark.id, maskDir: located.maskDir, keyframes }); + plan.checks.push(check); + } else { + fallbacks.push(`mark ${mark.id}: inpainting is unavailable; covered instead`); + cover(); + } + } + } + + // 5. Regions drawn in the EDL. + for (const region of edl.redactions.regions) { + const clip = edl.tracks.video.find((candidate) => candidate.id === region.clipId); + const plan = plans.find((candidate) => candidate.asset.id === clip?.assetId); + if (!clip || !plan) throw new PrivacyUnavailableError(`region ${region.id} has no clip`); + const keyframes = plan.probe.still ? stillKeyframes(region.keyframes) : region.keyframes; + const check: PrivacyCheck = { + kind: "region", + id: region.id, + personId: null, + markId: null, + keyframes, + maskDir: null, + }; + const apply = (mode: "blur" | "cover") => { + plan.instructions.push({ + keyframes, + mode, + paddingPct: 0, + featherPct: 0, + ...(mode === "cover" && colors.fill ? { color: colors.fill } : {}), + }); + plan.checks.push(check); + }; + if (region.treatment === "reframe_out") { + plan.reframe.push({ + check, + boxes: keyframes, + fallback: () => { + fallbacks.push(`region ${region.id}: no crop keeps it out of frame; covered instead`); + apply("cover"); + }, + }); + } else { + apply(region.treatment); + } + } + + // 6. Apply per asset. + const sources = new Map(); + for (const plan of plans) { + signal?.throwIfAborted(); + const crop = chooseCrop(plan, edl); + const hasWork = + plan.synthetic.length > 0 || + plan.inpaint.length > 0 || + plan.instructions.length > 0 || + crop !== null; + if (!hasWork) { + sources.set(plan.asset.id, compileSource(plan.base, plan.probe, plan.probe.rotation)); + continue; + } + const prepared = await applyPlan(deps, plan, crop, { + capabilities, + alterations, + fallbacks, + signal, + }); + const probed = await probeSource(prepared, plan.asset.kind, { ...deps, signal }); + verifyPrepared(plan, crop, probed); + sources.set(plan.asset.id, compileSource(prepared, probed, 0)); + record.assets[plan.asset.id] = { + basePath: plan.base, + preparedPath: prepared, + width: plan.probe.width, + height: plan.probe.height, + still: plan.probe.still, + crop, + checks: plan.checks, + }; + } + + record.aiAlterations = [...alterations].sort(); + await writePrivacyRecord(workDir, record); + return { sources, aiAlterations: record.aiAlterations, fallbacks }; +} + +// --------------------------------------------------------------------------- +// Faces +// --------------------------------------------------------------------------- + +/** + * Fresh tracks take over stored ones they overlap (keeping the stored id and + * person); unmatched fresh tracks are matched to people by embedding; stored + * tracks the fresh pass missed stay (fail closed). + */ +export function mergeTracks( + assetId: string, + stored: CandidateTrack[], + fresh: Array<{ + keyframes: Array; + embedding: number[] | null; + }>, + centroids: Array<{ personId: string; centroid: Uint8Array }>, +): CandidateTrack[] { + const taken = new Set(); + const merged: CandidateTrack[] = []; + fresh.forEach((track, index) => { + let best: { candidate: CandidateTrack; score: number } | null = null; + for (const candidate of stored) { + if (taken.has(candidate.id)) continue; + const score = trackOverlap(track.keyframes, candidate.keyframes); + if (score >= TRACK_MATCH_MIN_IOU && (!best || score > best.score)) + best = { candidate, score }; + } + if (best) { + taken.add(best.candidate.id); + merged.push({ + ...best.candidate, + personId: best.candidate.personId ?? matchPerson(track.embedding, centroids), + keyframes: [...track.keyframes, ...best.candidate.keyframes].sort((a, b) => a.tMs - b.tMs), + }); + return; + } + merged.push({ + id: `fresh-${assetId}-${index + 1}`, + storedId: null, + personId: matchPerson(track.embedding, centroids), + keyframes: track.keyframes, + }); + }); + for (const candidate of stored) if (!taken.has(candidate.id)) merged.push(candidate); + return merged; +} + +function nearWindows(tMs: number, windows: TimeWindow[]): boolean { + return windows.some( + (w) => tMs >= w.startMs - MAX_TRACKING_GAP_MS && tMs <= w.endMs + MAX_TRACKING_GAP_MS, + ); +} + +function spanOf(track: CandidateTrack): TimeWindow | null { + if (track.keyframes.length === 0) return null; + const times = track.keyframes.map((k) => k.tMs); + return { startMs: Math.min(...times), endMs: Math.max(...times) }; +} + +function visibleIn(track: CandidateTrack, plan: AssetPlan): boolean { + if (plan.probe.still) return track.keyframes.length > 0; + const span = spanOf(track); + return span !== null && overlapsWindows(span, plan.windows); +} + +/** A still shows one moment: every box at once, held over the whole image. */ +function stillKeyframes(keyframes: BoxKeyframe[]): BoxKeyframe[] { + const first = keyframes[0]; + if (!first) return []; + const box = keyframes.reduce((acc, k) => unionBox(acc, k.box), first.box); + return [ + { tMs: 0, box }, + { tMs: 1000, box }, + ]; +} + +/** Stable per-person seed for a synthetic identity (stored on first use). */ +export function syntheticSeedFor(id: string): number { + return createHash("sha256").update(`synthetic:${id}`).digest().readInt32LE(0); +} + +async function planFace( + deps: AnonymizeDeps, + plan: AssetPlan, + input: { + track: CandidateTrack; + protection: Protection; + keyframes: BoxKeyframe[]; + person: PersonRow | null; + syntheticReady: boolean; + capabilities: MediaCapabilities | null; + colors: { fill?: string; sticker?: string }; + fallbacks: string[]; + record: PrivacyRecord; + signal?: AbortSignal; + }, +): Promise { + const { track, protection, keyframes } = input; + const who = track.personId ? `person ${track.personId}` : `face ${track.id}`; + let mode = protection.mode; + const padded = keyframes.map((k) => ({ tMs: k.tMs, box: padBox(k.box, protection.paddingPct) })); + const check: PrivacyCheck = { + kind: "face", + id: track.id, + personId: track.personId, + markId: null, + keyframes: padded, + maskDir: null, + }; + const span = spanOf({ ...track, keyframes }); + if (mode === "synthetic_identity") { + if (input.syntheticReady) { + let seed: number; + if (input.person) { + seed = input.person.syntheticSeed ?? syntheticSeedFor(input.person.id); + if (input.person.syntheticSeed === null) { + await deps.repos.updatePerson(plan.asset.spaceId, input.person.id, { + syntheticSeed: seed, + }); + input.person.syntheticSeed = seed; + } + } else { + seed = syntheticSeedFor(track.id); + } + plan.synthetic.push({ personId: track.personId, keyframes, seed }); + plan.checks.push(check); + if (span) { + input.record.expectedFaces.push({ + assetId: plan.asset.id, + personId: track.personId, + mode, + ...span, + }); + } + return; + } + input.fallbacks.push(`${who}: synthetic identity is not available; blurred instead`); + mode = "blur"; + } + if (mode === "reframe_out") { + plan.reframe.push({ + check, + boxes: padded, + fallback: () => { + input.fallbacks.push(`${who}: no crop keeps the face out of frame; blurred instead`); + plan.instructions.push({ + keyframes, + mode: "blur", + paddingPct: protection.paddingPct, + featherPct: protection.featherPct, + }); + plan.checks.push(check); + }, + }); + return; + } + const instruction: RedactionInstruction = { + keyframes, + mode, + paddingPct: protection.paddingPct, + featherPct: protection.featherPct, + }; + if (mode === "solid" && input.colors.fill) instruction.color = input.colors.fill; + if (mode === "sticker" && input.colors.sticker) instruction.color = input.colors.sticker; + if (isPartial(mode)) { + const partial = mode; + const landmarks = await landmarksFor(deps, plan, keyframes, input.capabilities, input.signal); + if (landmarks.length > 0) instruction.landmarks = landmarks; + check.keyframes = padded.map((k) => ({ tMs: k.tMs, box: partialCheckBand(k.box, partial) })); + if (span) { + input.record.expectedFaces.push({ + assetId: plan.asset.id, + personId: track.personId, + mode, + ...span, + }); + } + } + plan.instructions.push(instruction); + plan.checks.push(check); +} + +/** Five-point landmarks per sampled keyframe for partial modes, when the provider has them. */ +async function landmarksFor( + deps: AnonymizeDeps, + plan: AssetPlan, + keyframes: BoxKeyframe[], + capabilities: MediaCapabilities | null, + signal?: AbortSignal, +): Promise> { + if (!deps.inference || !operationAvailable(capabilities, "faceLandmarks")) return []; + const step = Math.max(1, Math.ceil(keyframes.length / MAX_LANDMARK_SAMPLES)); + const timesMs = keyframes.filter((_, i) => i % step === 0).map((k) => k.tMs); + const file = + (await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan)); + const result = await deps.inference.faceLandmarks({ path: file, timesMs, signal }); + const out: Array<{ tMs: number; points: NormalizedPoint[] }> = []; + for (const frame of result.frames) { + const expected = boxAt(keyframes, frame.tMs); + if (!expected) continue; + const best = frame.faces + .map((face) => ({ face, score: iou(face.box, expected) })) + .sort((a, b) => b.score - a.score)[0]; + if (best && best.score > 0.2) out.push({ tMs: frame.tMs, points: best.face.landmarks }); + } + return out; +} + +// --------------------------------------------------------------------------- +// Marks +// --------------------------------------------------------------------------- + +type MarkRow = Awaited>[number]; + +/** + * Mask and keyframes of a mark in this asset. Marks drawn on the asset use + * their stored mask (segmenting now when none exists); standing person marks + * are re-prompted at the same place relative to the person's face. Low + * segmentation confidence sends the mark to review. + */ +async function locateMark( + deps: AnonymizeDeps, + plan: AssetPlan, + mark: MarkRow, + signal?: AbortSignal, +): Promise<{ maskDir: string; keyframes: BoxKeyframe[]; score: number; review: boolean } | null> { + const outDir = path.join(plan.dir, `mark-${mark.id}`); + if (mark.assetId === plan.asset.id) { + if (mark.maskDir) { + const maskDir = await resolveStored(deps.mediaRoot, mark.maskDir); + return { maskDir, keyframes: await maskKeyframes(maskDir), score: 1, review: false }; + } + if (!mark.prompt) throw new PrivacyUnavailableError(`mark ${mark.id} has no prompt`); + const file = + (await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan)); + const segmented = await segmentMark(deps, { file, prompt: mark.prompt, outDir, signal }); + return { + maskDir: segmented.maskDir, + keyframes: segmented.keyframes, + score: segmented.score, + review: false, + }; + } + // A standing mark of a person seen in this asset. + if (!mark.personId) return null; + const tracks = plan.candidates.filter((track) => track.personId === mark.personId); + if (tracks.length === 0) return null; + const anchorFace = await originFace(deps, plan.asset.spaceId, mark); + const target = tracks + .map((track) => + firstConfident(track.keyframes, Math.min(...plan.windows.map((w) => w.startMs))), + ) + .filter((k) => k !== null) + .sort((a, b) => a.tMs - b.tMs)[0]; + const placed = + anchorFace && target && mark.prompt + ? placeAnchor(anchorFromFace(promptBox(mark.prompt), anchorFace), target.box) + : null; + if (!placed || !target) { + // Cannot relate the mark to the face here: cover the person's face region and review. + const keyframes = tracks.flatMap((track) => + track.keyframes.map((k) => ({ tMs: k.tMs, box: k.box })), + ); + return { maskDir: await emptyMaskDir(outDir), keyframes, score: 0, review: true }; + } + const file = + (await analysisFile(deps, plan.asset.spaceId, plan.asset)) ?? (await workingCopy(deps, plan)); + const segmented = await segmentMark(deps, { + file, + prompt: { tMs: target.tMs, box: placed }, + outDir, + signal, + }); + return { + maskDir: segmented.maskDir, + keyframes: segmented.keyframes, + score: segmented.score, + review: segmented.score < MARK_REVIEW_MIN_SCORE, + }; +} + +/** The person's face in the frame a standing mark was drawn on. */ +async function originFace( + deps: AnonymizeDeps, + spaceId: string, + mark: MarkRow, +): Promise { + if (!mark.assetId || !mark.prompt || !mark.personId) return null; + const tracks = (await deps.repos.faceTrackRecords(spaceId, [mark.assetId])).filter( + (t) => t.personId === mark.personId, + ); + for (const track of tracks) { + const face = faceAt( + await readTrackKeyframes(deps.mediaRoot, track.keyframesPath), + mark.prompt.tMs, + ); + if (face) return face; + } + return null; +} + +async function emptyMaskDir(dir: string): Promise { + await mkdir(dir, { recursive: true }); + return dir; +} + +// --------------------------------------------------------------------------- +// Applying +// --------------------------------------------------------------------------- + +/** In-root copy of the base file for providers (once per asset). */ +async function workingCopy(deps: AnonymizeDeps, plan: AssetPlan): Promise { + if (!(await isInsideRoot(deps.mediaRoot, plan.base))) + plan.base = await copyInto(plan.base, plan.dir); + return plan.base; +} + +/** Static crop that keeps every reframe_out box out of frame; applies fallbacks otherwise. */ +function chooseCrop(plan: AssetPlan, edl: Edl): PixelBox | null { + if (plan.reframe.length === 0) return null; + const boxes = bucketBoxes(plan.reframe.flatMap((item) => item.boxes)); + const geometry = { + sourceWidth: plan.probe.width, + sourceHeight: plan.probe.height, + outWidth: edl.output.width, + outHeight: edl.output.height, + durationMs: plan.probe.durationMs, + }; + const keyframe = reframeExcluding(boxes, geometry)?.[0]; + const crop = keyframe ? cropPixels(geometry, keyframe) : null; + if ( + crop && + boxes.every(({ box }) => outsideCrop(box, crop, plan.probe.width, plan.probe.height)) + ) { + for (const item of plan.reframe) plan.checks.push(item.check); + return crop; + } + for (const item of plan.reframe) item.fallback(); + return null; +} + +/** Union boxes per second so the crop search stays small; unions only widen the excluded area. */ +function bucketBoxes(boxes: Array<{ tMs: number; box: NormalizedBox }>) { + const buckets = new Map(); + for (const { tMs, box } of boxes) { + const key = Math.floor(tMs / REFRAME_BUCKET_MS); + const current = buckets.get(key); + buckets.set(key, current ? unionBox(current, box) : box); + } + return [...buckets.entries()].map(([key, box]) => ({ tMs: key * REFRAME_BUCKET_MS, box })); +} + +function stepPath(plan: AssetPlan, file: string, step: string, ext = ".mp4"): string { + const stem = path + .basename(file, path.extname(file)) + .replace(/\.(?:synth|inpaint|redacted|cropped)[-\w]*$/, ""); + return path.join(plan.dir, `${stem}.${step}${ext}`); +} + +async function applyPlan( + deps: AnonymizeDeps, + plan: AssetPlan, + crop: PixelBox | null, + context: { + capabilities: MediaCapabilities | null; + alterations: Set; + fallbacks: string[]; + signal?: AbortSignal; + }, +): Promise { + const { signal } = context; + await mkdir(plan.dir, { recursive: true }); + const needsProvider = plan.synthetic.length + plan.inpaint.length + plan.instructions.length > 0; + if (needsProvider && !deps.inference) { + throw new PrivacyUnavailableError("redaction needs the media worker, which is not configured"); + } + let current = needsProvider ? await workingCopy(deps, plan) : plan.base; + const inference = deps.inference; + if (inference) { + for (const [index, synthetic] of plan.synthetic.entries()) { + const outPath = stepPath(plan, current, `synth-${index + 1}`); + current = ( + await inference.synthesizeIdentity({ + path: current, + track: synthetic.keyframes, + seed: synthetic.seed, + outPath, + signal, + }) + ).outPath; + context.alterations.add("synthetic_identity"); + } + for (const mark of plan.inpaint) { + const outPath = stepPath(plan, current, `inpaint-${mark.markId}`); + const result = await inference.inpaintVideo({ + path: current, + maskDir: mark.maskDir, + outPath, + signal, + }); + if (result.quality.score >= MARK_QUALITY_MIN) { + current = result.outPath; + context.alterations.add("inpaint"); + continue; + } + // Never ship artifacts silently: low quality falls back to a cover. + context.fallbacks.push( + `mark ${mark.markId}: inpainting scored ${result.quality.score.toFixed(2)}; covered instead`, + ); + await rm(result.outPath, { force: true }); + plan.instructions.push({ + keyframes: mark.keyframes, + mode: "cover", + paddingPct: MARK_PADDING_PCT, + featherPct: 0, + maskDir: mark.maskDir, + }); + } + if (plan.instructions.length > 0) { + if (!operationAvailable(context.capabilities, "redact")) { + throw new PrivacyUnavailableError("the media worker cannot redact"); + } + const outPath = stepPath(plan, current, "redacted"); + const result = await inference.redact({ + path: current, + redactions: plan.instructions, + outPath, + signal, + }); + if (result.framesProcessed <= 0) + throw new PrivacyUnavailableError("redaction processed no frames"); + current = result.outPath; + } + } + if (crop) { + const outPath = stepPath(plan, current, "cropped", plan.probe.still ? ".png" : ".mp4"); + await cropFile(current, outPath, crop, plan.probe.still, { ...deps, signal }); + current = outPath; + } + if (plan.probe.still && path.extname(current).toLowerCase() !== ".png" && current !== plan.base) { + const outPath = stepPath(plan, current, "still", ".png"); + await firstFrame(current, outPath, { ...deps, signal }); + current = outPath; + } + return current; +} + +/** The prepared file must show the whole clip range at the expected size. */ +function verifyPrepared(plan: AssetPlan, crop: PixelBox | null, probed: ProbedSource): void { + const width = crop ? crop.w : plan.probe.width; + const height = crop ? crop.h : plan.probe.height; + if (probed.width !== width || probed.height !== height) { + throw new PrivacyUnavailableError( + `redacted asset ${plan.asset.id} is ${probed.width}x${probed.height}, expected ${width}x${height}`, + ); + } + if (plan.probe.still) return; + const neededMs = Math.max(...plan.clips.map((clip) => clip.outMs)); + const frameMs = 1000 / Math.max(1, plan.probe.fps); + if (probed.durationMs + 2 * frameMs < Math.min(neededMs, plan.probe.durationMs)) { + throw new PrivacyUnavailableError(`redacted asset ${plan.asset.id} is shorter than its clips`); + } +} + +function compileSource( + file: string, + probe: ProbedSource, + rotation: CompileSource["rotation"], +): CompileSource { + const quarter = rotation === 90 || rotation === 270; + return { + path: file, + // CompileSource wants the coded size; probe reports the display size. + width: quarter ? probe.height : probe.width, + height: quarter ? probe.width : probe.height, + durationMs: probe.durationMs, + fps: probe.fps, + hasAudio: probe.hasAudio, + rotation, + still: probe.still, + }; +} diff --git a/packages/adapters/src/media-consent.test.ts b/packages/adapters/src/media-consent.test.ts new file mode 100644 index 0000000..1261e87 --- /dev/null +++ b/packages/adapters/src/media-consent.test.ts @@ -0,0 +1,97 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + assertExportConsent, + ConsentRequiredError, + consentIssues, + consentRequirements, + enforceConsentFlags, + participantsOf, +} from "./media-consent.js"; +import type { PrivacyHarness } from "./media-privacy-fixtures.js"; +import { ACTOR, createPrivacyHarness, testEdl, testProfile } from "./media-privacy-fixtures.js"; + +const now = new Date("2026-06-01T12:00:00Z"); +const past = new Date("2026-01-01T00:00:00Z"); + +let harness: PrivacyHarness | null = null; +afterEach(async () => { + await harness?.cleanup(); + harness = null; +}); + +describe("consent rules", () => { + it("always requires consent and age proof for paid video", () => { + expect(consentRequirements(testProfile())).toEqual({ consent: false, age: false }); + expect(consentRequirements(testProfile({ kind: "paid_video" }))).toEqual({ + consent: true, + age: true, + }); + expect(enforceConsentFlags(testProfile({ kind: "paid_video" })).consent).toEqual({ + requirePerformerConsent: true, + requireAgeVerification: true, + }); + }); + + it("accepts only recorded, past proof with a verification method", () => { + const paid = testProfile({ kind: "paid_video" }); + const ok = { + id: "a", + consentRecordedAt: past, + ageVerifiedAt: past, + ageVerificationMethod: "platform_verified", + }; + expect(consentIssues(paid, [ok], [], now)).toEqual([]); + const future = { ...ok, consentRecordedAt: new Date("2027-01-01T00:00:00Z") }; + const noMethod = { ...ok, id: "b", ageVerificationMethod: "none" }; + expect(consentIssues(paid, [future, noMethod], [{ atMs: 10 }], now).map((i) => i.code)).toEqual( + ["consent_missing", "age_unverified", "unknown_performer"], + ); + expect(consentIssues(testProfile(), [noMethod], [{}], now)).toEqual([]); + }); + + it("collects the people and unassigned faces inside the clip ranges", () => { + const edl = testEdl([{ id: "c1", assetId: "a1", inMs: 1000, outMs: 2000 }]); + expect( + participantsOf(edl, [ + { assetId: "a1", personId: "alice", startMs: 0, endMs: 1200 }, + { assetId: "a1", personId: "bob", startMs: 2500, endMs: 3000 }, + { assetId: "a1", personId: null, startMs: 1500, endMs: 1600 }, + { assetId: "a2", personId: "carol", startMs: 0, endMs: 5000 }, + ]), + ).toEqual({ + personIds: ["alice"], + unassigned: [{ assetId: "a1", startMs: 1500, endMs: 1600 }], + }); + }); +}); + +describe("export consent", () => { + it("blocks a paid export until every participant has consent and age proof", async () => { + harness = await createPrivacyHarness(() => null); + const alice = harness.addPerson("alice", "none"); + harness.store.tracks.push({ + id: "t1", + assetId: "a1", + personId: "alice", + startMs: 0, + endMs: 1000, + confidenceMin: 0.9, + keyframesPath: "x.json", + modelVersion: "v1", + }); + const edl = testEdl([{ id: "c1", assetId: "a1", inMs: 0, outMs: 1000 }]); + const paid = testProfile({ kind: "paid_video" }); + await expect(assertExportConsent(harness.deps, ACTOR, { edl, profile: paid })).rejects.toThrow( + ConsentRequiredError, + ); + await expect( + assertExportConsent(harness.deps, ACTOR, { edl, profile: testProfile() }), + ).resolves.toBeUndefined(); + alice.consentRecordedAt = past; + alice.ageVerifiedAt = past; + alice.ageVerificationMethod = "document_on_file"; + await expect( + assertExportConsent(harness.deps, ACTOR, { edl, profile: paid }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/packages/adapters/src/media-consent.ts b/packages/adapters/src/media-consent.ts new file mode 100644 index 0000000..bb9e51f --- /dev/null +++ b/packages/adapters/src/media-consent.ts @@ -0,0 +1,161 @@ +import type { Edl, GateIssue, PlatformProfile } from "@rakazo/contracts"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; +import { clipWindows, overlapsWindows } from "./media-privacy-plan.js"; + +/* + * Consent and age proof (spec 0 and 6.4). A paid-video profile always needs + * documented consent and age verification of every recognised participant, + * whatever its own flags say; other profiles follow their consent flags. + * Faces that belong to no person can never show consent and block. + */ + +export interface ConsentRequirements { + consent: boolean; + age: boolean; +} + +export function consentRequirements(profile: PlatformProfile): ConsentRequirements { + const paid = profile.kind === "paid_video"; + return { + consent: paid || profile.consent.requirePerformerConsent, + age: paid || profile.consent.requireAgeVerification, + }; +} + +/** The profile with its consent flags raised to what the hard rules demand. */ +export function enforceConsentFlags(profile: PlatformProfile): PlatformProfile { + const required = consentRequirements(profile); + return { + ...profile, + consent: { requirePerformerConsent: required.consent, requireAgeVerification: required.age }, + }; +} + +export interface PersonConsentState { + id: string; + consentRecordedAt: Date | string | null; + ageVerifiedAt: Date | string | null; + ageVerificationMethod: string; +} + +/** Valid proof: a recorded date that is not in the future. */ +function proven(value: Date | string | null, now: Date): boolean { + if (!value) return false; + const at = value instanceof Date ? value : new Date(value); + return !Number.isNaN(at.getTime()) && at.getTime() <= now.getTime(); +} + +export function hasConsent(person: PersonConsentState, now: Date): boolean { + return proven(person.consentRecordedAt, now); +} + +export function hasAgeProof(person: PersonConsentState, now: Date): boolean { + return person.ageVerificationMethod !== "none" && proven(person.ageVerifiedAt, now); +} + +/** + * Consent issues for the participants of an export. `unassigned` counts faces + * no person could be matched to; each blocks because nobody can consent for it. + */ +export function consentIssues( + profile: PlatformProfile, + people: PersonConsentState[], + unassigned: Array<{ atMs?: number; endMs?: number }>, + now: Date, +): GateIssue[] { + const required = consentRequirements(profile); + if (!required.consent && !required.age) return []; + const issues: GateIssue[] = []; + for (const person of [...people].sort((a, b) => a.id.localeCompare(b.id))) { + if (required.consent && !hasConsent(person, now)) { + issues.push({ + check: "consent", + severity: "block", + code: "consent_missing", + message: "no recorded consent", + personId: person.id, + }); + } + if (required.age && !hasAgeProof(person, now)) { + issues.push({ + check: "consent", + severity: "block", + code: "age_unverified", + message: "age is not verified", + personId: person.id, + }); + } + } + for (const face of unassigned) { + issues.push({ + check: "consent", + severity: "block", + code: "unknown_performer", + message: "a visible face is not assigned to a person with consent", + ...(face.atMs !== undefined ? { atMs: face.atMs } : {}), + ...(face.endMs !== undefined ? { endMs: face.endMs } : {}), + }); + } + return issues; +} + +/** + * People and unassigned faces in the clip ranges of an EDL, from the stored + * face tracks of its assets. + */ +export function participantsOf( + edl: Edl, + tracks: Array<{ assetId: string; personId: string | null; startMs: number; endMs: number }>, +): { + personIds: string[]; + unassigned: Array<{ assetId: string; startMs: number; endMs: number }>; +} { + const personIds = new Set(); + const unassigned: Array<{ assetId: string; startMs: number; endMs: number }> = []; + for (const track of tracks) { + const windows = clipWindows(edl.tracks.video.filter((clip) => clip.assetId === track.assetId)); + if (!overlapsWindows(track, windows)) continue; + if (track.personId) personIds.add(track.personId); + else unassigned.push({ assetId: track.assetId, startMs: track.startMs, endMs: track.endMs }); + } + return { personIds: [...personIds].sort(), unassigned }; +} + +export class ConsentRequiredError extends Error { + constructor(readonly issues: GateIssue[]) { + super( + `export blocked: ${issues.length} participant${issues.length === 1 ? "" : "s"} without documented consent or age proof`, + ); + this.name = "ConsentRequiredError"; + } +} + +/** + * Consent issues for exporting a render to a profile, re-read at export time + * so a withdrawn consent stops a bundle that passed the gate earlier. + */ +export async function exportConsentIssues( + deps: Pick, + actor: MediaActor, + input: { edl: Edl; profile: PlatformProfile }, +): Promise { + const required = consentRequirements(input.profile); + if (!required.consent && !required.age) return []; + const assetIds = [...new Set(input.edl.tracks.video.map((clip) => clip.assetId))]; + const tracks = await deps.repos.faceTrackRecords(actor.spaceId, assetIds); + const { personIds, unassigned } = participantsOf(input.edl, tracks); + const people = personIds.length + ? await deps.repos.listPersonRecords(actor.spaceId, personIds) + : []; + return consentIssues(input.profile, people, unassigned, deps.now()); +} + +/** Throws ConsentRequiredError when the export may not happen. */ +export async function assertExportConsent( + deps: Pick, + actor: MediaActor, + input: { edl: Edl; profile: PlatformProfile }, +): Promise { + const issues = await exportConsentIssues(deps, actor, input); + if (issues.length > 0) throw new ConsentRequiredError(issues); +} diff --git a/packages/adapters/src/media-gate-autofix.test.ts b/packages/adapters/src/media-gate-autofix.test.ts new file mode 100644 index 0000000..a1ac9a4 --- /dev/null +++ b/packages/adapters/src/media-gate-autofix.test.ts @@ -0,0 +1,188 @@ +import type { Edl, GateReport } from "@rakazo/contracts"; +import { MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { autoFixRender, nonCompliantChange, proposeAutoFix } from "./media-gate-autofix.js"; +import type { PrivacyHarness } from "./media-privacy-fixtures.js"; +import { ACTOR, createPrivacyHarness, testEdl } from "./media-privacy-fixtures.js"; + +let harness: PrivacyHarness | null = null; +afterEach(async () => { + await harness?.cleanup(); + harness = null; +}); + +function edlWithFace(): Edl { + return testEdl( + [ + { id: "c1", assetId: "a1", inMs: 0, outMs: 4000 }, + { id: "c2", assetId: "a2", inMs: 0, outMs: 4000 }, + ], + { + redactions: { + faces: [ + { + id: "f1", + target: { personId: "alice" }, + mode: "blur", + paddingPct: 20, + featherPct: 10, + minSizePct: 2, + }, + ], + marks: [{ markId: "m1", treatment: "inpaint" }], + regions: [], + }, + }, + ); +} + +function blocked(issues: GateReport["issues"], round = 0): GateReport { + return { + profileId: "test-profile", + passed: false, + checkedAt: "2026-06-01T12:00:00.000Z", + sampledFrames: 10, + issues, + autoFixRound: round, + }; +} + +const contentHit = { + check: "content" as const, + severity: "block" as const, + code: "content_threshold", + message: "explicit_nudity 0.90 above 0.5", + atMs: 7000, + endMs: 7500, + category: "explicit_nudity" as const, + score: 0.9, +}; + +describe("proposeAutoFix", () => { + it("only widens protection, covers marks, and cuts offending footage", () => { + const edl = edlWithFace(); + const proposal = proposeAutoFix({ + edl, + report: blocked([ + contentHit, + { + check: "anonymization", + severity: "block", + code: "face_not_covered", + message: "face covered 80 %", + atMs: 100, + personId: "alice", + }, + { + check: "anonymization", + severity: "block", + code: "mark_visible", + message: "visible", + atMs: 200, + markId: "m1", + }, + ]), + round: 0, + alternatives: [], + }); + expect("edl" in proposal).toBe(true); + if (!("edl" in proposal)) return; + expect(proposal.edl.redactions.faces[0]!.paddingPct).toBe(45); + expect(proposal.edl.redactions.marks[0]!.treatment).toBe("cover"); + expect(proposal.edl.tracks.video.find((c) => c.id === "c2")!.outMs).toBe(2500); + expect(proposal.changes.length).toBeGreaterThanOrEqual(3); + expect(nonCompliantChange(edl, proposal.edl)).toBeNull(); + }); + + it("stops after the round limit", () => { + const proposal = proposeAutoFix({ + edl: edlWithFace(), + report: blocked([contentHit], MAX_AUTO_FIX_ROUNDS), + round: MAX_AUTO_FIX_ROUNDS, + alternatives: [], + }); + expect(proposal).toEqual({ reason: "round_limit", changes: [] }); + }); + + it("rejects any change that weakens protection or adds content", () => { + const edl = edlWithFace(); + const weaker = structuredClone(edl); + weaker.redactions.faces[0]!.paddingPct = 5; + expect(nonCompliantChange(edl, weaker)).toMatch(/shrank/); + const removed = structuredClone(edl); + removed.redactions.faces = []; + expect(nonCompliantChange(edl, removed)).toMatch(/removed/); + const unmasked = structuredClone(edl); + unmasked.redactions.marks[0]!.treatment = "blur"; + expect(nonCompliantChange(edl, unmasked)).toMatch(/changed to blur/); + const grown = structuredClone(edl); + grown.tracks.video[0]!.outMs = 9000; + expect(nonCompliantChange(edl, grown)).toMatch(/grew/); + const overlay = structuredClone(edl); + overlay.overlays.push({ id: "o1", kind: "text", atMs: 0, durationMs: 100, text: "x" }); + expect(nonCompliantChange(edl, overlay)).toMatch(/overlays/); + const swapped = structuredClone(edl); + swapped.redactions.faces[0]!.mode = "partial_eyes"; + expect(nonCompliantChange(edl, swapped)).toMatch(/mode/); + }); +}); + +describe("autoFixRender", () => { + async function blockedRender(round: number) { + harness = await createPrivacyHarness(() => null); + const edl = edlWithFace(); + harness.store.plans.push({ id: "plan-1", projectId: "project-1", edl }); + harness.store.renders.push({ + id: "render-1", + planId: "plan-1", + projectId: "project-1", + quality: "draft", + status: "blocked", + autoFixRound: round, + gateReport: blocked([contentHit], round), + }); + harness.store.renders.push({ id: "render-2", status: "queued", autoFixRound: 0 }); + const submitPlan = vi.fn(async () => ({ + ok: true as const, + plan: { id: "plan-2" } as never, + })); + const start = vi.fn(async () => ({ id: "render-2" }) as never); + return { + services: { deps: harness.deps, marketing: { submitPlan }, render: { start } }, + submitPlan, + start, + }; + } + + it("submits a new plan with the reasons and renders it as the next round", async () => { + const { services, submitPlan, start } = await blockedRender(1); + const result = await autoFixRender(services, ACTOR, "render-1"); + expect(result.status).toBe("rerendering"); + expect(submitPlan).toHaveBeenCalledWith( + ACTOR, + expect.objectContaining({ + projectId: "project-1", + createdBy: "agent", + rationale: expect.stringMatching(/^Auto-fix round 2 of 3:\n- clip c2: /), + }), + ); + expect(start).toHaveBeenCalledWith(ACTOR, { planId: "plan-2", quality: "draft" }); + expect(harness!.store.renders[1]!.autoFixRound).toBe(2); + }); + + it("hands over to the user after three rounds", async () => { + const { services, submitPlan } = await blockedRender(MAX_AUTO_FIX_ROUNDS); + const result = await autoFixRender(services, ACTOR, "render-1"); + expect(result).toEqual({ status: "user_required", reason: "round_limit", changes: [] }); + expect(submitPlan).not.toHaveBeenCalled(); + }); + + it("does nothing for a render that is not blocked", async () => { + const { services } = await blockedRender(0); + harness!.store.renders[0]!.status = "done"; + expect(await autoFixRender(services, ACTOR, "render-1")).toMatchObject({ + status: "user_required", + reason: "not_blocked", + }); + }); +}); diff --git a/packages/adapters/src/media-gate-autofix.ts b/packages/adapters/src/media-gate-autofix.ts new file mode 100644 index 0000000..cbdc402 --- /dev/null +++ b/packages/adapters/src/media-gate-autofix.ts @@ -0,0 +1,193 @@ +import type { Edl, EdlError, GateReport, MediaRender, MediaRenderQuality } from "@rakazo/contracts"; +import { EdlSchema, GateReportSchema, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import { ACTIVE_RENDER_STATUSES } from "@rakazo/db"; +import { planAutoFix } from "@rakazo/media"; +import type { + MediaActor, + MediaMarketingService, + MediaRenderService, + MediaServiceDeps, +} from "./media-context.js"; + +/* + * Auto-fix for blocked renders (spec 6): only compliance-preserving repairs + * (trim or drop a clip, take another segment, change the crop, widen a + * redaction, cover a mark) through planAutoFix, re-checked here so no round + * can weaken protection or add anything new. Each round submits a new plan + * version with the reasons as rationale and starts a new render, which runs + * the gate again. After MAX_AUTO_FIX_ROUNDS the user takes over. + */ + +export type AutoFixResult = + | { status: "rerendering"; round: number; render: MediaRender; changes: string[] } + | { + status: "user_required"; + reason: "not_blocked" | "round_limit" | "no_safe_fix" | "plan_rejected"; + changes: string[]; + errors?: EdlError[]; + }; + +export interface AutoFixDeps { + deps: Pick; + marketing: Pick; + render: Pick; +} + +/** + * Why a proposed fix is not a pure compliance repair, or null when it is. + * Allowed: fewer or shorter clips, other segments, other crops, wider padding + * on the same face redactions, added blur redactions, marks switched to cover. + * Forbidden: anything that removes or weakens protection or adds content. + */ +export function nonCompliantChange(before: Edl, after: Edl): string | null { + const facesBefore = new Map(before.redactions.faces.map((f) => [f.id, f])); + for (const face of before.redactions.faces) { + const next = after.redactions.faces.find((f) => f.id === face.id); + if (!next) return `face redaction ${face.id} was removed`; + if (JSON.stringify(next.target) !== JSON.stringify(face.target)) + return `face redaction ${face.id} changed its target`; + if (next.mode !== face.mode) return `face redaction ${face.id} changed its mode`; + if (next.paddingPct < face.paddingPct || next.minSizePct < face.minSizePct) + return `face redaction ${face.id} shrank`; + } + for (const face of after.redactions.faces) { + if (!facesBefore.has(face.id) && face.mode !== "blur") + return `added face redaction ${face.id} is not a blur`; + } + for (const mark of before.redactions.marks) { + const next = after.redactions.marks.find((m) => m.markId === mark.markId); + if (!next) return `mark ${mark.markId} lost its treatment`; + if (next.treatment !== mark.treatment && next.treatment !== "cover") + return `mark ${mark.markId} changed to ${next.treatment}`; + } + if (after.redactions.marks.length !== before.redactions.marks.length) + return "mark treatments were added"; + const clipIds = new Set(after.tracks.video.map((clip) => clip.id)); + for (const region of before.redactions.regions) { + if (!clipIds.has(region.clipId)) continue; + const next = after.redactions.regions.find((r) => r.id === region.id); + if (!next || JSON.stringify(next) !== JSON.stringify(region)) + return `region ${region.id} changed`; + } + if (after.redactions.regions.length > before.redactions.regions.length) + return "regions were added"; + if (JSON.stringify(after.redactions.voice) !== JSON.stringify(before.redactions.voice)) + return "voice disguise changed"; + const beforeClips = new Map(before.tracks.video.map((clip) => [clip.id, clip])); + for (const clip of after.tracks.video) { + const old = beforeClips.get(clip.id); + if (!old) return `clip ${clip.id} was added`; + if (clip.speed !== old.speed || clip.muted !== old.muted) + return `clip ${clip.id} changed speed or sound`; + if (JSON.stringify(clip.color) !== JSON.stringify(old.color)) + return `clip ${clip.id} changed its colors`; + if (clip.assetId === old.assetId && (clip.inMs < old.inMs || clip.outMs > old.outMs)) { + if (clip.segmentId === old.segmentId) return `clip ${clip.id} grew`; + } + } + if (after.overlays.length > before.overlays.length) return "overlays were added"; + for (const overlay of after.overlays) { + const old = before.overlays.find((o) => o.id === overlay.id); + if (!old || old.text !== overlay.text || old.kind !== overlay.kind) + return `overlay ${overlay.id} changed`; + } + if ( + JSON.stringify(after.tracks.audio) !== JSON.stringify(before.tracks.audio) || + JSON.stringify(after.captions) !== JSON.stringify(before.captions) || + JSON.stringify(after.intro) !== JSON.stringify(before.intro) || + JSON.stringify(after.outro) !== JSON.stringify(before.outro) || + JSON.stringify(after.output) !== JSON.stringify(before.output) + ) { + return "audio, captions, bookends, or output settings changed"; + } + for (const alteration of after.metadata.aiAlterations) { + if (!before.metadata.aiAlterations.includes(alteration)) + return `AI alteration ${alteration} was added`; + } + return null; +} + +/** Plan one auto-fix round for a blocked render, or say why the user must take over. */ +export function proposeAutoFix(input: { + edl: Edl; + report: GateReport; + round: number; + alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>; +}): { edl: Edl; changes: string[] } | { reason: "round_limit" | "no_safe_fix"; changes: string[] } { + if (input.round >= MAX_AUTO_FIX_ROUNDS) return { reason: "round_limit", changes: [] }; + const fix = planAutoFix( + input.edl, + { ...input.report, autoFixRound: input.round }, + input.alternatives, + ); + if (!fix) return { reason: "no_safe_fix", changes: [] }; + const parsed = EdlSchema.safeParse(fix.edl); + if (!parsed.success) return { reason: "no_safe_fix", changes: fix.changes }; + const violation = nonCompliantChange(input.edl, parsed.data); + if (violation) + return { reason: "no_safe_fix", changes: [...fix.changes, `rejected: ${violation}`] }; + return { edl: parsed.data, changes: fix.changes }; +} + +/** + * Run one auto-fix round for a blocked render: new plan version (createdBy + * agent, rationale lists every change) and a new render carrying the next + * round number, whose gate decides again. + */ +export async function autoFixRender( + services: AutoFixDeps, + actor: MediaActor, + renderId: string, +): Promise { + const { repos } = services.deps; + const row = await repos.getRenderRecord(actor.spaceId, renderId); + const report = GateReportSchema.nullable().parse(row.gateReport ?? null); + if (row.status !== "blocked" || !report || report.passed) { + return { status: "user_required", reason: "not_blocked", changes: [] }; + } + const plan = await repos.getPlan(actor.spaceId, row.planId); + const project = await repos.getProjectRecord(actor.spaceId, row.projectId); + const assetIds = [ + ...new Set([...project.assetIds, ...plan.edl.tracks.video.map((clip) => clip.assetId)]), + ]; + const segments = await repos.listSegments(actor.spaceId, assetIds); + const alternatives = segments + .filter((segment) => !segment.paidOnly) + .map((segment) => ({ + segmentId: segment.id, + assetId: segment.assetId, + startMs: segment.startMs, + endMs: segment.endMs, + })); + const proposal = proposeAutoFix({ edl: plan.edl, report, round: row.autoFixRound, alternatives }); + if ("reason" in proposal) { + return { status: "user_required", reason: proposal.reason, changes: proposal.changes }; + } + const round = row.autoFixRound + 1; + const rationale = [ + `Auto-fix round ${round} of ${MAX_AUTO_FIX_ROUNDS}:`, + ...proposal.changes, + ].join("\n- "); + const submitted = await services.marketing.submitPlan(actor, { + projectId: row.projectId, + edl: proposal.edl, + rationale, + createdBy: "agent", + }); + if (!submitted.ok) { + return { + status: "user_required", + reason: "plan_rejected", + changes: proposal.changes, + errors: submitted.errors, + }; + } + const started = await services.render.start(actor, { + planId: submitted.plan.id, + quality: row.quality as MediaRenderQuality, + }); + const updated = await repos.updateRender(actor.spaceId, started.id, { autoFixRound: round }, [ + ...ACTIVE_RENDER_STATUSES, + ]); + return { status: "rerendering", round, render: updated ?? started, changes: proposal.changes }; +} diff --git a/packages/adapters/src/media-gate.test.ts b/packages/adapters/src/media-gate.test.ts new file mode 100644 index 0000000..871e532 --- /dev/null +++ b/packages/adapters/src/media-gate.test.ts @@ -0,0 +1,250 @@ +import { copyFile, rm } from "node:fs/promises"; +import path from "node:path"; +import type { MediaInferenceProvider, TrackFacesResult } from "@rakazo/adapter-kit"; +import type { AnonymizeMode, Edl, PlatformProfile } from "@rakazo/contracts"; +import { decodeVectors } from "@rakazo/media"; +import { afterEach, describe, expect, it } from "vitest"; +import { prepareSources } from "./media-anonymize.js"; +import { ffmpegAvailable } from "./media-emulator-clips.js"; +import { cutTimes, GATE_PIXEL_STEP_MS, pixelSampleTimes, runGate } from "./media-gate.js"; +import type { PrivacyHarness } from "./media-privacy-fixtures.js"; +import { + ACTOR, + cleanProbe, + createPrivacyHarness, + testEdl, + testProfile, +} from "./media-privacy-fixtures.js"; +import { PRIVACY_RECORD_FILE } from "./media-privacy-plan.js"; + +const hasFfmpeg = ffmpegAvailable(); +let harness: PrivacyHarness | null = null; + +afterEach(async () => { + await harness?.cleanup(); + harness = null; +}); + +/** Faces the fake detector reports on the render output. */ +let outputFaces: TrackFacesResult = { sampleFps: 4, tracks: [] }; +let classifierAvailable = true; + +function withOutputDetector(emulator: PrivacyHarness["inference"]): MediaInferenceProvider { + const provider = Object.create(emulator) as MediaInferenceProvider; + provider.trackFaces = async (request) => + path.basename(request.path).startsWith("output") ? outputFaces : emulator.trackFaces(request); + provider.capabilities = async () => { + const caps = await emulator.capabilities(); + caps.operations.classifyContent.available = classifierAvailable; + return caps; + }; + return provider; +} + +async function prepared( + mode: AnonymizeMode, + options: { file?: string; unassigned?: boolean } = {}, +) { + outputFaces = { sampleFps: 4, tracks: [] }; + classifierAvailable = true; + harness = await createPrivacyHarness(withOutputDetector); + const asset = await harness.addAsset(options.file ?? "clip__faces-2____person-alice__.mp4"); + harness.addPerson("alice", mode); + await harness.analyseFaces(asset.id, ["alice", options.unassigned ? null : "alice"]); + await harness.learnPerson("alice", asset.id); + const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]); + const workDir = await harness.workDir(`gate-${mode}`); + const result = await prepareSources(harness.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + const output = path.join(workDir, "output.mp4"); + await copyFile(result.sources.get(asset.id)!.path, output); + return { h: harness, asset, edl, workDir, output }; +} + +function gate( + h: PrivacyHarness, + input: { edl: Edl; output: string }, + options: { + profile?: PlatformProfile; + copy?: { title: string | null; caption: string | null; tags: string[] }; + probe?: Partial>; + } = {}, +) { + return runGate(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl: input.edl, + profile: options.profile ?? testProfile(), + outputPath: input.output, + ffprobe: { ...cleanProbe(input.edl), ...options.probe }, + copy: options.copy ?? { title: null, caption: null, tags: [] }, + bundleHasDisclosure: false, + autoFixRound: 0, + }); +} + +const box = { x: 0.4, y: 0.4, w: 0.2, h: 0.2 }; +const strangerEmbedding = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0)); + +describe("gate sampling", () => { + it("samples each clip range and spreads long ones", () => { + const edl = testEdl([ + { id: "a", assetId: "x", inMs: 0, outMs: 1000 }, + { id: "b", assetId: "x", inMs: 5000, outMs: 6000 }, + ]); + expect(cutTimes(edl)).toEqual([0, 999, 1000, 1999]); + expect(pixelSampleTimes(edl.tracks.video, false)).toEqual([0, 500, 960, 5000, 5500, 5960]); + expect(pixelSampleTimes(edl.tracks.video, true)).toEqual([0]); + const long = pixelSampleTimes([{ ...edl.tracks.video[0]!, outMs: 600_000 }], false); + expect(long).toHaveLength(40); + expect(long[1]! - long[0]!).toBeGreaterThanOrEqual(GATE_PIXEL_STEP_MS); + }); +}); + +describe.skipIf(!hasFfmpeg)("runGate", () => { + it("passes a protected render with a clean classifier", async () => { + const { h, edl, output } = await prepared("blur"); + const report = await gate(h, { edl, output }); + expect(report.issues).toEqual([]); + expect(report.passed).toBe(true); + expect(report.sampledFrames).toBeGreaterThan(0); + }); + + it("classifies the rendered output and blocks above the threshold", async () => { + const { h, edl, workDir, output } = await prepared("blur"); + const flagged = path.join(workDir, "output__nsfw-explicit_nudity-0.9__.mp4"); + await copyFile(output, flagged); + const report = await gate(h, { edl, output: flagged }); + expect(report.passed).toBe(false); + expect(report.issues).toContainEqual( + expect.objectContaining({ + check: "content", + code: "content_threshold", + category: "explicit_nudity", + score: 0.9, + atMs: 0, + }), + ); + }); + + it("blocks when the classifier is unavailable", async () => { + const { h, edl, output } = await prepared("blur"); + classifierAvailable = false; + const report = await gate(h, { edl, output }); + expect(report.issues).toContainEqual( + expect.objectContaining({ code: "classifier_unavailable", severity: "block" }), + ); + }); + + it("blocks a recognisable protected face and an unknown face in the output", async () => { + const { h, edl, output } = await prepared("blur"); + const alice = h.store.people[0]!; + const embedding = [...decodeVectors(alice.centroid!, 16)[0]!]; + outputFaces = { + sampleFps: 4, + tracks: [ + { trackId: "o1", keyframes: [{ tMs: 250, box, confidence: 0.9 }], embedding }, + { + trackId: "o2", + keyframes: [{ tMs: 750, box, confidence: 0.9 }], + embedding: strangerEmbedding, + }, + ], + }; + const report = await gate(h, { edl, output }); + expect(report.issues).toContainEqual( + expect.objectContaining({ code: "face_not_covered", personId: "alice", atMs: 250 }), + ); + expect(report.issues).toContainEqual( + expect.objectContaining({ code: "unknown_face", atMs: 750 }), + ); + }); + + it("lets a partially covered face stay detectable without counting it as unknown", async () => { + const { h, edl, workDir, output } = await prepared("partial_eyes"); + expect(path.basename(workDir)).toBe("gate-partial_eyes"); + outputFaces = { + sampleFps: 4, + tracks: [ + { + trackId: "o1", + keyframes: [{ tMs: 250, box, confidence: 0.9 }], + embedding: strangerEmbedding, + }, + ], + }; + const report = await gate(h, { edl, output }); + expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([]); + }); + + it("requires consent and age proof of every participant for paid video", async () => { + const { h, edl, output } = await prepared("none"); + // The profile's own flags are off; paid video turns them on regardless. + const paid = testProfile({ kind: "paid_video" }); + const blocked = await gate(h, { edl, output }, { profile: paid }); + expect(blocked.passed).toBe(false); + expect(blocked.issues.filter((i) => i.check === "consent").map((i) => i.code)).toEqual([ + "consent_missing", + "age_unverified", + ]); + const alice = h.store.people[0]!; + alice.consentRecordedAt = new Date("2026-01-01T00:00:00Z"); + alice.ageVerifiedAt = new Date("2026-01-01T00:00:00Z"); + alice.ageVerificationMethod = "document_on_file"; + const allowed = await gate(h, { edl, output }, { profile: paid }); + expect(allowed.issues.filter((i) => i.check === "consent")).toEqual([]); + expect(allowed.passed).toBe(true); + }); + + it("blocks consent profiles when a face belongs to nobody", async () => { + const { h, edl, output } = await prepared("none", { unassigned: true }); + const profile = testProfile({ + consent: { requirePerformerConsent: true, requireAgeVerification: true }, + }); + const alice = h.store.people[0]!; + alice.consentRecordedAt = new Date("2026-01-01T00:00:00Z"); + alice.ageVerifiedAt = new Date("2026-01-01T00:00:00Z"); + alice.ageVerificationMethod = "platform_verified"; + const report = await gate(h, { edl, output }, { profile }); + expect(report.issues).toEqual([ + expect.objectContaining({ check: "consent", code: "unknown_performer", atMs: 0 }), + ]); + }); + + it("blocks banned text and a duration outside the profile", async () => { + const { h, edl, output } = await prepared("blur"); + const profile = testProfile({ + duration: { hardMinMs: 0, hardMaxMs: 1500 }, + text: { + captionMaxChars: 100, + hashtagLimit: 3, + linkPolicy: "forbidden", + bannedPatterns: [{ pattern: "free\\s+money", reason: "spam" }], + }, + }); + const report = await gate( + h, + { edl, output }, + { profile, copy: { title: "Free money", caption: "see example.com", tags: [] } }, + ); + const codes = report.issues.map((issue) => issue.code); + expect(codes).toEqual( + expect.arrayContaining(["duration_too_long", "banned_pattern", "link_forbidden"]), + ); + expect(report.passed).toBe(false); + }); + + it("blocks when the sources were not prepared for this render", async () => { + const { h, edl, workDir, output } = await prepared("blur"); + await rm(path.join(workDir, PRIVACY_RECORD_FILE)); + const report = await gate(h, { edl, output }); + expect(report.issues).toContainEqual( + expect.objectContaining({ code: "privacy_unverified", severity: "block" }), + ); + }); +}); diff --git a/packages/adapters/src/media-gate.ts b/packages/adapters/src/media-gate.ts new file mode 100644 index 0000000..47ce5c7 --- /dev/null +++ b/packages/adapters/src/media-gate.ts @@ -0,0 +1,396 @@ +import path from "node:path"; +import type { + ContentCategory, + Edl, + EdlVideoClip, + FfprobeReport, + GateIssue, + GateReport, + NormalizedBox, + PlatformProfile, +} from "@rakazo/contracts"; +import type { GateInput } from "@rakazo/media"; +import { clipLengthMs, evaluateGate, introMs, MAX_MARK_RESIDUAL } from "@rakazo/media"; +import { assertInside } from "@rakazo/media/node"; +import { consentIssues, enforceConsentFlags, hasAgeProof, hasConsent } from "./media-consent.js"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; +import type { PrivacyToolOptions } from "./media-privacy-io.js"; +import { inferenceCapabilities, maskTimes, operationAvailable } from "./media-privacy-io.js"; +import type { LumaFrame } from "./media-privacy-pixels.js"; +import { + controlAgreement, + extractLuma, + readPngLuma, + residualVisibility, +} from "./media-privacy-pixels.js"; +import type { PrivacyCheck, PrivacyRecord, TimeWindow } from "./media-privacy-plan.js"; +import { + boxAt, + isPartial, + matchPerson, + readPrivacyRecord, + sourceToOutput, +} from "./media-privacy-plan.js"; + +/* + * Gate execution on the rendered output (spec 6). Content is classified on the + * output file at the profile's sample rate plus every cut; faces are tracked on + * the output and matched to people locally. Protection is verified pixel by + * pixel: for every protected region prepareSources recorded, the prepared + * frame is compared with the unredacted frame at the same moment. The + * compiler only crops, scales, and composites prepared frames, so a region + * altered there stays altered in the output. Anything the gate cannot check + * blocks (fail closed). + */ + +export type GateDeps = Pick & + PrivacyToolOptions; + +/** Face tracking rate on the output. */ +export const GATE_FACE_FPS = 4; +/** Pixel verification step inside each clip range. */ +export const GATE_PIXEL_STEP_MS = 500; +/** Pixel samples per asset at most (spread evenly). */ +export const MAX_PIXEL_SAMPLES_PER_ASSET = 40; +/** Below this agreement outside the treated regions, two frames are not comparable. */ +const MIN_CONTROL_AGREEMENT = 0.8; + +export interface GateRunInput { + actor: MediaActor; + projectId: string; + edl: Edl; + profile: PlatformProfile; + outputPath: string; + ffprobe: FfprobeReport; + copy: { title: string | null; caption: string | null; tags: string[] }; + bundleHasDisclosure: boolean; + autoFixRound: number; + signal?: AbortSignal; +} + +type Faces = GateInput["faces"]; +type Marks = GateInput["marks"]; + +function block(issue: Omit): GateIssue { + return { severity: "block", ...issue }; +} + +/** Output times of every cut: clip starts and ends, intro end. */ +export function cutTimes(edl: Edl): number[] { + const intro = introMs(edl); + const times = new Set(); + if (intro > 0) times.add(intro); + for (const clip of edl.tracks.video) { + times.add(intro + clip.atMs); + times.add(Math.max(0, intro + clip.atMs + clipLengthMs(clip) - 1)); + } + return [...times].sort((a, b) => a - b); +} + +function clipsOf(edl: Edl, assetId: string): EdlVideoClip[] { + return edl.tracks.video.filter((clip) => clip.assetId === assetId); +} + +/** Output windows of a source span across every clip of its asset. */ +function toOutput(edl: Edl, assetId: string, span: TimeWindow): TimeWindow[] { + return clipsOf(edl, assetId).flatMap((clip) => { + const mapped = sourceToOutput(edl, clip, span); + return mapped ? [mapped] : []; + }); +} + +/** Evaluate the full gate for a render output. */ +export async function runGate(deps: GateDeps, input: GateRunInput): Promise { + const { actor, edl, signal } = input; + const profile = enforceConsentFlags(input.profile); + const outputPath = await assertInside(deps.mediaRoot, input.outputPath); + const record = await readPrivacyRecord(path.dirname(outputPath)); + const capabilities = await inferenceCapabilities(deps.inference, signal); + const extra: GateIssue[] = []; + const now = deps.now(); + + if (!record) { + extra.push( + block({ + check: "anonymization", + code: "privacy_unverified", + message: "sources were not prepared for this render, so protection cannot be shown", + }), + ); + } + + // Content on the rendered output. + let content: GateInput["content"] = []; + let sampledFrames = 0; + if (Object.keys(profile.content.thresholds).length > 0) { + if (deps.inference && operationAvailable(capabilities, "classifyContent")) { + const classified = await deps.inference.classifyContent({ + path: outputPath, + sampleFps: profile.content.sampleFps, + extraTimesMs: cutTimes(edl), + signal, + }); + content = classified.frames.map((frame) => ({ + tMs: frame.tMs, + scores: frame.scores as Partial>, + })); + sampledFrames = content.length; + } else { + extra.push( + block({ + check: "content", + code: "classifier_unavailable", + message: "the content classifier is unavailable, so the output cannot be checked", + }), + ); + } + } + + // Faces re-detected on the output. + const faces: Faces = []; + const matchedPeople = new Set(); + const protectedPeople = record?.protectedPeople ?? {}; + const consentRequired = + profile.consent.requirePerformerConsent || profile.consent.requireAgeVerification; + const needFaces = Object.keys(protectedPeople).length > 0 || consentRequired; + if (needFaces) { + if (deps.inference && operationAvailable(capabilities, "trackFaces")) { + const tracked = await deps.inference.trackFaces({ + path: outputPath, + sampleFps: GATE_FACE_FPS, + signal, + }); + const centroids = await deps.repos.personCentroids(actor.spaceId); + const expected = expectedOutputFaces(edl, record); + const unmatchedAt = new Map(); + const slot = (tMs: number) => Math.round(tMs / (1000 / GATE_FACE_FPS)); + for (const track of tracked.tracks) { + const personId = matchPerson(track.embedding, centroids); + if (personId) matchedPeople.add(personId); + const mode = personId ? protectedPeople[personId] : undefined; + for (const keyframe of track.keyframes) { + if (personId) { + // Partial modes and synthetic identities stay detectable by design; + // their regions are verified pixel by pixel below. + if (mode && (isPartial(mode) || mode === "synthetic_identity")) continue; + faces.push({ tMs: keyframe.tMs, box: keyframe.box, personId, coverage: mode ? 0 : 1 }); + continue; + } + const key = slot(keyframe.tMs); + const seen = unmatchedAt.get(key) ?? 0; + unmatchedAt.set(key, seen + 1); + const allowance = expected.filter( + (w) => keyframe.tMs >= w.startMs && keyframe.tMs <= w.endMs, + ).length; + if (seen < allowance) continue; + faces.push({ tMs: keyframe.tMs, box: keyframe.box, personId: null, coverage: 0 }); + } + } + } else { + extra.push( + block({ + check: "anonymization", + code: "face_check_unavailable", + message: "face detection is unavailable, so faces in the output cannot be checked", + }), + ); + } + } + + // Pixel verification of every recorded protection. + const marks: Marks = []; + if (record) { + const verified = await verifyPixels(deps, edl, record, signal); + for (const sample of verified) { + const covered = sample.residual <= MAX_MARK_RESIDUAL; + const { check } = sample; + if (check.kind === "face") { + if (check.personId && protectedPeople[check.personId]) { + faces.push({ + tMs: sample.outputMs, + box: sample.box, + personId: check.personId, + coverage: covered ? 1 : 1 - sample.residual, + }); + } else if (!covered) { + extra.push( + block({ + check: "anonymization", + code: "face_not_covered", + message: `face covered ${((1 - sample.residual) * 100).toFixed(0)} %`, + atMs: sample.outputMs, + endMs: sample.outputMs, + score: 1 - sample.residual, + ...(check.personId ? { personId: check.personId } : {}), + }), + ); + } + } else { + marks.push({ + markId: check.markId ?? check.id, + tMs: sample.outputMs, + residualVisibility: sample.residual, + }); + } + } + for (const review of record.reviews) { + const window = toOutput(edl, review.assetId, { + startMs: 0, + endMs: Number.MAX_SAFE_INTEGER, + })[0]; + extra.push( + block({ + check: "anonymization", + code: "mark_needs_review", + message: `mark found with low confidence (${review.score.toFixed(2)}); review it`, + markId: review.markId, + ...(window ? { atMs: window.startMs, endMs: window.endMs } : {}), + }), + ); + } + } + + const trackingGaps: GateInput["trackingGaps"] = (record?.trackingGaps ?? []).flatMap((gap) => + toOutput(edl, gap.assetId, gap).map((window) => ({ ...window, personId: gap.personId })), + ); + + // People in the output and their consent state. + const ids = [...new Set([...(record?.visiblePeople ?? []), ...matchedPeople])].sort(); + const persons = ids.length ? await deps.repos.listPersonRecords(actor.spaceId, ids) : []; + const people: GateInput["people"] = persons.map((person) => ({ + id: person.id, + anonymizeByDefault: protectedPeople[person.id] ?? "none", + consentRecordedAt: hasConsent(person, now) + ? new Date(person.consentRecordedAt!).toISOString() + : null, + ageVerifiedAt: hasAgeProof(person, now) ? new Date(person.ageVerifiedAt!).toISOString() : null, + })); + if (record && consentRequired) { + const unassigned = record.unassignedFaces.flatMap((face) => + toOutput(edl, face.assetId, face).map((w) => ({ atMs: w.startMs, endMs: w.endMs })), + ); + extra.push(...consentIssues(profile, [], unassigned, now)); + } + + const report = evaluateGate({ + profile, + edl, + ffprobe: input.ffprobe, + content, + faces, + trackingGaps, + marks, + people, + copy: input.copy, + aiAlterations: record?.aiAlterations ?? [], + bundleHasDisclosure: input.bundleHasDisclosure, + watermarkBox: null, + sampledFrames, + autoFixRound: input.autoFixRound, + now, + }); + const issues = [...report.issues, ...extra]; + return { ...report, issues, passed: !issues.some((issue) => issue.severity === "block") }; +} + +/** Output windows where a face stays detectable by design (synthetic identity, partial modes). */ +function expectedOutputFaces(edl: Edl, record: PrivacyRecord | null): TimeWindow[] { + return (record?.expectedFaces ?? []).flatMap((face) => toOutput(edl, face.assetId, face)); +} + +interface PixelSample { + check: PrivacyCheck; + outputMs: number; + box: NormalizedBox; + residual: number; +} + +/** Source times to verify per asset: every GATE_PIXEL_STEP_MS inside its clips, spread evenly. */ +export function pixelSampleTimes(clips: EdlVideoClip[], still: boolean): number[] { + if (still) return [0]; + const times = new Set(); + for (const clip of clips) { + for (let t = clip.inMs; t < clip.outMs; t += GATE_PIXEL_STEP_MS) times.add(t); + times.add(Math.max(clip.inMs, clip.outMs - 40)); + } + const sorted = [...times].sort((a, b) => a - b); + if (sorted.length <= MAX_PIXEL_SAMPLES_PER_ASSET) return sorted; + const step = (sorted.length - 1) / (MAX_PIXEL_SAMPLES_PER_ASSET - 1); + return Array.from( + { length: MAX_PIXEL_SAMPLES_PER_ASSET }, + (_, i) => sorted[Math.round(i * step)]!, + ); +} + +async function verifyPixels( + deps: GateDeps, + edl: Edl, + record: PrivacyRecord, + signal?: AbortSignal, +): Promise { + const samples: PixelSample[] = []; + const maskCache = new Map(); + for (const [assetId, entry] of Object.entries(record.assets).sort((a, b) => + a[0].localeCompare(b[0]), + )) { + if (entry.checks.length === 0) continue; + const clips = clipsOf(edl, assetId); + const treatedSize = entry.crop + ? { width: entry.crop.w, height: entry.crop.h } + : { width: entry.width, height: entry.height }; + const offset = entry.crop ? { x: entry.crop.x, y: entry.crop.y } : undefined; + for (const tMs of pixelSampleTimes(clips, entry.still)) { + signal?.throwIfAborted(); + const active = entry.checks.flatMap((check) => { + const box = entry.still ? (check.keyframes[0]?.box ?? null) : boxAt(check.keyframes, tMs); + return box ? [{ check, box }] : []; + }); + if (active.length === 0) continue; + const tool = { ffmpegPath: deps.ffmpegPath, signal }; + const source = await extractLuma( + entry.basePath, + { tMs, width: entry.width, height: entry.height, still: entry.still }, + tool, + ); + const treated = await extractLuma( + entry.preparedPath, + { tMs, ...treatedSize, still: entry.still }, + tool, + ); + const aligned = + controlAgreement({ source, treated, boxes: active.map((a) => a.box), offset }) >= + MIN_CONTROL_AGREEMENT; + const outputTimes = entry.still + ? clips.map((clip) => introMs(edl) + clip.atMs) + : clips.flatMap((clip) => { + const mapped = sourceToOutput(edl, clip, { startMs: tMs, endMs: tMs }); + return mapped ? [mapped.startMs] : []; + }); + for (const { check, box } of active) { + let residual = 1; + if (aligned) { + const mask = check.maskDir ? await maskAt(check.maskDir, tMs, maskCache) : undefined; + residual = residualVisibility({ source, treated, box, offset, mask }); + } + for (const outputMs of outputTimes) samples.push({ check, outputMs, box, residual }); + } + } + } + return samples; +} + +/** The mask frame nearest at or before `tMs` (or the first one), if the directory has any. */ +async function maskAt( + maskDir: string, + tMs: number, + cache: Map, +): Promise { + let times = cache.get(maskDir); + if (!times) { + times = await maskTimes(maskDir).catch(() => []); + cache.set(maskDir, times); + } + if (times.length === 0) return undefined; + const before = times.filter((t) => t <= tMs).at(-1) ?? times[0]!; + return readPngLuma(path.join(maskDir, `${before}.png`)); +} diff --git a/packages/adapters/src/media-marks.test.ts b/packages/adapters/src/media-marks.test.ts new file mode 100644 index 0000000..64f9752 --- /dev/null +++ b/packages/adapters/src/media-marks.test.ts @@ -0,0 +1,300 @@ +import { copyFile, stat } from "node:fs/promises"; +import path from "node:path"; +import type { MediaInferenceProvider } from "@rakazo/adapter-kit"; +import type { Edl } from "@rakazo/contracts"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { prepareSources } from "./media-anonymize.js"; +import { ffmpegAvailable } from "./media-emulator-clips.js"; +import { runGate } from "./media-gate.js"; +import { anchorFromFace, createMark, placeAnchor, promptBox, updateMark } from "./media-marks.js"; +import { createMediaPrivacyService } from "./media-privacy.js"; +import type { PrivacyHarness } from "./media-privacy-fixtures.js"; +import { + ACTOR, + cleanProbe, + createPrivacyHarness, + testEdl, + testProfile, +} from "./media-privacy-fixtures.js"; +import { PrivacyUnavailableError } from "./media-privacy-io.js"; +import { readPrivacyRecord } from "./media-privacy-plan.js"; + +const hasFfmpeg = ffmpegAvailable(); +let harness: PrivacyHarness | null = null; + +afterEach(async () => { + await harness?.cleanup(); + harness = null; +}); + +const tattoo = { tMs: 500, box: { x: 0.6, y: 0.55, w: 0.2, h: 0.25 } }; + +async function render(h: PrivacyHarness, edl: Edl, name: string) { + const workDir = await h.workDir(name); + const prepared = await prepareSources(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + quality: "final", + workDir, + }); + return { workDir, prepared }; +} + +async function gate(h: PrivacyHarness, edl: Edl, workDir: string, prepared: string) { + const outputPath = path.join(workDir, "output.mp4"); + await copyFile(prepared, outputPath); + return runGate(h.deps, { + actor: ACTOR, + projectId: "project-1", + edl, + profile: testProfile(), + outputPath, + ffprobe: cleanProbe(edl), + copy: { title: null, caption: null, tags: [] }, + bundleHasDisclosure: false, + autoFixRound: 0, + }); +} + +describe("mark geometry", () => { + it("turns a point prompt into a small box inside the frame", () => { + expect(promptBox({ tMs: 0, point: { x: 0.99, y: 0.01 } })).toEqual({ + x: 0.88, + y: 0, + w: 0.12, + h: 0.12, + }); + expect(() => promptBox({ tMs: 0 })).toThrow(PrivacyUnavailableError); + }); + + it("keeps a mark's place relative to the face", () => { + const anchor = anchorFromFace( + { x: 0.5, y: 0.5, w: 0.1, h: 0.1 }, + { x: 0.4, y: 0.3, w: 0.2, h: 0.2 }, + ); + const placed = placeAnchor(anchor, { x: 0.1, y: 0.1, w: 0.4, h: 0.4 })!; + expect(placed.x).toBeCloseTo(0.3); + expect(placed.y).toBeCloseTo(0.5); + expect(placed.w).toBeCloseTo(0.2); + expect(placeAnchor(anchor, { x: 0.95, y: 0.95, w: 0.4, h: 0.4 })).toBeNull(); + }); +}); + +describe.skipIf(!hasFfmpeg)("marks", () => { + it("segments, propagates, scores, and stores the mask under the media root", async () => { + harness = await createPrivacyHarness(); + const asset = await harness.addAsset("clip.mp4"); + const mark = await createMark(harness.deps, ACTOR, { + kind: "tattoo", + label: null, + personId: null, + assetId: asset.id, + prompt: tattoo, + treatment: "blur", + }); + expect(mark).toMatchObject({ treatment: "blur", qualityScore: 0.9, suggestedFallback: null }); + const stored = harness.store.marks[0]!; + expect(path.isAbsolute(stored.maskDir!)).toBe(false); + expect((await stat(path.join(harness.root, stored.maskDir!, "0.png"))).isFile()).toBe(true); + }); + + it("suggests a fallback when inpainting scores low and rescoring follows the treatment", async () => { + harness = await createPrivacyHarness(); + const asset = await harness.addAsset("clip__inpaintq-0.3__.mp4"); + const mark = await createMark(harness.deps, ACTOR, { + kind: "tattoo", + label: null, + personId: null, + assetId: asset.id, + prompt: tattoo, + treatment: "inpaint", + }); + expect(mark.qualityScore).toBeLessThanOrEqual(0.3); + expect(mark.suggestedFallback).toBe("cover"); + const blurred = await updateMark(harness.deps, ACTOR, { markId: mark.id, treatment: "blur" }); + expect(blurred).toMatchObject({ + treatment: "blur", + qualityScore: 0.9, + suggestedFallback: null, + }); + }); + + it("keeps a mark without a worker and rendering it fails closed", async () => { + harness = await createPrivacyHarness(() => null); + const asset = await harness.addAsset("clip.mp4"); + const mark = await createMark(harness.deps, ACTOR, { + kind: "license_plate", + label: null, + personId: null, + assetId: asset.id, + prompt: tattoo, + treatment: "cover", + }); + expect(mark.qualityScore).toBeNull(); + const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]); + await expect(render(harness, edl, "no-worker")).rejects.toBeInstanceOf(PrivacyUnavailableError); + }); + + it("treats a marked region through the video and the gate checks what is left", async () => { + harness = await createPrivacyHarness(); + const asset = await harness.addAsset("clip.mp4"); + const mark = await createMark(harness.deps, ACTOR, { + kind: "tattoo", + label: null, + personId: null, + assetId: asset.id, + prompt: tattoo, + treatment: "none", + }); + // The plan asks for a cover even though the mark itself says none. + const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }], { + redactions: { faces: [], marks: [{ markId: mark.id, treatment: "cover" }], regions: [] }, + }); + const { workDir, prepared } = await render(harness, edl, "cover"); + const record = await readPrivacyRecord(workDir); + expect(record?.assets[asset.id]?.checks).toEqual([ + expect.objectContaining({ kind: "mark", markId: mark.id }), + ]); + const report = await gate(harness, edl, workDir, prepared.sources.get(asset.id)!.path); + expect(report.passed).toBe(true); + }); + + it("blocks when the treatment left the mark visible", async () => { + harness = await createPrivacyHarness((emulator) => { + const provider = Object.create(emulator) as MediaInferenceProvider; + provider.redact = async (request) => { + await copyFile(request.path, request.outPath); + return { outPath: request.outPath, framesProcessed: 50 }; + }; + return provider; + }); + const asset = await harness.addAsset("clip.mp4"); + const mark = await createMark(harness.deps, ACTOR, { + kind: "background_detail", + label: null, + personId: null, + assetId: asset.id, + prompt: tattoo, + treatment: "blur", + }); + const edl = testEdl([{ id: "c1", assetId: asset.id, inMs: 0, outMs: 2000 }]); + const { workDir, prepared } = await render(harness, edl, "leak"); + const report = await gate(harness, edl, workDir, prepared.sources.get(asset.id)!.path); + expect(report.passed).toBe(false); + expect(report.issues).toContainEqual( + expect.objectContaining({ code: "mark_visible", markId: mark.id }), + ); + }); + + it("inpaints when quality holds and covers instead when it does not", async () => { + harness = await createPrivacyHarness(); + const good = await harness.addAsset("good.mp4"); + const poor = await harness.addAsset("poor__inpaintq-0.3__.mp4"); + for (const asset of [good, poor]) { + await createMark(harness.deps, ACTOR, { + kind: "scar", + label: null, + personId: null, + assetId: asset.id, + prompt: tattoo, + treatment: "inpaint", + }); + } + const edl = testEdl([ + { id: "c1", assetId: good.id, inMs: 0, outMs: 1000 }, + { id: "c2", assetId: poor.id, inMs: 0, outMs: 1000 }, + ]); + const { prepared } = await render(harness, edl, "inpaint"); + expect(prepared.aiAlterations).toEqual(["inpaint"]); + expect(prepared.fallbacks).toEqual([ + expect.stringMatching(/inpainting scored 0\.30; covered instead$/), + ]); + }); + + it("re-finds a person's standing mark in another video relative to the face", async () => { + harness = await createPrivacyHarness(); + const origin = await harness.addAsset("a__faces-1____person-alice__.mp4"); + const other = await harness.addAsset("b__faces-1____person-alice__.mp4"); + harness.addPerson("alice", "none"); + await harness.analyseFaces(origin.id, ["alice"]); + await harness.analyseFaces(other.id, ["alice"]); + const segment = vi.spyOn(harness.inference, "segmentPrompted"); + const mark = await createMark(harness.deps, ACTOR, { + kind: "tattoo", + label: "left forearm", + personId: "alice", + assetId: origin.id, + prompt: tattoo, + treatment: "blur", + }); + const edl = testEdl([{ id: "c1", assetId: other.id, inMs: 0, outMs: 2000 }]); + const { workDir, prepared } = await render(harness, edl, "standing"); + expect(segment).toHaveBeenCalledTimes(2); + const reprompt = segment.mock.calls[1]![0]; + expect(reprompt.path).toContain("b__faces-1"); + expect(reprompt.prompt.box).toBeDefined(); + const record = await readPrivacyRecord(workDir); + expect(record?.reviews).toEqual([]); + expect(record?.assets[other.id]?.checks).toEqual([ + expect.objectContaining({ kind: "mark", markId: mark.id }), + ]); + const report = await gate(harness, edl, workDir, prepared.sources.get(other.id)!.path); + expect(report.passed).toBe(true); + }); + + it("sends a low-confidence re-find to review, which blocks the export", async () => { + harness = await createPrivacyHarness((emulator) => { + const provider = Object.create(emulator) as MediaInferenceProvider; + provider.segmentPrompted = async (request) => { + const result = await emulator.segmentPrompted(request); + return request.path.includes("b__") ? { ...result, score: 0.3 } : result; + }; + return provider; + }); + const origin = await harness.addAsset("a__faces-1____person-alice__.mp4"); + const other = await harness.addAsset("b__faces-1____person-alice__.mp4"); + harness.addPerson("alice", "none"); + await harness.analyseFaces(origin.id, ["alice"]); + await harness.analyseFaces(other.id, ["alice"]); + const mark = await createMark(harness.deps, ACTOR, { + kind: "jewelry", + label: null, + personId: "alice", + assetId: origin.id, + prompt: tattoo, + treatment: "blur", + }); + const edl = testEdl([{ id: "c1", assetId: other.id, inMs: 0, outMs: 2000 }]); + const { workDir, prepared } = await render(harness, edl, "review"); + const report = await gate(harness, edl, workDir, prepared.sources.get(other.id)!.path); + expect(report.passed).toBe(false); + expect(report.issues).toContainEqual( + expect.objectContaining({ code: "mark_needs_review", markId: mark.id }), + ); + }); + + it("removes a person with their tracks, marks, and files", async () => { + harness = await createPrivacyHarness(); + const asset = await harness.addAsset("clip__faces-1____person-alice__.mp4"); + harness.addPerson("alice", "blur"); + await harness.analyseFaces(asset.id, ["alice"]); + await createMark(harness.deps, ACTOR, { + kind: "tattoo", + label: null, + personId: "alice", + assetId: asset.id, + prompt: tattoo, + treatment: "blur", + }); + const files = [ + path.join(harness.root, harness.store.tracks[0]!.keyframesPath), + path.join(harness.root, harness.store.marks[0]!.maskDir!), + ]; + const service = createMediaPrivacyService(harness.deps); + await service.removePerson(ACTOR, "alice"); + expect(harness.store.people).toEqual([]); + expect(harness.store.tracks).toEqual([]); + for (const file of files) await expect(stat(file)).rejects.toThrow(); + }); +}); diff --git a/packages/adapters/src/media-marks.ts b/packages/adapters/src/media-marks.ts new file mode 100644 index 0000000..99cde46 --- /dev/null +++ b/packages/adapters/src/media-marks.ts @@ -0,0 +1,322 @@ +import { mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import type { SegmentPromptedResult } from "@rakazo/adapter-kit"; +import type { + BoxKeyframe, + MarkTreatment, + MediaMark, + MediaMarkPrompt, + NormalizedBox, +} from "@rakazo/contracts"; +import type { CreateMarkInputType, MediaActor, MediaServiceDeps } from "./media-context.js"; +import type { PrivacyToolOptions } from "./media-privacy-io.js"; +import { + analysisFile, + inferenceCapabilities, + operationAvailable, + PrivacyUnavailableError, + probeSource, + resolveStored, + toStored, +} from "./media-privacy-io.js"; +import { extractLuma, temporalStability } from "./media-privacy-pixels.js"; +import type { TrackKeyframe } from "./media-privacy-plan.js"; +import { boxAt, FACE_MIN_CONFIDENCE, MARK_QUALITY_MIN, unionBox } from "./media-privacy-plan.js"; + +/* + * Identifying marks (spec 7): the user marks once by click or box, the + * segmenter (SAM2 in the sidecar) propagates the mask through the video, and + * the treatment is scored. A score under MARK_QUALITY_MIN never ships + * silently: the mark carries a suggested fallback and rendering applies it. + */ + +/** Side of the box a point prompt stands for, normalized. */ +const POINT_PROMPT_SIZE = 0.12; +/** Frames sampled for the local flicker measurement of an inpaint preview. */ +const STABILITY_FRAMES = 6; +const STABILITY_STEP_MS = 200; + +export type MarkDeps = Pick & + PrivacyToolOptions; + +/** Directory of a mark's masks and previews, relative to the media root. */ +export function markDir(spaceId: string, markId: string): string { + return path.join(spaceId, "marks", markId); +} + +/** The prompt as a box: a point prompt stands for a small box around it. */ +export function promptBox(prompt: MediaMarkPrompt): NormalizedBox { + if (prompt.box) return prompt.box; + if (!prompt.point) throw new PrivacyUnavailableError("a mark needs a box or a point"); + const x = Math.min(Math.max(0, prompt.point.x - POINT_PROMPT_SIZE / 2), 1 - POINT_PROMPT_SIZE); + const y = Math.min(Math.max(0, prompt.point.y - POINT_PROMPT_SIZE / 2), 1 - POINT_PROMPT_SIZE); + return { x, y, w: POINT_PROMPT_SIZE, h: POINT_PROMPT_SIZE }; +} + +/** Fallback to propose when a treatment scores too low. */ +export function fallbackFor(treatment: MarkTreatment): MarkTreatment { + return treatment === "cover" ? "reframe_out" : "cover"; +} + +/** + * Score a treatment. Segmentation confidence bounds every treatment (a mask + * that does not follow the mark cannot hide it); inpainting also needs the + * provider's temporal and edge quality and a local flicker check of its + * preview. Returns the score and, under MARK_QUALITY_MIN, the fallback. + */ +export async function scoreTreatment( + deps: MarkDeps, + input: { + treatment: MarkTreatment; + segmentationScore: number; + file: string; + maskDir: string; + keyframes: BoxKeyframe[]; + previewDir: string; + signal?: AbortSignal; + }, +): Promise<{ qualityScore: number; suggestedFallback: MarkTreatment | null }> { + let score = input.segmentationScore; + if (input.treatment === "inpaint") { + const capabilities = await inferenceCapabilities(deps.inference, input.signal); + if (!deps.inference || !operationAvailable(capabilities, "inpaintVideo")) { + return { qualityScore: 0, suggestedFallback: "cover" }; + } + await mkdir(input.previewDir, { recursive: true }); + const preview = await deps.inference.inpaintVideo({ + path: input.file, + maskDir: input.maskDir, + outPath: path.join(input.previewDir, "inpaint-preview.mp4"), + signal: input.signal, + }); + const local = await previewStability(deps, preview.outPath, input.keyframes, input.signal); + score = Math.min(score, preview.quality.score, preview.quality.temporalStability, local); + } + const qualityScore = Math.round(score * 1000) / 1000; + return { + qualityScore, + suggestedFallback: + qualityScore < MARK_QUALITY_MIN && input.treatment !== "none" + ? fallbackFor(input.treatment) + : null, + }; +} + +/** Flicker inside the mark region of an inpainted preview around the first keyframe. */ +async function previewStability( + deps: MarkDeps, + file: string, + keyframes: BoxKeyframe[], + signal?: AbortSignal, +): Promise { + const first = keyframes[0]; + if (!first) return 1; + const { width, height } = await frameSize(file, deps, signal); + const frames = []; + let region: NormalizedBox = first.box; + for (let i = 0; i < STABILITY_FRAMES; i++) { + const tMs = first.tMs + i * STABILITY_STEP_MS; + const box = boxAt(keyframes, tMs, STABILITY_STEP_MS); + if (box) region = unionBox(region, box); + try { + frames.push( + await extractLuma(file, { tMs, width, height }, { ffmpegPath: deps.ffmpegPath, signal }), + ); + } catch { + break; + } + } + return temporalStability(frames, region); +} + +async function frameSize(file: string, deps: PrivacyToolOptions, signal?: AbortSignal) { + const probed = await probeSource(file, "video", { ffprobePath: deps.ffprobePath, signal }); + return { width: probed.width, height: probed.height }; +} + +/** Segment a prompt and propagate it through the file into `outDir`. */ +export async function segmentMark( + deps: MarkDeps, + input: { file: string; prompt: MediaMarkPrompt; outDir: string; signal?: AbortSignal }, +): Promise { + const capabilities = await inferenceCapabilities(deps.inference, input.signal); + if (!deps.inference || !operationAvailable(capabilities, "segmentPrompted")) { + throw new PrivacyUnavailableError("mark segmentation needs the media worker"); + } + await rm(input.outDir, { recursive: true, force: true }); + await mkdir(input.outDir, { recursive: true }); + return deps.inference.segmentPrompted({ + path: input.file, + prompt: input.prompt, + propagate: true, + outDir: input.outDir, + signal: input.signal, + }); +} + +/** + * Where a person-bound mark sits relative to the person's face, from the + * frame it was drawn in: offsets and size in face-box units. + */ +export interface MarkAnchor { + dx: number; + dy: number; + w: number; + h: number; +} + +export function anchorFromFace(mark: NormalizedBox, face: NormalizedBox): MarkAnchor { + return { + dx: (mark.x - face.x) / face.w, + dy: (mark.y - face.y) / face.h, + w: mark.w / face.w, + h: mark.h / face.h, + }; +} + +/** Place an anchor at a face, clamped to the frame. Null when it falls outside. */ +export function placeAnchor(anchor: MarkAnchor, face: NormalizedBox): NormalizedBox | null { + const x0 = face.x + anchor.dx * face.w; + const y0 = face.y + anchor.dy * face.h; + const x1 = x0 + anchor.w * face.w; + const y1 = y0 + anchor.h * face.h; + const x = Math.max(0, x0); + const y = Math.max(0, y0); + const w = Math.min(1, x1) - x; + const h = Math.min(1, y1) - y; + return w > 0.001 && h > 0.001 ? { x, y, w, h } : null; +} + +/** First confident face keyframe of a track at or after `fromMs`. */ +export function firstConfident(keyframes: TrackKeyframe[], fromMs = 0): TrackKeyframe | null { + return ( + [...keyframes] + .filter((k) => k.tMs >= fromMs && (k.confidence ?? 1) >= FACE_MIN_CONFIDENCE) + .sort((a, b) => a.tMs - b.tMs)[0] ?? null + ); +} + +/** Face box of a track nearest to `tMs` (within 250 ms). */ +export function faceAt(keyframes: TrackKeyframe[], tMs: number): NormalizedBox | null { + let best: TrackKeyframe | null = null; + for (const k of keyframes) { + if (Math.abs(k.tMs - tMs) > 250) continue; + if (!best || Math.abs(k.tMs - tMs) < Math.abs(best.tMs - tMs)) best = k; + } + return best?.box ?? null; +} + +// --------------------------------------------------------------------------- +// Service operations +// --------------------------------------------------------------------------- + +/** + * Create a mark: store it, segment from the prompt, propagate through the + * asset, score the treatment, and store a fallback suggestion when the score + * is low. Without the media worker the mark is kept unsegmented; rendering a + * clip of that asset then fails closed until segmentation is possible. + */ +export async function createMark( + deps: MarkDeps, + actor: MediaActor, + input: CreateMarkInputType, + signal?: AbortSignal, +): Promise { + const asset = await deps.repos.getAssetRecord(actor.spaceId, input.assetId); + const mark = await deps.repos.createMark(actor.spaceId, { + kind: input.kind, + label: input.label, + personId: input.personId, + assetId: input.assetId, + prompt: input.prompt, + treatment: input.treatment, + }); + const file = await analysisFile(deps, actor.spaceId, asset); + const capabilities = await inferenceCapabilities(deps.inference, signal); + if (!file || !operationAvailable(capabilities, "segmentPrompted")) return mark; + const relative = markDir(actor.spaceId, mark.id); + const outDir = await resolveStored(deps.mediaRoot, path.join(relative, "masks")); + try { + const segmented = await segmentMark(deps, { file, prompt: input.prompt, outDir, signal }); + const scored = await scoreTreatment(deps, { + treatment: input.treatment, + segmentationScore: segmented.score, + file, + maskDir: segmented.maskDir, + keyframes: segmented.keyframes, + previewDir: await resolveStored(deps.mediaRoot, relative), + signal, + }); + const record = await deps.repos.recordMarkResult(actor.spaceId, mark.id, { + ...scored, + maskDir: await toStored(deps.mediaRoot, segmented.maskDir), + }); + const { maskDir: _maskDir, ...dto } = record; + return dto; + } catch (error) { + // The mark stays without a mask (rendering fails closed); the caller sees why. + await deps.repos.removeMark(actor.spaceId, mark.id); + await rm(await resolveStored(deps.mediaRoot, relative), { recursive: true, force: true }); + throw error; + } +} + +/** + * Update treatment, label, or person. A new treatment is segmented and scored + * again so its suggestion reflects that treatment. + */ +export async function updateMark( + deps: MarkDeps, + actor: MediaActor, + input: { + markId: string; + treatment?: MarkTreatment; + label?: string | null; + personId?: string | null; + }, + signal?: AbortSignal, +): Promise { + const before = await deps.repos.getMark(actor.spaceId, input.markId); + const updated = await deps.repos.updateMark(actor.spaceId, input.markId, { + ...(input.treatment !== undefined ? { treatment: input.treatment } : {}), + ...(input.label !== undefined ? { label: input.label } : {}), + ...(input.personId !== undefined ? { personId: input.personId } : {}), + }); + if (input.treatment === undefined || input.treatment === before.treatment) return updated; + if (!before.assetId || !before.prompt) return updated; + const asset = await deps.repos.getAssetRecord(actor.spaceId, before.assetId); + const file = await analysisFile(deps, actor.spaceId, asset); + const capabilities = await inferenceCapabilities(deps.inference, signal); + if (!file || !operationAvailable(capabilities, "segmentPrompted")) { + // Without the worker the old score says nothing about the new treatment. + const record = await deps.repos.recordMarkResult(actor.spaceId, input.markId, { + qualityScore: null, + suggestedFallback: null, + maskDir: before.maskDir, + }); + const { maskDir: _maskDir, ...dto } = record; + return dto; + } + const relative = markDir(actor.spaceId, input.markId); + const segmented = await segmentMark(deps, { + file, + prompt: before.prompt, + outDir: await resolveStored(deps.mediaRoot, path.join(relative, "masks")), + signal, + }); + const scored = await scoreTreatment(deps, { + treatment: input.treatment, + segmentationScore: segmented.score, + file, + maskDir: segmented.maskDir, + keyframes: segmented.keyframes, + previewDir: await resolveStored(deps.mediaRoot, relative), + signal, + }); + const record = await deps.repos.recordMarkResult(actor.spaceId, input.markId, { + ...scored, + maskDir: await toStored(deps.mediaRoot, segmented.maskDir), + }); + const { maskDir: _maskDir, ...dto } = record; + return dto; +} diff --git a/packages/adapters/src/media-privacy-fixtures.ts b/packages/adapters/src/media-privacy-fixtures.ts new file mode 100644 index 0000000..4c86ec2 --- /dev/null +++ b/packages/adapters/src/media-privacy-fixtures.ts @@ -0,0 +1,437 @@ +import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { MediaInferenceProvider } from "@rakazo/adapter-kit"; +import type { + AnonymizeMode, + Edl, + FfprobeReport, + MarkTreatment, + MediaMarkKind, + MediaMarkPrompt, + PlatformProfile, +} from "@rakazo/contracts"; +import { EdlSchema, PlatformProfileSchema } from "@rakazo/contracts"; +import type { MediaRepos } from "@rakazo/db"; +import { edlDurationMs, encodeVectors } from "@rakazo/media"; +import type { MediaServiceDeps } from "./media-context.js"; +import { EmulatorMediaInferenceProvider } from "./media-emulator.js"; +import { renderTestClip } from "./media-emulator-clips.js"; +import { runMediaTool } from "./media-folder-source-probe.js"; + +/* + * In-memory repositories and synthetic ffmpeg clips for the privacy tests. + * Clip names carry media-emulator markers (faces, person seeds, detection + * gaps, classifier scores); nothing here shows a real person. + */ + +export const SPACE = "space-1"; +export const USER = "user-1"; +export const ACTOR = { spaceId: SPACE, userId: USER }; + +interface AssetRow { + id: string; + spaceId: string; + sourceId: string; + externalId: string; + kind: string; + storage: string; + storagePath: string | null; + proxyPath: string | null; +} + +interface PersonRow { + id: string; + spaceId: string; + displayName: string; + anonymizeByDefault: string; + consentRecordedAt: Date | null; + ageVerifiedAt: Date | null; + ageVerificationMethod: string; + syntheticSeed: number | null; + centroid: Uint8Array | null; +} + +interface TrackRow { + id: string; + assetId: string; + personId: string | null; + startMs: number; + endMs: number; + confidenceMin: number; + keyframesPath: string; + modelVersion: string; +} + +interface MarkRow { + id: string; + kind: MediaMarkKind; + label: string | null; + personId: string | null; + assetId: string | null; + prompt: MediaMarkPrompt | null; + treatment: MarkTreatment; + qualityScore: number | null; + suggestedFallback: MarkTreatment | null; + maskDir: string | null; + createdAt: string; +} + +export interface PrivacyHarness { + root: string; + inference: EmulatorMediaInferenceProvider; + deps: MediaServiceDeps; + store: { + assets: AssetRow[]; + people: PersonRow[]; + tracks: TrackRow[]; + marks: MarkRow[]; + renders: Array>; + plans: Array>; + segments: Array>; + }; + /** A synthetic clip, or a still (PNG) when the name ends in .png. */ + addAsset(fileName: string, options?: { seconds?: number; id?: string }): Promise; + addPerson( + id: string, + mode: AnonymizeMode, + consent?: { consent?: boolean; age?: boolean }, + ): PersonRow; + /** Run emulator tracking on the asset and store the tracks (the analysis result). */ + analyseFaces(assetId: string, personIds: Array, sampleFps?: number): Promise; + /** Store the emulator identity embedding of a person seed as the person's centroid. */ + learnPerson(personId: string, assetId: string, trackIndex?: number): Promise; + workDir(name: string): Promise; + cleanup(): Promise; +} + +export function testEdl( + clips: Array<{ id: string; assetId: string; inMs: number; outMs: number; atMs?: number }>, + extra: Partial> & { width?: number; height?: number } = {}, +): Edl { + let cursor = 0; + const video = clips.map((clip) => { + const atMs = clip.atMs ?? cursor; + cursor = atMs + clip.outMs - clip.inMs; + return { ...clip, atMs, reframe: "none" as const }; + }); + return EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: "test-profile", + width: extra.width ?? 320, + height: extra.height ?? 180, + fps: 25, + durationTargetMs: cursor, + durationHardMinMs: 0, + durationHardMaxMs: 600_000, + }, + tracks: { video }, + redactions: extra.redactions ?? {}, + metadata: {}, + }); +} + +export function testProfile(overrides: Record = {}): PlatformProfile { + return PlatformProfileSchema.parse({ + id: "test-profile", + name: "Test profile", + kind: "free_social", + reviewedAt: "2026-01-01", + video: { + aspectRatios: ["16:9", "9:16"], + width: 1920, + height: 1920, + fps: 60, + codec: "h264", + maxBitrateKbps: 20_000, + maxFileSizeMb: 500, + }, + duration: { hardMinMs: 0, hardMaxMs: 600_000 }, + safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 }, + captions: { recommended: false }, + audio: {}, + text: { captionMaxChars: 2200, hashtagLimit: 30, linkPolicy: "allowed" }, + content: { thresholds: { explicit_nudity: 0.5 } }, + disclosure: { aiAlteredLabelRequired: false }, + consent: { requirePerformerConsent: false, requireAgeVerification: false }, + export: {}, + ...overrides, + }); +} + +/** An ffprobe report that passes the technical check for an EDL. */ +export function cleanProbe(edl: Edl): FfprobeReport { + return { + durationMs: edlDurationMs(edl), + width: edl.output.width, + height: edl.output.height, + fps: edl.output.fps, + videoCodec: "h264", + audioCodec: null, + sizeBytes: 10_000, + integratedLufs: null, + truePeakDb: null, + tags: {}, + }; +} + +function isolation(): Error { + return new Error("not found in space"); +} + +export async function createPrivacyHarness( + inferenceOverride?: (emulator: EmulatorMediaInferenceProvider) => MediaInferenceProvider | null, +): Promise { + const root = await realpath(await mkdtemp(path.join(tmpdir(), "media-privacy-"))); + const emulator = new EmulatorMediaInferenceProvider({ root }); + const store: PrivacyHarness["store"] = { + assets: [], + people: [], + tracks: [], + marks: [], + renders: [], + plans: [], + segments: [], + }; + let counter = 0; + const nextId = (prefix: string) => `${prefix}${++counter}`; + const markDto = (row: MarkRow) => { + const { maskDir: _maskDir, ...dto } = row; + return dto; + }; + const findMark = (id: string) => { + const row = store.marks.find((mark) => mark.id === id); + if (!row) throw isolation(); + return row; + }; + const repos = { + async listAssetRecords(_spaceId: string, ids: string[]) { + return store.assets.filter((asset) => ids.includes(asset.id)); + }, + async getAssetRecord(_spaceId: string, id: string) { + const asset = store.assets.find((row) => row.id === id); + if (!asset) throw isolation(); + return asset; + }, + async getSource() { + return { kind: "folder", config: {} }; + }, + async getProjectRecord(_spaceId: string, projectId: string) { + return { + id: projectId, + brandKitId: null, + assetIds: store.assets.map((a) => a.id), + }; + }, + async getBrandKit() { + throw isolation(); + }, + async listPersonRecords(_spaceId: string, ids?: string[]) { + return store.people.filter((person) => !ids || ids.includes(person.id)); + }, + async personCentroids() { + return store.people.flatMap((person) => + person.centroid ? [{ personId: person.id, centroid: person.centroid }] : [], + ); + }, + async updatePerson(_spaceId: string, personId: string, patch: { syntheticSeed?: number }) { + const person = store.people.find((row) => row.id === personId); + if (!person) throw isolation(); + if (patch.syntheticSeed !== undefined) person.syntheticSeed = patch.syntheticSeed; + return person; + }, + async faceTrackRecords(_spaceId: string, assetIds: string[]) { + return store.tracks.filter((track) => assetIds.includes(track.assetId)); + }, + async listFaceTracks() { + return store.tracks; + }, + async latestAnalysis(_spaceId: string, assetId: string, kind: string) { + return kind === "faces" && store.tracks.some((t) => t.assetId === assetId) ? { kind } : null; + }, + async marksForAsset(_spaceId: string, assetId: string) { + const peopleHere = new Set( + store.tracks.filter((t) => t.assetId === assetId && t.personId).map((t) => t.personId), + ); + return store.marks.filter( + (mark) => mark.assetId === assetId || (mark.personId && peopleHere.has(mark.personId)), + ); + }, + async createMark( + _spaceId: string, + input: Omit, + ) { + const row: MarkRow = { + ...input, + id: nextId("mark"), + qualityScore: null, + suggestedFallback: null, + maskDir: null, + createdAt: new Date(0).toISOString(), + }; + store.marks.push(row); + return markDto(row); + }, + async getMark(_spaceId: string, id: string) { + return { ...findMark(id) }; + }, + async updateMark(_spaceId: string, id: string, patch: Partial) { + Object.assign(findMark(id), patch); + return markDto(findMark(id)); + }, + async recordMarkResult( + _spaceId: string, + id: string, + outcome: Pick, + ) { + Object.assign(findMark(id), outcome); + return { ...findMark(id) }; + }, + async removeMark(_spaceId: string, id: string) { + const row = findMark(id); + store.marks = store.marks.filter((mark) => mark.id !== id); + return { maskDir: row.maskDir }; + }, + async removePersonCompletely(_spaceId: string, personId: string) { + const tracks = store.tracks.filter((t) => t.personId === personId); + const marks = store.marks.filter((m) => m.personId === personId); + store.tracks = store.tracks.filter((t) => t.personId !== personId); + store.marks = store.marks.filter((m) => m.personId !== personId); + store.people = store.people.filter((p) => p.id !== personId); + return { + deletedFaceTracks: tracks.length, + deletedMarks: marks.length, + keyframesPaths: tracks.map((t) => t.keyframesPath), + maskDirs: marks.flatMap((m) => (m.maskDir ? [m.maskDir] : [])), + }; + }, + async getRenderRecord(_spaceId: string, id: string) { + const render = store.renders.find((row) => row.id === id); + if (!render) throw isolation(); + return render; + }, + async getPlan(_spaceId: string, id: string) { + const plan = store.plans.find((row) => row.id === id); + if (!plan) throw isolation(); + return plan; + }, + async listSegments() { + return store.segments; + }, + async updateRender(_spaceId: string, id: string, patch: Record) { + const render = store.renders.find((row) => row.id === id); + if (!render) return null; + Object.assign(render, patch); + return render; + }, + }; + + const libraryDir = path.join(root, SPACE, "library"); + await mkdir(libraryDir, { recursive: true }); + const deps = { + repos: repos as unknown as MediaRepos, + inference: inferenceOverride ? inferenceOverride(emulator) : emulator, + mediaRoot: root, + sources: {}, + now: () => new Date("2026-06-01T12:00:00Z"), + } as unknown as MediaServiceDeps; + + const harness: PrivacyHarness = { + root, + inference: emulator, + deps, + store, + async addAsset(fileName, options = {}) { + const file = path.join(libraryDir, fileName); + const still = fileName.endsWith(".png"); + if (still) { + await runMediaTool("ffmpeg", [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:r=1:d=1", + "-frames:v", + "1", + file, + ]); + } else { + await renderTestClip(file, { seconds: options.seconds ?? 2 }); + } + const row: AssetRow = { + id: options.id ?? nextId("asset"), + spaceId: SPACE, + sourceId: "source-1", + externalId: fileName, + kind: still ? "photo" : "video", + storage: "uploaded", + storagePath: path.relative(root, file), + proxyPath: null, + }; + store.assets.push(row); + return row; + }, + addPerson(id, mode, consent = {}) { + const at = new Date("2026-01-01T00:00:00Z"); + const row: PersonRow = { + id, + spaceId: SPACE, + displayName: id, + anonymizeByDefault: mode, + consentRecordedAt: consent.consent ? at : null, + ageVerifiedAt: consent.age ? at : null, + ageVerificationMethod: consent.age ? "document_on_file" : "none", + syntheticSeed: null, + centroid: null, + }; + store.people.push(row); + return row; + }, + async analyseFaces(assetId, personIds, sampleFps = 10) { + const asset = store.assets.find((row) => row.id === assetId)!; + const tracked = await emulator.trackFaces({ + path: path.join(root, asset.storagePath!), + sampleFps, + }); + const dir = path.join(root, SPACE, "faces", assetId); + await mkdir(dir, { recursive: true }); + for (const [index, track] of tracked.tracks.entries()) { + const file = path.join(dir, `${track.trackId}.json`); + await writeFile(file, JSON.stringify({ keyframes: track.keyframes })); + const times = track.keyframes.map((k) => k.tMs); + store.tracks.push({ + id: nextId("track"), + assetId, + personId: personIds[index] ?? null, + startMs: Math.min(...times), + endMs: Math.max(...times), + confidenceMin: 0.9, + keyframesPath: path.relative(root, file), + modelVersion: "emulator", + }); + } + }, + async learnPerson(personId, assetId, trackIndex = 0) { + const asset = store.assets.find((row) => row.id === assetId)!; + const tracked = await emulator.trackFaces({ + path: path.join(root, asset.storagePath!), + sampleFps: 2, + }); + const embedding = tracked.tracks[trackIndex]?.embedding; + const person = store.people.find((row) => row.id === personId)!; + person.centroid = embedding ? encodeVectors([embedding]) : null; + }, + async workDir(name) { + const dir = path.join(root, SPACE, "renders", name); + await mkdir(dir, { recursive: true }); + return dir; + }, + async cleanup() { + await rm(root, { recursive: true, force: true }); + }, + }; + return harness; +} diff --git a/packages/adapters/src/media-privacy-io.ts b/packages/adapters/src/media-privacy-io.ts new file mode 100644 index 0000000..b817d82 --- /dev/null +++ b/packages/adapters/src/media-privacy-io.ts @@ -0,0 +1,264 @@ +import { constants } from "node:fs"; +import { copyFile, mkdir, readdir, readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import type { MediaInferenceProvider } from "@rakazo/adapter-kit"; +import type { + BoxKeyframe, + MediaCapabilities, + MediaInferenceOperation, + MediaRenderQuality, +} from "@rakazo/contracts"; +import type { MediaRepos } from "@rakazo/db"; +import { assertInside, resolveInside } from "@rakazo/media/node"; +import type { MediaServiceDeps } from "./media-context.js"; +import { probeMedia, runMediaTool } from "./media-folder-source-probe.js"; +import { maskBounds, readPngLuma } from "./media-privacy-pixels.js"; +import type { PixelBox, TrackKeyframe } from "./media-privacy-plan.js"; +import { parseStoredKeyframes } from "./media-privacy-plan.js"; + +/* + * File and tool helpers shared by anonymization, marks, and the gate. Stored + * media paths are relative to the media root; every resolution goes through + * the root checks of @rakazo/media/node. ffmpeg runs as an argv array only. + */ + +/** A redaction or check that cannot be carried out. Rendering stops instead of exporting. */ +export class PrivacyUnavailableError extends Error { + constructor(message: string) { + super(message); + this.name = "PrivacyUnavailableError"; + } +} + +export interface PrivacyToolOptions { + ffmpegPath?: string; + ffprobePath?: string; +} + +/** Absolute path of a stored (root-relative) or already absolute path inside the media root. */ +export async function resolveStored(root: string, stored: string): Promise { + return path.isAbsolute(stored) ? assertInside(root, stored) : resolveInside(root, stored); +} + +export async function isInsideRoot(root: string, file: string): Promise { + try { + await assertInside(root, file); + return true; + } catch { + return false; + } +} + +/** The path of a file relative to the media root, for storing. */ +export async function toStored(root: string, absolute: string): Promise { + const inside = await assertInside(root, absolute); + return path.relative(await realpath(root), inside); +} + +type AssetRow = Awaited>; + +/** + * The file a render reads for an asset: the 720p proxy for drafts (when one + * exists), otherwise the original. Originals still on a device cannot render. + */ +export async function resolveAssetFile( + deps: Pick, + spaceId: string, + asset: AssetRow, + quality: MediaRenderQuality, +): Promise { + if (quality === "draft" && asset.proxyPath) return resolveStored(deps.mediaRoot, asset.proxyPath); + return resolveOriginal(deps, spaceId, asset); +} + +export async function resolveOriginal( + deps: Pick, + spaceId: string, + asset: AssetRow, +): Promise { + if (asset.storage === "uploaded") { + if (!asset.storagePath) throw new PrivacyUnavailableError(`asset ${asset.id} has no file yet`); + return resolveStored(deps.mediaRoot, asset.storagePath); + } + if (asset.storage === "referenced") { + const source = await deps.repos.getSource(spaceId, asset.sourceId); + const provider = deps.sources[source.kind as "folder" | "emulator"]; + if (!provider) throw new PrivacyUnavailableError(`source of asset ${asset.id} is unavailable`); + return provider.resolve(source.config, asset.externalId); + } + throw new PrivacyUnavailableError(`asset ${asset.id} is still on a device; upload it first`); +} + +/** + * A file inference can read: the proxy when it exists (normalized coordinates + * and the timeline match the original), else the original when it lies inside + * the media root. Null when neither does. + */ +export async function analysisFile( + deps: Pick, + spaceId: string, + asset: AssetRow, +): Promise { + if (asset.proxyPath) return resolveStored(deps.mediaRoot, asset.proxyPath); + try { + const original = await resolveOriginal(deps, spaceId, asset); + return (await isInsideRoot(deps.mediaRoot, original)) ? original : null; + } catch { + return null; + } +} + +export interface ProbedSource { + width: number; + height: number; + durationMs: number; + fps: number; + hasAudio: boolean; + rotation: 0 | 90 | 180 | 270; + still: boolean; +} + +/** Display size (rotation applied), duration, and audio of a source file. */ +export async function probeSource( + file: string, + kind: string, + options: PrivacyToolOptions & { signal?: AbortSignal } = {}, +): Promise { + const probed = await probeMedia(file, { + ffprobePath: options.ffprobePath, + signal: options.signal, + }); + if (!probed.hasVideo || !probed.width || !probed.height) + throw new PrivacyUnavailableError("source has no readable video stream"); + const still = kind === "photo" || !probed.durationMs; + const rotation = (((Math.round(probed.rotation / 90) * 90) % 360) + 360) % 360; + return { + width: probed.width, + height: probed.height, + durationMs: still ? 0 : (probed.durationMs ?? 0), + fps: probed.fps ?? 30, + hasAudio: probed.hasAudio, + rotation: rotation as ProbedSource["rotation"], + still, + }; +} + +/** Copy a file into `dir` under its own name (reflink when the filesystem allows). */ +export async function copyInto(file: string, dir: string): Promise { + await mkdir(dir, { recursive: true }); + const target = path.join(dir, path.basename(file)); + await copyFile(file, target, constants.COPYFILE_FICLONE); + return target; +} + +/** Keyframes of a stored face track file (relative to the media root). */ +export async function readTrackKeyframes(root: string, stored: string): Promise { + const file = await resolveStored(root, stored); + return parseStoredKeyframes(JSON.parse(await readFile(file, "utf8"))); +} + +/** Mask frame times of a mask directory, ascending. */ +export async function maskTimes(maskDir: string): Promise { + const names = await readdir(maskDir); + return names + .filter((name) => /^\d+\.png$/.test(name)) + .map((name) => Number.parseInt(name, 10)) + .sort((a, b) => a - b); +} + +/** Bounding box per mask frame. Empty masks are skipped. */ +export async function maskKeyframes(maskDir: string): Promise { + const keyframes: BoxKeyframe[] = []; + for (const tMs of await maskTimes(maskDir)) { + const box = maskBounds(await readPngLuma(path.join(maskDir, `${tMs}.png`))); + if (box) keyframes.push({ tMs, box }); + } + return keyframes; +} + +/** Whether an inference operation is available; synthetic identity also needs a commercial license. */ +export function operationAvailable( + capabilities: MediaCapabilities | null, + operation: MediaInferenceOperation, +): boolean { + const entry = capabilities?.operations[operation]; + if (!entry?.available) return false; + if (operation === "synthesizeIdentity") return entry.commercialUse === true; + return true; +} + +export async function inferenceCapabilities( + inference: MediaInferenceProvider | null, + signal?: AbortSignal, +): Promise { + return inference ? inference.capabilities(signal) : null; +} + +/** + * Lossless static crop of a video (or a still to PNG). Audio is copied; all + * container metadata is dropped. + */ +export async function cropFile( + input: string, + output: string, + crop: PixelBox, + still: boolean, + options: PrivacyToolOptions & { signal?: AbortSignal } = {}, +): Promise { + const filter = `crop=${crop.w}:${crop.h}:${crop.x}:${crop.y}`; + const args = still + ? [ + "-v", + "error", + "-y", + "-i", + input, + "-vf", + filter, + "-frames:v", + "1", + "-map_metadata", + "-1", + output, + ] + : [ + "-v", + "error", + "-y", + "-i", + input, + "-map", + "0:v:0", + "-map", + "0:a?", + "-vf", + filter, + "-c:v", + "libx264", + "-qp", + "0", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-c:a", + "copy", + "-map_metadata", + "-1", + output, + ]; + await runMediaTool(options.ffmpegPath ?? "ffmpeg", args, { signal: options.signal }); +} + +/** First frame of a video as PNG (redaction providers write videos, stills render from images). */ +export async function firstFrame( + input: string, + output: string, + options: PrivacyToolOptions & { signal?: AbortSignal } = {}, +): Promise { + await runMediaTool( + options.ffmpegPath ?? "ffmpeg", + ["-v", "error", "-y", "-i", input, "-frames:v", "1", "-map_metadata", "-1", output], + { signal: options.signal }, + ); +} diff --git a/packages/adapters/src/media-privacy-pixels.test.ts b/packages/adapters/src/media-privacy-pixels.test.ts new file mode 100644 index 0000000..be04c72 --- /dev/null +++ b/packages/adapters/src/media-privacy-pixels.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "vitest"; +import { encodeMaskPng } from "./media-emulator.js"; +import type { LumaFrame } from "./media-privacy-pixels.js"; +import { + controlAgreement, + decodePngLuma, + maskBounds, + residualVisibility, + temporalStability, +} from "./media-privacy-pixels.js"; + +/** A 64x64 checkerboard of 4 px squares: detail everywhere. */ +function checker(width = 64, height = 64, shift = 0): LumaFrame { + const luma = new Uint8Array(width * height); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + luma[y * width + x] = (Math.floor((x + shift) / 4) + Math.floor(y / 4)) % 2 ? 220 : 30; + } + } + return { width, height, luma }; +} + +function fill( + frame: LumaFrame, + box: { x: number; y: number; w: number; h: number }, + value: number, +) { + const out = { ...frame, luma: frame.luma.slice() }; + for ( + let y = Math.floor(box.y * frame.height); + y < Math.ceil((box.y + box.h) * frame.height); + y++ + ) { + for ( + let x = Math.floor(box.x * frame.width); + x < Math.ceil((box.x + box.w) * frame.width); + x++ + ) { + out.luma[y * frame.width + x] = value; + } + } + return out; +} + +const region = { x: 0.25, y: 0.25, w: 0.5, h: 0.5 }; + +describe("PNG masks", () => { + it("decodes the masks the segmenter writes", () => { + const mask = decodePngLuma(encodeMaskPng(40, 20, { x: 0.5, y: 0.25, w: 0.25, h: 0.5 })); + expect(mask).toMatchObject({ width: 40, height: 20 }); + expect(maskBounds(mask)).toEqual({ x: 0.5, y: 0.25, w: 0.25, h: 0.5 }); + expect(maskBounds(decodePngLuma(encodeMaskPng(8, 8, { x: 0, y: 0, w: 0, h: 0 })))).toBeNull(); + expect(() => decodePngLuma(Buffer.from("nope"))).toThrow("not a PNG"); + }); +}); + +describe("residual visibility", () => { + it("is 1 for untouched detail and 0 for a filled or flat region", () => { + const source = checker(); + expect(residualVisibility({ source, treated: source, box: region })).toBe(1); + expect(residualVisibility({ source, treated: fill(source, region, 128), box: region })).toBe(0); + const flat = fill(source, region, 90); + expect( + residualVisibility({ source: flat, treated: flat, box: { x: 0.3, y: 0.3, w: 0.4, h: 0.4 } }), + ).toBe(0); + }); + + it("counts only the part a crop still shows and only mask pixels", () => { + const source = checker(); + // The treated frame is the right half of the source, untouched. + const half: LumaFrame = { width: 32, height: 64, luma: new Uint8Array(32 * 64) }; + for (let y = 0; y < 64; y++) + half.luma.set(source.luma.subarray(y * 64 + 32, y * 64 + 64), y * 32); + const left = { x: 0, y: 0.25, w: 0.4, h: 0.5 }; + expect(residualVisibility({ source, treated: half, box: left, offset: { x: 32, y: 0 } })).toBe( + 0, + ); + expect( + residualVisibility({ source, treated: half, box: region, offset: { x: 32, y: 0 } }), + ).toBe(1); + const mask = decodePngLuma(encodeMaskPng(64, 64, { x: 0.5, y: 0.25, w: 0.25, h: 0.5 })); + const treated = fill(source, { x: 0.25, y: 0.25, w: 0.25, h: 0.5 }, 128); + expect(residualVisibility({ source, treated, box: region, mask })).toBe(1); + }); + + it("detects frames that are not the same moment", () => { + const source = checker(); + const treated = fill(source, region, 128); + expect(controlAgreement({ source, treated, boxes: [region] })).toBe(1); + expect(controlAgreement({ source, treated: checker(64, 64, 2), boxes: [region] })).toBeLessThan( + 0.8, + ); + }); +}); + +describe("temporal stability", () => { + it("drops when the treated region flickers more than the frame", () => { + const a = checker(); + expect(temporalStability([a, a, a], region)).toBe(1); + const flicker = [fill(a, region, 0), fill(a, region, 255), fill(a, region, 0)]; + expect(temporalStability(flicker, region)).toBe(0); + expect(temporalStability([a], region)).toBe(1); + }); +}); diff --git a/packages/adapters/src/media-privacy-pixels.ts b/packages/adapters/src/media-privacy-pixels.ts new file mode 100644 index 0000000..0a94b79 --- /dev/null +++ b/packages/adapters/src/media-privacy-pixels.ts @@ -0,0 +1,310 @@ +import { readFile } from "node:fs/promises"; +import { inflateSync } from "node:zlib"; +import type { NormalizedBox } from "@rakazo/contracts"; +import { runMediaTool } from "./media-folder-source-probe.js"; + +/* + * Pixel checks for the privacy gate and mark quality. Video frames are decoded + * by ffmpeg straight to 8-bit luma; mask PNGs from the segmenter go through the + * decoder below (8-bit gray, gray+alpha, RGB, RGBA; not interlaced, not palette). + */ + +export interface LumaFrame { + width: number; + height: number; + /** One luma byte per pixel, row-major. */ + luma: Uint8Array; +} + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const CHANNELS: Record = { 0: 1, 2: 3, 4: 2, 6: 4 }; + +function paeth(a: number, b: number, c: number): number { + const p = a + b - c; + const pa = Math.abs(p - a); + const pb = Math.abs(p - b); + const pc = Math.abs(p - c); + if (pa <= pb && pa <= pc) return a; + return pb <= pc ? b : c; +} + +/** Decode a PNG to luma (Rec. 601 weights). Throws on anything it cannot read exactly. */ +export function decodePngLuma(png: Buffer): LumaFrame { + if (png.length < 8 || !png.subarray(0, 8).equals(PNG_SIGNATURE)) throw new Error("not a PNG"); + let offset = 8; + let width = 0; + let height = 0; + let channels = 0; + const data: Buffer[] = []; + while (offset + 8 <= png.length) { + const length = png.readUInt32BE(offset); + const type = png.toString("ascii", offset + 4, offset + 8); + const body = png.subarray(offset + 8, offset + 8 + length); + offset += 12 + length; + if (type === "IHDR") { + width = body.readUInt32BE(0); + height = body.readUInt32BE(4); + const depth = body[8]; + const colorType = body[9] ?? -1; + const interlace = body[12]; + channels = CHANNELS[colorType] ?? 0; + if (depth !== 8 || channels === 0 || interlace !== 0) + throw new Error("unsupported PNG layout"); + } else if (type === "IDAT") { + data.push(body); + } else if (type === "IEND") { + break; + } + } + if (width === 0 || height === 0 || channels === 0) throw new Error("PNG has no image header"); + const raw = inflateSync(Buffer.concat(data)); + const stride = width * channels; + if (raw.length < (stride + 1) * height) throw new Error("PNG data is truncated"); + const pixels = Buffer.alloc(stride * height); + for (let y = 0; y < height; y++) { + const filter = raw[y * (stride + 1)]; + const line = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1)); + const row = y * stride; + for (let x = 0; x < stride; x++) { + const left = x >= channels ? (pixels[row + x - channels] ?? 0) : 0; + const up = y > 0 ? (pixels[row - stride + x] ?? 0) : 0; + const upLeft = y > 0 && x >= channels ? (pixels[row - stride + x - channels] ?? 0) : 0; + const value = line[x] ?? 0; + let out: number; + if (filter === 0) out = value; + else if (filter === 1) out = value + left; + else if (filter === 2) out = value + up; + else if (filter === 3) out = value + ((left + up) >> 1); + else if (filter === 4) out = value + paeth(left, up, upLeft); + else throw new Error("unknown PNG filter"); + pixels[row + x] = out & 0xff; + } + } + const luma = new Uint8Array(width * height); + for (let i = 0; i < width * height; i++) { + const base = i * channels; + if (channels <= 2) { + luma[i] = pixels[base] ?? 0; + } else { + const r = pixels[base] ?? 0; + const g = pixels[base + 1] ?? 0; + const b = pixels[base + 2] ?? 0; + luma[i] = Math.round(0.299 * r + 0.587 * g + 0.114 * b); + } + } + return { width, height, luma }; +} + +export async function readPngLuma(file: string): Promise { + return decodePngLuma(await readFile(file)); +} + +/** + * Decode the frame shown at `tMs` as luma, scaled to `width` x `height` with + * nearest-neighbour sampling (a no-op at the display size). Throws unless + * ffmpeg returns a full frame. + */ +export async function extractLuma( + file: string, + input: { tMs: number; width: number; height: number; still?: boolean }, + options: { ffmpegPath?: string; signal?: AbortSignal } = {}, +): Promise { + const seek = input.still ? [] : ["-ss", (Math.max(0, input.tMs) / 1000).toFixed(3)]; + const { stdout } = await runMediaTool( + options.ffmpegPath ?? "ffmpeg", + [ + "-v", + "error", + ...seek, + "-i", + file, + "-frames:v", + "1", + "-vf", + `scale=${input.width}:${input.height}:flags=neighbor`, + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "-", + ], + { signal: options.signal }, + ); + const size = input.width * input.height; + if (stdout.length < size) throw new Error(`no frame at ${input.tMs} ms`); + return { + width: input.width, + height: input.height, + luma: new Uint8Array(stdout.subarray(0, size)), + }; +} + +/** Bounding box of the bright (>= 128) pixels of a mask, or null when the mask is empty. */ +export function maskBounds(mask: LumaFrame): NormalizedBox | null { + let minX = mask.width; + let minY = mask.height; + let maxX = -1; + let maxY = -1; + for (let y = 0; y < mask.height; y++) { + for (let x = 0; x < mask.width; x++) { + if ((mask.luma[y * mask.width + x] ?? 0) < 128) continue; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + if (maxX < 0) return null; + return { + x: minX / mask.width, + y: minY / mask.height, + w: (maxX + 1 - minX) / mask.width, + h: (maxY + 1 - minY) / mask.height, + }; +} + +/** A luma step at least this large marks a detail pixel (an edge someone could recognise). */ +export const DETAIL_MIN_STEP = 12; +/** A detail pixel within this many luma levels of the source counts as unchanged. */ +export const UNCHANGED_MAX_DIFF = 2; +/** Looser tolerance for the control area outside treated regions (codec noise). */ +const CONTROL_MAX_DIFF = 6; +/** Regions with fewer detail pixels show nothing identifying. */ +const MIN_DETAIL_PIXELS = 16; + +function pixelRange(box: NormalizedBox, width: number, height: number) { + return { + x0: Math.max(0, Math.floor(box.x * width)), + y0: Math.max(0, Math.floor(box.y * height)), + x1: Math.min(width, Math.ceil((box.x + box.w) * width)), + y1: Math.min(height, Math.ceil((box.y + box.h) * height)), + }; +} + +function detailStep(frame: LumaFrame, x: number, y: number): number { + const at = y * frame.width + x; + const value = frame.luma[at] ?? 0; + const right = x + 1 < frame.width ? (frame.luma[at + 1] ?? value) : value; + const down = y + 1 < frame.height ? (frame.luma[at + frame.width] ?? value) : value; + return Math.abs(value - right) + Math.abs(value - down); +} + +/** + * Residual visibility of a region (0..1): the share of the source's detail + * pixels (edges) that the treated frame left unchanged, value and edge + * strength alike. Blur, pixelation, fills, and inpainting all rewrite edges, + * so a treated region scores near 0, an untouched one 1. A flat region + * without detail scores 0: there is nothing to recognise. + * + * `source` and `treated` are pixel-aligned at full resolution; when `treated` + * is a crop of the source, `offset` is the crop origin in source pixels and + * detail outside the crop is invisible. Optional `mask` (normalized over the + * source frame) limits the region to mask pixels. + */ +export function residualVisibility(input: { + source: LumaFrame; + treated: LumaFrame; + box: NormalizedBox; + offset?: { x: number; y: number }; + mask?: LumaFrame; +}): number { + const { source, treated, box, mask } = input; + const offset = input.offset ?? { x: 0, y: 0 }; + const { x0, y0, x1, y1 } = pixelRange(box, source.width, source.height); + let detail = 0; + let unchanged = 0; + for (let y = y0; y < y1; y++) { + for (let x = x0; x < x1; x++) { + if (mask) { + const mx = Math.min(mask.width - 1, Math.floor((x / source.width) * mask.width)); + const my = Math.min(mask.height - 1, Math.floor((y / source.height) * mask.height)); + if ((mask.luma[my * mask.width + mx] ?? 0) < 128) continue; + } + const step = detailStep(source, x, y); + if (step < DETAIL_MIN_STEP) continue; + const tx = x - offset.x; + const ty = y - offset.y; + if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue; + detail += 1; + const value = source.luma[y * source.width + x] ?? 0; + // Visible means the pixel and its edge survived, not a value that happens to match. + if ( + Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= UNCHANGED_MAX_DIFF && + Math.abs(step - detailStep(treated, tx, ty)) <= 2 * UNCHANGED_MAX_DIFF + ) + unchanged += 1; + } + } + return detail < MIN_DETAIL_PIXELS ? 0 : unchanged / detail; +} + +/** + * Share of detail pixels outside every treated box that the treated frame kept + * (within codec tolerance). A low value means the two frames are not the same + * moment or framing, so a residual measured between them proves nothing. + * Returns 1 when the control area has too little detail to judge. + */ +export function controlAgreement(input: { + source: LumaFrame; + treated: LumaFrame; + boxes: NormalizedBox[]; + offset?: { x: number; y: number }; +}): number { + const { source, treated, boxes } = input; + const offset = input.offset ?? { x: 0, y: 0 }; + const ranges = boxes.map((box) => pixelRange(box, source.width, source.height)); + let detail = 0; + let kept = 0; + for (let y = 0; y < source.height; y++) { + for (let x = 0; x < source.width; x++) { + const tx = x - offset.x; + const ty = y - offset.y; + if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue; + if (ranges.some((r) => x >= r.x0 - 2 && x < r.x1 + 2 && y >= r.y0 - 2 && y < r.y1 + 2)) + continue; + if (detailStep(source, x, y) < DETAIL_MIN_STEP) continue; + detail += 1; + const value = source.luma[y * source.width + x] ?? 0; + if (Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= CONTROL_MAX_DIFF) + kept += 1; + } + } + return detail < MIN_DETAIL_PIXELS ? 1 : kept / detail; +} + +/** + * Temporal stability of a treated region over consecutive frames (0..1): + * 1 when the region changes no more between neighbours than the rest of the + * frame, falling to 0 as it flickers 24+ luma levels more on average. + */ +export function temporalStability(frames: LumaFrame[], box: NormalizedBox): number { + if (frames.length < 2) return 1; + let excess = 0; + for (let i = 1; i < frames.length; i++) { + const a = frames[i - 1]!; + const b = frames[i]!; + if (a.width !== b.width || a.height !== b.height) throw new Error("frame sizes differ"); + const r = pixelRange(box, a.width, a.height); + let inside = 0; + let insideCount = 0; + let outside = 0; + let outsideCount = 0; + for (let y = 0; y < a.height; y++) { + for (let x = 0; x < a.width; x++) { + const at = y * a.width + x; + const diff = Math.abs((a.luma[at] ?? 0) - (b.luma[at] ?? 0)); + if (x >= r.x0 && x < r.x1 && y >= r.y0 && y < r.y1) { + inside += diff; + insideCount += 1; + } else { + outside += diff; + outsideCount += 1; + } + } + } + const meanInside = insideCount ? inside / insideCount : 0; + const meanOutside = outsideCount ? outside / outsideCount : 0; + excess += Math.max(0, meanInside - meanOutside); + } + return Math.max(0, Math.min(1, 1 - excess / (frames.length - 1) / 24)); +} diff --git a/packages/adapters/src/media-privacy-plan.test.ts b/packages/adapters/src/media-privacy-plan.test.ts new file mode 100644 index 0000000..950b5c0 --- /dev/null +++ b/packages/adapters/src/media-privacy-plan.test.ts @@ -0,0 +1,210 @@ +import type { FaceRedaction } from "@rakazo/contracts"; +import { encodeVectors } from "@rakazo/media"; +import { describe, expect, it } from "vitest"; +import { mergeTracks, syntheticSeedFor } from "./media-anonymize.js"; +import { testEdl } from "./media-privacy-fixtures.js"; +import { + boxAt, + cropPixels, + effectiveModes, + ensureMinSize, + GAP_HOLD_GROWTH, + matchPerson, + outsideCrop, + padBox, + parseStoredKeyframes, + protectKeyframes, + resolveProtection, + sourceToOutput, +} from "./media-privacy-plan.js"; + +const box = { x: 0.4, y: 0.4, w: 0.1, h: 0.1 }; + +function face( + id: string, + target: FaceRedaction["target"], + mode: FaceRedaction["mode"], +): FaceRedaction { + return { id, target, mode, paddingPct: 20, featherPct: 10, minSizePct: 2 }; +} + +describe("protectKeyframes", () => { + it("interpolates short gaps and holds a grown box over long ones for review", () => { + const result = protectKeyframes( + [ + { tMs: 0, box }, + { tMs: 400, box: { ...box, x: 0.44 } }, + { tMs: 1400, box: { ...box, x: 0.5 } }, + ], + [{ startMs: 0, endMs: 2000 }], + 2, + ); + expect(result.gaps).toEqual([{ startMs: 400, endMs: 1400 }]); + expect(result.keyframes.map((k) => k.tMs)).toEqual([0, 100, 200, 300, 400, 401, 1399, 1400]); + const held = result.keyframes.find((k) => k.tMs === 401)!.box; + expect(held.w).toBeCloseTo(0.1 * (1 + 2 * GAP_HOLD_GROWTH), 3); + }); + + it("treats weak detections as lost but still covers them", () => { + const result = protectKeyframes( + [ + { tMs: 0, box, confidence: 0.9 }, + { tMs: 400, box: { ...box, x: 0.8 }, confidence: 0.2 }, + { tMs: 800, box, confidence: 0.9 }, + ], + [{ startMs: 0, endMs: 1000 }], + 2, + ); + expect(result.gaps).toEqual([{ startMs: 0, endMs: 800 }]); + const held = result.keyframes.find((k) => k.tMs === 1)!.box; + expect(held.x + held.w).toBeGreaterThanOrEqual(0.9); + }); + + it("ignores detections far outside the clip ranges and enforces a minimum size", () => { + const result = protectKeyframes( + [ + { tMs: 0, box: { x: 0.5, y: 0.5, w: 0.001, h: 0.001 } }, + { tMs: 5000, box }, + ], + [{ startMs: 0, endMs: 100 }], + 5, + ); + expect(result.keyframes).toHaveLength(1); + expect(result.keyframes[0]!.box.w).toBeGreaterThanOrEqual(0.05); + expect(ensureMinSize({ x: 0.99, y: 0.99, w: 0.01, h: 0.01 }, 10)).toEqual({ + x: 0.9, + y: 0.9, + w: 0.1, + h: 0.1, + }); + }); +}); + +describe("protection", () => { + const defaults = new Map([ + ["alice", "pixelate" as const], + ["bob", "none" as const], + ]); + const faces = [ + face("by-track", { trackId: "t1" }, "solid"), + face("by-person", { personId: "bob" }, "partial_eyes"), + face("strangers", { unassignedFaces: true }, "pixelate"), + ]; + + it("prefers track over person over default over unassigned", () => { + const resolve = (storedId: string | null, personId: string | null, protectUnassigned = false) => + resolveProtection({ storedId, personId }, { faces, defaults, protectUnassigned })?.mode; + expect(resolve("t1", "alice")).toBe("solid"); + expect(resolve("t2", "bob")).toBe("partial_eyes"); + expect(resolve("t2", "alice")).toBe("pixelate"); + expect(resolve(null, null)).toBe("pixelate"); + expect( + resolveProtection( + { storedId: null, personId: null }, + { faces: [], defaults, protectUnassigned: true }, + ), + ).toMatchObject({ mode: "blur", origin: "unassigned" }); + expect( + resolveProtection( + { storedId: null, personId: null }, + { faces: [], defaults, protectUnassigned: false }, + ), + ).toBeNull(); + expect(effectiveModes(faces, defaults).get("bob")).toBe("partial_eyes"); + }); + + it("matches faces to people only above the similarity threshold", () => { + const centroids = [ + { personId: "alice", centroid: encodeVectors([[1, 0, 0]]) }, + { personId: "bob", centroid: encodeVectors([[0, 1, 0]]) }, + ]; + expect(matchPerson([0.9, 0.1, 0], centroids)).toBe("alice"); + expect(matchPerson([0.5, 0.5, 0.7], centroids)).toBeNull(); + expect(matchPerson(null, centroids)).toBeNull(); + }); + + it("merges fresh tracks into stored ones and keeps what the fresh pass missed", () => { + const stored = [ + { id: "s1", storedId: "s1", personId: "alice", keyframes: [{ tMs: 0, box }] }, + { id: "s2", storedId: "s2", personId: null, keyframes: [{ tMs: 0, box: { ...box, x: 0 } }] }, + ]; + const merged = mergeTracks( + "a1", + stored, + [ + { + keyframes: [ + { tMs: 0, box, confidence: 0.9 }, + { tMs: 100, box, confidence: 0.9 }, + ], + embedding: null, + }, + { keyframes: [{ tMs: 0, box: { ...box, y: 0.8 }, confidence: 0.9 }], embedding: [1, 0, 0] }, + ], + [{ personId: "carol", centroid: encodeVectors([[1, 0, 0]]) }], + ); + expect(merged.map((t) => [t.id, t.personId, t.keyframes.length])).toEqual([ + ["s1", "alice", 3], + ["fresh-a1-2", "carol", 1], + ["s2", null, 1], + ]); + expect(syntheticSeedFor("alice")).toBe(syntheticSeedFor("alice")); + expect(syntheticSeedFor("alice")).not.toBe(syntheticSeedFor("bob")); + }); +}); + +describe("geometry", () => { + it("crops with the output aspect and knows what lies outside", () => { + const geometry = { sourceWidth: 1920, sourceHeight: 1080, outWidth: 1080, outHeight: 1920 }; + const crop = cropPixels(geometry, { cx: 0.8, cy: 0.5, scale: 1 }); + expect(crop).toEqual({ x: 1232, y: 0, w: 608, h: 1080 }); + expect(outsideCrop({ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }, crop, 1920, 1080)).toBe(true); + expect(outsideCrop({ x: 0.6, y: 0.1, w: 0.2, h: 0.2 }, crop, 1920, 1080)).toBe(false); + expect(padBox(box, 20)).toEqual({ + x: 0.39, + y: 0.39, + w: expect.closeTo(0.12), + h: expect.closeTo(0.12), + }); + }); + + it("maps source spans onto the output timeline", () => { + const edl = testEdl([ + { id: "a", assetId: "x", inMs: 1000, outMs: 3000 }, + { id: "b", assetId: "y", inMs: 0, outMs: 1000 }, + ]); + const [a, b] = edl.tracks.video; + expect(sourceToOutput(edl, a!, { startMs: 1500, endMs: 5000 })).toEqual({ + startMs: 500, + endMs: 2000, + }); + expect(sourceToOutput(edl, b!, { startMs: 200, endMs: 300 })).toEqual({ + startMs: 2200, + endMs: 2300, + }); + expect(sourceToOutput(edl, a!, { startMs: 0, endMs: 500 })).toBeNull(); + }); + + it("finds the box between keyframes", () => { + const keyframes = [ + { tMs: 0, box }, + { tMs: 100, box: { ...box, x: 0.5 } }, + ]; + expect(boxAt(keyframes, 50)).toEqual({ + x: 0.4, + y: 0.4, + w: expect.closeTo(0.2), + h: expect.closeTo(0.1), + }); + expect(boxAt(keyframes, 150)).toEqual({ ...box, x: 0.5 }); + expect(boxAt(keyframes, 500)).toBeNull(); + }); + + it("reads stored keyframe files in both shapes", () => { + expect(parseStoredKeyframes([{ tMs: 0, box, extra: 1 }])).toEqual([{ tMs: 0, box }]); + expect(parseStoredKeyframes({ keyframes: [{ tMs: 5, box, confidence: 0.4 }] })).toEqual([ + { tMs: 5, box, confidence: 0.4 }, + ]); + expect(() => parseStoredKeyframes({ frames: [] })).toThrow(); + }); +}); diff --git a/packages/adapters/src/media-privacy-plan.ts b/packages/adapters/src/media-privacy-plan.ts new file mode 100644 index 0000000..7112466 --- /dev/null +++ b/packages/adapters/src/media-privacy-plan.ts @@ -0,0 +1,499 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { + AnonymizeMode, + BoxKeyframe, + Edl, + EdlVideoClip, + FaceRedaction, + NormalizedBox, +} from "@rakazo/contracts"; +import { AiAlterationSchema, AnonymizeModeSchema, BoxKeyframeSchema } from "@rakazo/contracts"; +import { cosineSimilarity, decodeVectors, introMs, MAX_TRACKING_GAP_MS } from "@rakazo/media"; +import { z } from "zod"; + +/* + * Pure planning helpers for face, mark, and region protection, plus the + * privacy record that prepareSources leaves next to the render output for the + * gate. Times are source times of the asset unless a name says otherwise; + * boxes are normalized to the displayed (rotation-applied) frame. + */ + +/** Tracking rate of the fresh detection pass on clip sources. */ +export const FACE_TRACK_FPS = 10; +/** + * Minimum cosine similarity between a face embedding and a person centroid to + * count as that person. Below it the face is unassigned, which fails closed: + * unassigned faces are redacted and block the gate while a protected person + * appears in the project. + */ +export const FACE_MATCH_MIN_COSINE = 0.6; +/** Detections below this confidence count as lost (spec 7: hold and grow the last mask). */ +export const FACE_MIN_CONFIDENCE = 0.5; +/** A fresh track inherits a stored track's person when their boxes overlap this much on average. */ +export const TRACK_MATCH_MIN_IOU = 0.3; +/** Step for linear interpolation across short detection gaps (<= MAX_TRACKING_GAP_MS). */ +export const INTERPOLATE_STEP_MS = 100; +/** Lost detections hold the last box, enlarged by this share on every side (spec 7). */ +export const GAP_HOLD_GROWTH = 0.25; +/** Padding around marks and regions, in percent of the box. */ +export const MARK_PADDING_PCT = 10; +/** Person-bound marks re-found below this segmentation score go to review and block the gate. */ +export const MARK_REVIEW_MIN_SCORE = 0.6; +/** Treatment quality below this suggests (and, while rendering, applies) a fallback. */ +export const MARK_QUALITY_MIN = 0.7; + +/** Name of the privacy record prepareSources writes into the render work directory. */ +export const PRIVACY_RECORD_FILE = "privacy.json"; + +export type ProtectMode = Exclude; +export type PartialMode = "partial_eyes" | "partial_upper" | "partial_lower"; + +export interface Protection { + mode: ProtectMode; + paddingPct: number; + featherPct: number; + minSizePct: number; + /** Redaction id from the EDL, "default" (person setting), or "unassigned". */ + origin: string; +} + +export type TrackKeyframe = BoxKeyframe & { confidence?: number }; + +export interface CandidateTrack { + /** Stable id: the stored track id, or `fresh-` for faces only the fresh pass found. */ + id: string; + /** Stored track id when the face matches an analysed track, for EDL track targets. */ + storedId: string | null; + personId: string | null; + keyframes: TrackKeyframe[]; +} + +export interface TimeWindow { + startMs: number; + endMs: number; +} + +const DEFAULT_PROTECTION = { paddingPct: 20, featherPct: 10, minSizePct: 2 }; + +export function isPartial(mode: AnonymizeMode): mode is PartialMode { + return mode === "partial_eyes" || mode === "partial_upper" || mode === "partial_lower"; +} + +function fromRedaction(redaction: FaceRedaction): Protection { + return { + mode: redaction.mode, + paddingPct: redaction.paddingPct, + featherPct: redaction.featherPct, + minSizePct: redaction.minSizePct, + origin: redaction.id, + }; +} + +/** + * Protection for one face track. Most specific wins: EDL track target, EDL + * person target, the person's default, then the EDL unassigned target, and + * finally a blur for unassigned faces while any protected person appears in + * the project. + */ +export function resolveProtection( + track: Pick, + context: { + faces: FaceRedaction[]; + defaults: Map; + protectUnassigned: boolean; + }, +): Protection | null { + if (track.storedId) { + const byTrack = context.faces.find( + (face) => "trackId" in face.target && face.target.trackId === track.storedId, + ); + if (byTrack) return fromRedaction(byTrack); + } + if (track.personId) { + const byPerson = context.faces.find( + (face) => "personId" in face.target && face.target.personId === track.personId, + ); + if (byPerson) return fromRedaction(byPerson); + const mode = context.defaults.get(track.personId); + if (mode && mode !== "none") return { mode, ...DEFAULT_PROTECTION, origin: "default" }; + return null; + } + const unassigned = context.faces.find((face) => "unassignedFaces" in face.target); + if (unassigned) return fromRedaction(unassigned); + if (context.protectUnassigned) + return { mode: "blur", ...DEFAULT_PROTECTION, origin: "unassigned" }; + return null; +} + +/** Effective mode per person: an EDL person target wins over the person's default. */ +export function effectiveModes( + faces: FaceRedaction[], + defaults: Map, +): Map { + const modes = new Map(defaults); + for (const face of faces) { + if ("personId" in face.target) modes.set(face.target.personId, face.mode); + } + return modes; +} + +/** Best person for an embedding by cosine to the centroids, or null below the threshold. */ +export function matchPerson( + embedding: ArrayLike | null, + centroids: Array<{ personId: string; centroid: Uint8Array }>, +): string | null { + if (!embedding || embedding.length === 0) return null; + let best: { personId: string; score: number } | null = null; + for (const { personId, centroid } of centroids) { + if (centroid.byteLength !== embedding.length * 4) continue; + const vector = decodeVectors(centroid, embedding.length)[0]; + if (!vector) continue; + const score = cosineSimilarity(embedding, vector); + if (score >= FACE_MATCH_MIN_COSINE && (!best || score > best.score)) best = { personId, score }; + } + return best?.personId ?? null; +} + +const clampUnit = (value: number) => Math.min(1, Math.max(0, value)); + +/** Grow a box by `share` of its size on every side, clamped to the frame. */ +export function growBox(box: NormalizedBox, share: number): NormalizedBox { + const x = clampUnit(box.x - box.w * share); + const y = clampUnit(box.y - box.h * share); + return { + x, + y, + w: clampUnit(box.x + box.w + box.w * share) - x, + h: clampUnit(box.y + box.h + box.h * share) - y, + }; +} + +/** Padding as the redaction providers apply it: paddingPct / 2 of the size on each side. */ +export function padBox(box: NormalizedBox, paddingPct: number): NormalizedBox { + return growBox(box, Math.max(0, paddingPct) / 200); +} + +/** Enlarge a box around its centre until both sides reach `minSizePct` of the frame. */ +export function ensureMinSize(box: NormalizedBox, minSizePct: number): NormalizedBox { + const min = minSizePct / 100; + if (box.w >= min && box.h >= min) return box; + const w = Math.min(1, Math.max(box.w, min)); + const h = Math.min(1, Math.max(box.h, min)); + const x = clampUnit(Math.min(box.x + box.w / 2 - w / 2, 1 - w)); + const y = clampUnit(Math.min(box.y + box.h / 2 - h / 2, 1 - h)); + return { x, y, w, h }; +} + +export function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox { + const x = Math.min(a.x, b.x); + const y = Math.min(a.y, b.y); + return { + x, + y, + w: Math.max(a.x + a.w, b.x + b.w) - x, + h: Math.max(a.y + a.h, b.y + b.h) - y, + }; +} + +export function iou(a: NormalizedBox, b: NormalizedBox): number { + const w = Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x); + const h = Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y); + if (w <= 0 || h <= 0) return 0; + const inter = w * h; + return inter / (a.w * a.h + b.w * b.h - inter); +} + +function lerpBox(a: NormalizedBox, b: NormalizedBox, f: number): NormalizedBox { + const mix = (p: number, q: number) => p + (q - p) * f; + return { x: mix(a.x, b.x), y: mix(a.y, b.y), w: mix(a.w, b.w), h: mix(a.h, b.h) }; +} + +/** Round outward so a rounded box never covers less than the exact one. */ +export function roundBoxOut(box: NormalizedBox): NormalizedBox { + const x = Math.floor(box.x * 10_000) / 10_000; + const y = Math.floor(box.y * 10_000) / 10_000; + return { + x, + y, + w: Math.min(1 - x, Math.ceil((box.x + box.w - x) * 10_000) / 10_000), + h: Math.min(1 - y, Math.ceil((box.y + box.h - y) * 10_000) / 10_000), + }; +} + +function nearWindows(tMs: number, windows: TimeWindow[]): boolean { + return windows.some( + (w) => tMs >= w.startMs - MAX_TRACKING_GAP_MS && tMs <= w.endMs + MAX_TRACKING_GAP_MS, + ); +} + +/** + * Keyframes for redaction near the windows (clip ranges plus the bridging + * limit on both sides). Detections below FACE_MIN_CONFIDENCE count as lost. + * Gaps up to MAX_TRACKING_GAP_MS between confident detections are + * interpolated; longer ones hold the last confident box grown by + * GAP_HOLD_GROWTH (unioned with any weak detections) until the face is found + * again, and are returned so the gate blocks them as needing review. + */ +export function protectKeyframes( + keyframes: TrackKeyframe[], + windows: TimeWindow[], + minSizePct: number, +): { keyframes: BoxKeyframe[]; gaps: TimeWindow[] } { + const inside = [...keyframes] + .filter((k) => nearWindows(k.tMs, windows)) + .sort((a, b) => a.tMs - b.tMs) + .map((k) => ({ + tMs: k.tMs, + box: ensureMinSize(k.box, minSizePct), + confident: (k.confidence ?? 1) >= FACE_MIN_CONFIDENCE, + })); + const out: BoxKeyframe[] = []; + const gaps: TimeWindow[] = []; + const confident = inside.filter((k) => k.confident); + const weak = inside.filter((k) => !k.confident); + if (confident.length === 0) { + // Only weak detections: cover them grown and send the whole span to review. + for (const k of weak) + out.push({ tMs: k.tMs, box: roundBoxOut(growBox(k.box, GAP_HOLD_GROWTH)) }); + const first = weak[0]; + const last = weak.at(-1); + if (first && last) gaps.push({ startMs: first.tMs, endMs: Math.max(last.tMs, first.tMs + 1) }); + return { keyframes: out, gaps }; + } + confident.forEach((current, index) => { + out.push({ tMs: current.tMs, box: roundBoxOut(current.box) }); + const next = confident[index + 1]; + if (!next) return; + const span = next.tMs - current.tMs; + const between = weak.filter((k) => k.tMs > current.tMs && k.tMs < next.tMs); + if (span <= MAX_TRACKING_GAP_MS) { + for (let t = current.tMs + INTERPOLATE_STEP_MS; t < next.tMs; t += INTERPOLATE_STEP_MS) { + let box = lerpBox(current.box, next.box, (t - current.tMs) / span); + for (const w of between) + if (Math.abs(w.tMs - t) < INTERPOLATE_STEP_MS) box = unionBox(box, w.box); + out.push({ tMs: t, box: roundBoxOut(box) }); + } + return; + } + let held = growBox(current.box, GAP_HOLD_GROWTH); + for (const w of between) held = unionBox(held, w.box); + const box = roundBoxOut(held); + out.push({ tMs: current.tMs + 1, box }, { tMs: next.tMs - 1, box }); + gaps.push({ startMs: current.tMs, endMs: next.tMs }); + }); + return { keyframes: out, gaps }; +} + +/** Box a keyframe list covers at `tMs`: the union of the bracketing keyframes, or null outside. */ +export function boxAt(keyframes: BoxKeyframe[], tMs: number, holdMs = 100): NormalizedBox | null { + let before: BoxKeyframe | null = null; + let after: BoxKeyframe | null = null; + for (const k of keyframes) { + if (k.tMs <= tMs && (!before || k.tMs > before.tMs)) before = k; + if (k.tMs >= tMs && (!after || k.tMs < after.tMs)) after = k; + } + if (before && after) return unionBox(before.box, after.box); + if (before && tMs - before.tMs <= holdMs) return before.box; + return null; +} + +/** Mean IoU of two tracks over the times both observe (nearest keyframe within 60 ms). */ +export function trackOverlap(a: BoxKeyframe[], b: BoxKeyframe[]): number { + const scores: number[] = []; + for (const k of a) { + let nearest: BoxKeyframe | null = null; + for (const other of b) { + if (Math.abs(other.tMs - k.tMs) > 60) continue; + if (!nearest || Math.abs(other.tMs - k.tMs) < Math.abs(nearest.tMs - k.tMs)) nearest = other; + } + if (nearest) scores.push(iou(k.box, nearest.box)); + } + return scores.length === 0 ? 0 : scores.reduce((sum, s) => sum + s, 0) / scores.length; +} + +/** Source windows each clip of an asset shows. */ +export function clipWindows(clips: EdlVideoClip[]): TimeWindow[] { + return clips.map((clip) => ({ startMs: clip.inMs, endMs: clip.outMs })); +} + +export function overlapsWindows(span: TimeWindow, windows: TimeWindow[]): boolean { + return windows.some((w) => span.startMs <= w.endMs && w.startMs <= span.endMs); +} + +/** + * Inner part of a face box that a partial mode must alter. It sits inside the + * band every provider covers (eyes 22-55 %, upper 0-55 %, lower 50-100 % of + * the padded box), so the gate checks only pixels the mode promised to change. + */ +export function partialCheckBand(box: NormalizedBox, mode: PartialMode): NormalizedBox { + const [top, bottom] = + mode === "partial_eyes" ? [0.3, 0.48] : mode === "partial_upper" ? [0.05, 0.5] : [0.55, 0.95]; + return { + x: box.x + box.w * 0.1, + y: box.y + box.h * top, + w: box.w * 0.8, + h: box.h * (bottom - top), + }; +} + +export interface PixelBox { + x: number; + y: number; + w: number; + h: number; +} + +/** + * Static crop window in display pixels with the output aspect, as the + * compiler sizes it for a reframe keyframe. Even sizes and offsets for yuv420p. + */ +export function cropPixels( + geometry: { sourceWidth: number; sourceHeight: number; outWidth: number; outHeight: number }, + keyframe: { cx: number; cy: number; scale: number }, +): PixelBox { + const outAspect = geometry.outWidth / geometry.outHeight; + let w: number; + let h: number; + if (geometry.sourceWidth / geometry.sourceHeight > outAspect) { + h = geometry.sourceHeight; + w = h * outAspect; + } else { + w = geometry.sourceWidth; + h = w / outAspect; + } + const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2); + const even = (v: number) => Math.max(0, Math.floor(v / 2) * 2); + const maxW = geometry.sourceWidth - (geometry.sourceWidth % 2); + const maxH = geometry.sourceHeight - (geometry.sourceHeight % 2); + const cw = Math.min(maxW, roundEven(w / keyframe.scale)); + const ch = Math.min(maxH, roundEven(h / keyframe.scale)); + const x = even( + Math.min(Math.max(0, keyframe.cx * geometry.sourceWidth - cw / 2), geometry.sourceWidth - cw), + ); + const y = even( + Math.min(Math.max(0, keyframe.cy * geometry.sourceHeight - ch / 2), geometry.sourceHeight - ch), + ); + return { x, y, w: cw, h: ch }; +} + +/** Whether a normalized box lies completely outside a pixel crop. */ +export function outsideCrop(box: NormalizedBox, crop: PixelBox, width: number, height: number) { + const x0 = box.x * width; + const y0 = box.y * height; + const x1 = (box.x + box.w) * width; + const y1 = (box.y + box.h) * height; + return x1 <= crop.x || x0 >= crop.x + crop.w || y1 <= crop.y || y0 >= crop.y + crop.h; +} + +/** Map a source window of one clip onto the output timeline (intro included), or null. */ +export function sourceToOutput(edl: Edl, clip: EdlVideoClip, span: TimeWindow): TimeWindow | null { + const start = Math.max(span.startMs, clip.inMs); + const end = Math.min(span.endMs, clip.outMs); + if (end < start) return null; + const toOutput = (t: number) => + Math.round(introMs(edl) + clip.atMs + (t - clip.inMs) / clip.speed); + return { startMs: toOutput(start), endMs: toOutput(end) }; +} + +// --------------------------------------------------------------------------- +// Privacy record (prepareSources → gate) +// --------------------------------------------------------------------------- + +const PixelBoxSchema = z.object({ + x: z.number().int().min(0), + y: z.number().int().min(0), + w: z.number().int().min(1), + h: z.number().int().min(1), +}); + +const SpanSchema = { startMs: z.number(), endMs: z.number() }; + +/** One protected region the gate verifies pixel by pixel. */ +export const PrivacyCheckSchema = z.object({ + kind: z.enum(["face", "mark", "region"]), + /** Track, mark, or region id. */ + id: z.string(), + personId: z.string().nullable(), + markId: z.string().nullable(), + /** Boxes the treatment promised to alter, normalized to the base frame. */ + keyframes: z.array(BoxKeyframeSchema), + /** Absolute mask directory (`.png` per frame) or null for box-only regions. */ + maskDir: z.string().nullable(), +}); +export type PrivacyCheck = z.infer; + +export const PrivacyRecordSchema = z.object({ + version: z.literal(1), + assets: z.record( + z.string(), + z.object({ + /** Unredacted file the prepared source derives from (pixel reference). */ + basePath: z.string(), + preparedPath: z.string(), + width: z.number().int().positive(), + height: z.number().int().positive(), + still: z.boolean(), + /** Static crop applied for reframe_out, in base display pixels. */ + crop: PixelBoxSchema.nullable(), + checks: z.array(PrivacyCheckSchema), + }), + ), + /** Detection gaps over MAX_TRACKING_GAP_MS per asset (source time), for review. */ + trackingGaps: z.array( + z.object({ assetId: z.string(), personId: z.string().nullable(), ...SpanSchema }), + ), + /** Person-bound marks re-found with low confidence, for review. */ + reviews: z.array(z.object({ markId: z.string(), assetId: z.string(), score: z.number() })), + /** Faces that stay detectable by design (synthetic identity, partial modes), source time. */ + expectedFaces: z.array( + z.object({ + assetId: z.string(), + personId: z.string().nullable(), + mode: AnonymizeModeSchema, + ...SpanSchema, + }), + ), + /** Faces in the clip ranges that belong to no person (consent cannot be shown). */ + unassignedFaces: z.array(z.object({ assetId: z.string(), ...SpanSchema })), + aiAlterations: z.array(AiAlterationSchema), + /** Effective protection per person (EDL target over person default). */ + protectedPeople: z.record(z.string(), AnonymizeModeSchema), + /** People whose faces appear in the clip ranges. */ + visiblePeople: z.array(z.string()), + protectUnassigned: z.boolean(), +}); +export type PrivacyRecord = z.infer; + +export async function writePrivacyRecord(workDir: string, record: PrivacyRecord): Promise { + await writeFile(path.join(workDir, PRIVACY_RECORD_FILE), JSON.stringify(record)); +} + +/** The record prepareSources wrote, or null when it is missing. Malformed records throw. */ +export async function readPrivacyRecord(workDir: string): Promise { + let text: string; + try { + text = await readFile(path.join(workDir, PRIVACY_RECORD_FILE), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + return PrivacyRecordSchema.parse(JSON.parse(text)); +} + +/** Stored face keyframe files: an array of keyframes or a tracking result with `keyframes`. */ +const StoredKeyframeSchema = BoxKeyframeSchema.extend({ confidence: z.number().optional() }); +const StoredKeyframesSchema = z.union([ + z.array(StoredKeyframeSchema.passthrough()), + z.object({ keyframes: z.array(StoredKeyframeSchema.passthrough()) }).passthrough(), +]); + +export function parseStoredKeyframes(json: unknown): TrackKeyframe[] { + const parsed = StoredKeyframesSchema.parse(json); + const list = Array.isArray(parsed) ? parsed : parsed.keyframes; + return list.map((k) => ({ + tMs: k.tMs, + box: k.box, + ...(k.confidence !== undefined ? { confidence: k.confidence } : {}), + })); +} diff --git a/packages/adapters/src/media-privacy.ts b/packages/adapters/src/media-privacy.ts new file mode 100644 index 0000000..40abe93 --- /dev/null +++ b/packages/adapters/src/media-privacy.ts @@ -0,0 +1,39 @@ +import { rm } from "node:fs/promises"; +import path from "node:path"; +import { prepareSources } from "./media-anonymize.js"; +import type { MediaPrivacyService, MediaServiceDeps } from "./media-context.js"; +import { runGate } from "./media-gate.js"; +import { createMark, updateMark } from "./media-marks.js"; +import type { PrivacyToolOptions } from "./media-privacy-io.js"; +import { resolveStored } from "./media-privacy-io.js"; + +/* + * MediaPrivacyService: anonymization before rendering, the gate after it, + * identifying marks, and deleting a person with their biometric data. Face + * data never leaves this host: matching, tracking, and clustering run against + * the local inference provider and database only. + */ + +export function createMediaPrivacyService( + deps: MediaServiceDeps, + tools: PrivacyToolOptions = {}, +): MediaPrivacyService { + const withTools = { ...deps, ...tools }; + return { + prepareSources: (input) => prepareSources(withTools, input), + gate: (input) => runGate(withTools, input), + createMark: (actor, input) => createMark(withTools, actor, input), + updateMark: (actor, input) => updateMark(withTools, actor, input), + async removePerson(actor, personId) { + const removed = await deps.repos.removePersonCompletely(actor.spaceId, personId); + // Rows are gone; now the files: track keyframes and each mark's directory + // (masks plus previews). A path outside the media root throws untouched. + const markDirs = removed.maskDirs.map((dir) => + path.basename(dir) === "masks" ? path.dirname(dir) : dir, + ); + for (const stored of [...removed.keyframesPaths, ...markDirs]) { + await rm(await resolveStored(deps.mediaRoot, stored), { recursive: true, force: true }); + } + }, + }; +}