diff --git a/packages/adapters/src/media-privacy-pixels.ts b/packages/adapters/src/media-privacy-pixels.ts new file mode 100644 index 0000000..2e9716a --- /dev/null +++ b/packages/adapters/src/media-privacy-pixels.ts @@ -0,0 +1,179 @@ +import { readFile } from "node:fs/promises"; +import { inflateSync } from "node:zlib"; +import type { NormalizedBox } from "@rakazo/contracts"; + +/* + * Pixel checks for the privacy gate. Frames come from MediaRenderer.extractFrame + * as PNG; the decoder below covers the PNGs ffmpeg and the mask writers produce + * (8-bit gray, gray+alpha, RGB, RGBA; not interlaced, not palette). + */ + +export interface LumaFrame { + width: number; + height: number; + /** One luma byte per pixel, row-major. */ + luma: Uint8Array; +} + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const CHANNELS: Record = { 0: 1, 2: 3, 4: 2, 6: 4 }; + +function paeth(a: number, b: number, c: number): number { + const p = a + b - c; + const pa = Math.abs(p - a); + const pb = Math.abs(p - b); + const pc = Math.abs(p - c); + if (pa <= pb && pa <= pc) return a; + return pb <= pc ? b : c; +} + +/** Decode a PNG to luma (Rec. 601 weights). Throws on anything it cannot read exactly. */ +export function decodePngLuma(png: Buffer): LumaFrame { + if (png.length < 8 || !png.subarray(0, 8).equals(PNG_SIGNATURE)) throw new Error("not a PNG"); + let offset = 8; + let width = 0; + let height = 0; + let channels = 0; + const data: Buffer[] = []; + while (offset + 8 <= png.length) { + const length = png.readUInt32BE(offset); + const type = png.toString("ascii", offset + 4, offset + 8); + const body = png.subarray(offset + 8, offset + 8 + length); + offset += 12 + length; + if (type === "IHDR") { + width = body.readUInt32BE(0); + height = body.readUInt32BE(4); + const depth = body[8]; + const colorType = body[9] ?? -1; + const interlace = body[12]; + channels = CHANNELS[colorType] ?? 0; + if (depth !== 8 || channels === 0 || interlace !== 0) + throw new Error("unsupported PNG layout"); + } else if (type === "IDAT") { + data.push(body); + } else if (type === "IEND") { + break; + } + } + if (width === 0 || height === 0 || channels === 0) throw new Error("PNG has no image header"); + const raw = inflateSync(Buffer.concat(data)); + const stride = width * channels; + if (raw.length < (stride + 1) * height) throw new Error("PNG data is truncated"); + const pixels = Buffer.alloc(stride * height); + for (let y = 0; y < height; y++) { + const filter = raw[y * (stride + 1)]; + const line = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1)); + const row = y * stride; + for (let x = 0; x < stride; x++) { + const left = x >= channels ? (pixels[row + x - channels] ?? 0) : 0; + const up = y > 0 ? (pixels[row - stride + x] ?? 0) : 0; + const upLeft = y > 0 && x >= channels ? (pixels[row - stride + x - channels] ?? 0) : 0; + const value = line[x] ?? 0; + let out: number; + if (filter === 0) out = value; + else if (filter === 1) out = value + left; + else if (filter === 2) out = value + up; + else if (filter === 3) out = value + ((left + up) >> 1); + else if (filter === 4) out = value + paeth(left, up, upLeft); + else throw new Error("unknown PNG filter"); + pixels[row + x] = out & 0xff; + } + } + const luma = new Uint8Array(width * height); + for (let i = 0; i < width * height; i++) { + const base = i * channels; + if (channels <= 2) { + luma[i] = pixels[base] ?? 0; + } else { + const r = pixels[base] ?? 0; + const g = pixels[base + 1] ?? 0; + const b = pixels[base + 2] ?? 0; + luma[i] = Math.round(0.299 * r + 0.587 * g + 0.114 * b); + } + } + return { width, height, luma }; +} + +export async function readPngLuma(file: string): Promise { + return decodePngLuma(await readFile(file)); +} + +/** Bounding box of the non-black pixels of a mask, or null when the mask is empty. */ +export function maskBounds(mask: LumaFrame): NormalizedBox | null { + let minX = mask.width; + let minY = mask.height; + let maxX = -1; + let maxY = -1; + for (let y = 0; y < mask.height; y++) { + for (let x = 0; x < mask.width; x++) { + if ((mask.luma[y * mask.width + x] ?? 0) < 128) continue; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + if (maxX < 0) return null; + return { + x: minX / mask.width, + y: minY / mask.height, + w: (maxX + 1 - minX) / mask.width, + h: (maxY + 1 - minY) / mask.height, + }; +} + +/** A luma step at least this large marks a detail pixel (an edge someone could recognise). */ +export const DETAIL_MIN_STEP = 12; +/** A detail pixel within this many luma levels of the source counts as unchanged. */ +export const UNCHANGED_MAX_DIFF = 2; +/** Regions with fewer detail pixels show nothing identifying. */ +const MIN_DETAIL_PIXELS = 16; + +/** + * Residual visibility of a region (0..1): the share of the source's detail + * pixels that the treated frame left unchanged. Blur, pixelation, fills, and + * inpainting all rewrite edges, so a treated region scores near 0, an untouched + * one 1. A flat region without detail scores 0: there is nothing to recognise. + * + * `source` and `treated` are pixel-aligned at full resolution; when `treated` + * is a crop of the source, `offset` is the crop origin in source pixels. + * Optional `mask` (normalized over the source frame) limits the region to mask pixels. + */ +export function residualVisibility(input: { + source: LumaFrame; + treated: LumaFrame; + box: NormalizedBox; + offset?: { x: number; y: number }; + mask?: LumaFrame; +}): number { + const { source, treated, box, mask } = input; + const offset = input.offset ?? { x: 0, y: 0 }; + const x0 = Math.max(0, Math.floor(box.x * source.width)); + const y0 = Math.max(0, Math.floor(box.y * source.height)); + const x1 = Math.min(source.width - 1, Math.ceil((box.x + box.w) * source.width) - 1); + const y1 = Math.min(source.height - 1, Math.ceil((box.y + box.h) * source.height) - 1); + let detail = 0; + let unchanged = 0; + for (let y = y0; y < y1; y++) { + for (let x = x0; x < x1; x++) { + if (mask) { + const mx = Math.min(mask.width - 1, Math.floor((x / source.width) * mask.width)); + const my = Math.min(mask.height - 1, Math.floor((y / source.height) * mask.height)); + if ((mask.luma[my * mask.width + mx] ?? 0) < 128) continue; + } + const at = y * source.width + x; + const value = source.luma[at] ?? 0; + const step = + Math.abs(value - (source.luma[at + 1] ?? value)) + + Math.abs(value - (source.luma[at + source.width] ?? value)); + if (step < DETAIL_MIN_STEP) continue; + const tx = x - offset.x; + const ty = y - offset.y; + if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue; + detail += 1; + if (Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= UNCHANGED_MAX_DIFF) + unchanged += 1; + } + } + return detail < MIN_DETAIL_PIXELS ? 0 : unchanged / detail; +} diff --git a/packages/adapters/src/media-privacy-plan.ts b/packages/adapters/src/media-privacy-plan.ts new file mode 100644 index 0000000..cdd938a --- /dev/null +++ b/packages/adapters/src/media-privacy-plan.ts @@ -0,0 +1,391 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { + AnonymizeMode, + BoxKeyframe, + Edl, + EdlVideoClip, + FaceRedaction, + NormalizedBox, +} from "@rakazo/contracts"; +import { + AiAlterationSchema, + AnonymizeModeSchema, + BoxKeyframeSchema, + MarkTreatmentSchema, +} from "@rakazo/contracts"; +import { cosineSimilarity, decodeVectors, MAX_TRACKING_GAP_MS } from "@rakazo/media"; +import { z } from "zod"; + +/* + * Pure planning helpers for face, mark, and region protection, plus the + * privacy record that prepareSources leaves next to the render output for the + * gate. Times are source times of the asset unless a name says otherwise. + */ + +/** Tracking rate for the re-detection pass on clip sources (faces the stored analysis missed). */ +export const FACE_TRACK_FPS = 10; +/** + * Minimum cosine similarity between a face embedding and a person centroid to + * count as that person. Below it the face is unassigned, which fails closed: + * unassigned faces are redacted and block the gate while a protected person + * appears in the project. + */ +export const FACE_MATCH_MIN_COSINE = 0.6; +/** Step for linear interpolation across short detection gaps (<= MAX_TRACKING_GAP_MS). */ +export const INTERPOLATE_STEP_MS = 100; +/** Longer gaps hold the last box, enlarged by this share on every side (spec 7). */ +export const GAP_HOLD_GROWTH = 0.25; +/** Padding around marks and regions, in percent of the box. */ +export const MARK_PADDING_PCT = 10; + +/** Name of the privacy record prepareSources writes into the render work directory. */ +export const PRIVACY_RECORD_FILE = "privacy.json"; + +export type ProtectMode = Exclude; +export type PartialMode = "partial_eyes" | "partial_upper" | "partial_lower"; + +export interface Protection { + mode: ProtectMode; + paddingPct: number; + featherPct: number; + minSizePct: number; + /** Redaction id from the EDL, "default" (person setting), or "unassigned". */ + origin: string; +} + +export interface CandidateTrack { + /** Stored track id, or null for faces found by the re-detection pass. */ + storedId: string | null; + personId: string | null; + keyframes: BoxKeyframe[]; +} + +export interface TimeWindow { + startMs: number; + endMs: number; +} + +const DEFAULT_PROTECTION = { paddingPct: 20, featherPct: 10, minSizePct: 2 }; + +export function isPartial(mode: AnonymizeMode): mode is PartialMode { + return mode === "partial_eyes" || mode === "partial_upper" || mode === "partial_lower"; +} + +function fromRedaction(redaction: FaceRedaction): Protection { + return { + mode: redaction.mode, + paddingPct: redaction.paddingPct, + featherPct: redaction.featherPct, + minSizePct: redaction.minSizePct, + origin: redaction.id, + }; +} + +/** + * Protection for one face track. Most specific wins: EDL track target, EDL + * person target, EDL unassigned target, the person's default, and finally a + * blur for unassigned faces while any protected person appears in the project. + */ +export function resolveProtection( + track: Pick, + context: { + faces: FaceRedaction[]; + defaults: Map; + protectUnassigned: boolean; + }, +): Protection | null { + const byTrack = context.faces.find( + (face) => "trackId" in face.target && face.target.trackId === track.storedId, + ); + if (byTrack && track.storedId) return fromRedaction(byTrack); + if (track.personId) { + const byPerson = context.faces.find( + (face) => "personId" in face.target && face.target.personId === track.personId, + ); + if (byPerson) return fromRedaction(byPerson); + const mode = context.defaults.get(track.personId); + if (mode && mode !== "none") return { mode, ...DEFAULT_PROTECTION, origin: "default" }; + return null; + } + const unassigned = context.faces.find((face) => "unassignedFaces" in face.target); + if (unassigned) return fromRedaction(unassigned); + if (context.protectUnassigned) return { mode: "blur", ...DEFAULT_PROTECTION, origin: "unassigned" }; + return null; +} + +/** Best person for an embedding by cosine to the centroids, or null below the threshold. */ +export function matchPerson( + embedding: ArrayLike | null, + centroids: Array<{ personId: string; centroid: Uint8Array }>, +): string | null { + if (!embedding || embedding.length === 0) return null; + let best: { personId: string; score: number } | null = null; + for (const { personId, centroid } of centroids) { + if (centroid.byteLength !== embedding.length * 4) continue; + const vector = decodeVectors(centroid, embedding.length)[0]; + if (!vector) continue; + const score = cosineSimilarity(embedding, vector); + if (score >= FACE_MATCH_MIN_COSINE && (!best || score > best.score)) best = { personId, score }; + } + return best?.personId ?? null; +} + +const clampUnit = (value: number) => Math.min(1, Math.max(0, value)); + +/** Grow a box by `share` of its size on every side, clamped to the frame. */ +export function growBox(box: NormalizedBox, share: number): NormalizedBox { + const x = clampUnit(box.x - box.w * share); + const y = clampUnit(box.y - box.h * share); + return { + x, + y, + w: clampUnit(box.x + box.w + box.w * share) - x, + h: clampUnit(box.y + box.h + box.h * share) - y, + }; +} + +/** Enlarge a box around its centre until both sides reach `minSizePct` of the frame. */ +export function ensureMinSize(box: NormalizedBox, minSizePct: number): NormalizedBox { + const min = minSizePct / 100; + if (box.w >= min && box.h >= min) return box; + const w = Math.max(box.w, min); + const h = Math.max(box.h, min); + const x = clampUnit(box.x + box.w / 2 - w / 2); + const y = clampUnit(box.y + box.h / 2 - h / 2); + return { x, y, w: Math.min(w, 1 - x), h: Math.min(h, 1 - y) }; +} + +export function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox { + const x = Math.min(a.x, b.x); + const y = Math.min(a.y, b.y); + return { + x, + y, + w: Math.max(a.x + a.w, b.x + b.w) - x, + h: Math.max(a.y + a.h, b.y + b.h) - y, + }; +} + +export function padBox(box: NormalizedBox, paddingPct: number): NormalizedBox { + return growBox(box, Math.max(0, paddingPct) / 200); +} + +function lerpBox(a: NormalizedBox, b: NormalizedBox, f: number): NormalizedBox { + const mix = (p: number, q: number) => p + (q - p) * f; + return { x: mix(a.x, b.x), y: mix(a.y, b.y), w: mix(a.w, b.w), h: mix(a.h, b.h) }; +} + +const round4 = (value: number) => Math.round(value * 10_000) / 10_000; +const roundBox = (box: NormalizedBox): NormalizedBox => ({ + x: round4(box.x), + y: round4(box.y), + w: round4(box.w), + h: round4(box.h), +}); + +/** + * Keyframes for redaction inside the windows (clip ranges plus the bridging + * limit on both sides). Gaps up to MAX_TRACKING_GAP_MS are interpolated; + * longer ones hold the last box grown by GAP_HOLD_GROWTH until the face is + * found again and are returned so the gate blocks them (needs review). + */ +export function protectKeyframes( + keyframes: BoxKeyframe[], + windows: TimeWindow[], + minSizePct: number, +): { keyframes: BoxKeyframe[]; gaps: TimeWindow[] } { + const inside = [...keyframes] + .filter((k) => + windows.some( + (w) => k.tMs >= w.startMs - MAX_TRACKING_GAP_MS && k.tMs <= w.endMs + MAX_TRACKING_GAP_MS, + ), + ) + .sort((a, b) => a.tMs - b.tMs) + .map((k) => ({ tMs: k.tMs, box: ensureMinSize(k.box, minSizePct) })); + const out: BoxKeyframe[] = []; + const gaps: TimeWindow[] = []; + inside.forEach((current, index) => { + out.push({ tMs: current.tMs, box: roundBox(current.box) }); + const next = inside[index + 1]; + if (!next) return; + const span = next.tMs - current.tMs; + if (span <= MAX_TRACKING_GAP_MS) { + for (let t = current.tMs + INTERPOLATE_STEP_MS; t < next.tMs; t += INTERPOLATE_STEP_MS) { + out.push({ + tMs: t, + box: roundBox(lerpBox(current.box, next.box, (t - current.tMs) / span)), + }); + } + return; + } + const held = roundBox(growBox(current.box, GAP_HOLD_GROWTH)); + out.push({ tMs: current.tMs + 1, box: held }, { tMs: next.tMs - 1, box: held }); + gaps.push({ startMs: current.tMs, endMs: next.tMs }); + }); + return { keyframes: out, gaps }; +} + +/** Source windows each clip of an asset shows. */ +export function clipWindows(clips: EdlVideoClip[]): TimeWindow[] { + return clips.map((clip) => ({ startMs: clip.inMs, endMs: clip.outMs })); +} + +export function overlapsWindows(keyframes: BoxKeyframe[], windows: TimeWindow[]): boolean { + return keyframes.some((k) => + windows.some( + (w) => k.tMs >= w.startMs - MAX_TRACKING_GAP_MS && k.tMs <= w.endMs + MAX_TRACKING_GAP_MS, + ), + ); +} + +/** + * Inner part of a face box that a partial mode must alter. It sits inside the + * band every provider covers (eyes 22-55 %, upper 0-55 %, lower 50-100 % of + * the padded box), so the gate checks only pixels the mode promised to change. + */ +export function partialCheckBand(box: NormalizedBox, mode: PartialMode): NormalizedBox { + const [top, bottom] = + mode === "partial_eyes" ? [0.3, 0.48] : mode === "partial_upper" ? [0.05, 0.5] : [0.55, 0.95]; + return { + x: box.x + box.w * 0.1, + y: box.y + box.h * top, + w: box.w * 0.8, + h: box.h * (bottom - top), + }; +} + +/** Crop window in source pixels with the output aspect, as the compiler sizes it. */ +export function cropPixels( + geometry: { sourceWidth: number; sourceHeight: number; outWidth: number; outHeight: number }, + keyframe: { cx: number; cy: number; scale: number }, +): { x: number; y: number; w: number; h: number } { + const outAspect = geometry.outWidth / geometry.outHeight; + let w: number; + let h: number; + if (geometry.sourceWidth / geometry.sourceHeight > outAspect) { + h = geometry.sourceHeight; + w = h * outAspect; + } else { + w = geometry.sourceWidth; + h = w / outAspect; + } + const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2); + const even = (v: number) => Math.max(0, Math.floor(v / 2) * 2); + const cw = Math.min(geometry.sourceWidth - (geometry.sourceWidth % 2), roundEven(w / keyframe.scale)); + const ch = Math.min( + geometry.sourceHeight - (geometry.sourceHeight % 2), + roundEven(h / keyframe.scale), + ); + const x = even( + Math.min(Math.max(0, keyframe.cx * geometry.sourceWidth - cw / 2), geometry.sourceWidth - cw), + ); + const y = even( + Math.min(Math.max(0, keyframe.cy * geometry.sourceHeight - ch / 2), geometry.sourceHeight - ch), + ); + return { x, y, w: cw, h: ch }; +} + +/** Map source gaps of one clip onto the output timeline (intro included). */ +export function gapToOutput( + edl: Edl, + clip: EdlVideoClip, + gap: TimeWindow, +): { startMs: number; endMs: number } | null { + const start = Math.max(gap.startMs, clip.inMs); + const end = Math.min(gap.endMs, clip.outMs); + if (end <= start) return null; + const intro = edl.intro?.durationMs ?? 0; + const toOutput = (t: number) => Math.round(intro + clip.atMs + (t - clip.inMs) / clip.speed); + return { startMs: toOutput(start), endMs: toOutput(end) }; +} + +// --------------------------------------------------------------------------- +// Privacy record (prepareSources → gate) +// --------------------------------------------------------------------------- + +const PixelBoxSchema = z.object({ + x: z.number().int().min(0), + y: z.number().int().min(0), + w: z.number().int().min(1), + h: z.number().int().min(1), +}); + +export const PrivacyRecordSchema = z.object({ + version: z.literal(1), + assets: z.record( + z.string(), + z.object({ + basePath: z.string(), + preparedPath: z.string(), + still: z.boolean(), + /** Static crop applied for reframe_out, in base display pixels. */ + crop: PixelBoxSchema.nullable(), + marks: z.array( + z.object({ + markId: z.string(), + treatment: MarkTreatmentSchema, + /** Absolute mask directory (`.png` per frame) or null for box-only regions. */ + maskDir: z.string().nullable(), + keyframes: z.array(BoxKeyframeSchema), + }), + ), + partialFaces: z.array( + z.object({ + personId: z.string().nullable(), + mode: z.enum(["partial_eyes", "partial_upper", "partial_lower"]), + keyframes: z.array(BoxKeyframeSchema), + }), + ), + }), + ), + /** Source-time gaps over MAX_TRACKING_GAP_MS per clip. */ + trackingGaps: z.array( + z.object({ + clipId: z.string(), + startMs: z.number(), + endMs: z.number(), + personId: z.string().nullable(), + }), + ), + /** Person-bound marks re-found with low confidence, for review. */ + reviews: z.array(z.object({ markId: z.string(), assetId: z.string(), score: z.number() })), + aiAlterations: z.array(AiAlterationSchema), + /** Effective protection per person (EDL target over person default). */ + protectedPeople: z.record(z.string(), AnonymizeModeSchema), + /** People whose faces appear in the clip ranges. */ + visiblePeople: z.array(z.string()), + protectUnassigned: z.boolean(), +}); +export type PrivacyRecord = z.infer; + +export async function writePrivacyRecord(workDir: string, record: PrivacyRecord): Promise { + await writeFile(path.join(workDir, PRIVACY_RECORD_FILE), JSON.stringify(record)); +} + +export async function readPrivacyRecord(workDir: string): Promise { + let text: string; + try { + text = await readFile(path.join(workDir, PRIVACY_RECORD_FILE), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error( + "privacy record is missing: sources must be prepared in the render's work directory", + ); + } + throw error; + } + return PrivacyRecordSchema.parse(JSON.parse(text)); +} + +/** Stored face keyframe files: an array of keyframes or a tracking result with `keyframes`. */ +const StoredKeyframesSchema = z.union([ + z.array(BoxKeyframeSchema.passthrough()), + z.object({ keyframes: z.array(BoxKeyframeSchema.passthrough()) }).passthrough(), +]); + +export function parseStoredKeyframes(json: unknown): BoxKeyframe[] { + const parsed = StoredKeyframesSchema.parse(json); + const list = Array.isArray(parsed) ? parsed : parsed.keyframes; + return list.map((k) => ({ tMs: k.tMs, box: k.box })); +}