cynarAI/evOo-local
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/media-wip/w2-privacy' into worktree-agent-ac408b7c790dd9920

Browse Source
This commit is contained in:
Claude 2026-10-03 18:28:38 +00:00
commit d6957c4670
No known key found for this signature in database
2 changed files with 570 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
packages/adapters/src/media-privacy-pixels.ts Normal file
View File

@ -0,0 +1,179 @@
import { readFile } from "node:fs/promises";
import { inflateSync } from "node:zlib";
import type { NormalizedBox } from "@rakazo/contracts";
/*
* Pixel checks for the privacy gate. Frames come from MediaRenderer.extractFrame
* as PNG; the decoder below covers the PNGs ffmpeg and the mask writers produce
* (8-bit gray, gray+alpha, RGB, RGBA; not interlaced, not palette).
*/
export interface LumaFrame {
width: number;
height: number;
/** One luma byte per pixel, row-major. */
luma: Uint8Array;
}
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
const CHANNELS: Record<number, number> = { 0: 1, 2: 3, 4: 2, 6: 4 };
function paeth(a: number, b: number, c: number): number {
const p = a + b - c;
const pa = Math.abs(p - a);
const pb = Math.abs(p - b);
const pc = Math.abs(p - c);
if (pa <= pb && pa <= pc) return a;
return pb <= pc ? b : c;
}
/** Decode a PNG to luma (Rec. 601 weights). Throws on anything it cannot read exactly. */
export function decodePngLuma(png: Buffer): LumaFrame {
if (png.length < 8 || !png.subarray(0, 8).equals(PNG_SIGNATURE)) throw new Error("not a PNG");
let offset = 8;
let width = 0;
let height = 0;
let channels = 0;
const data: Buffer[] = [];
while (offset + 8 <= png.length) {
const length = png.readUInt32BE(offset);
const type = png.toString("ascii", offset + 4, offset + 8);
const body = png.subarray(offset + 8, offset + 8 + length);
offset += 12 + length;
if (type === "IHDR") {
width = body.readUInt32BE(0);
height = body.readUInt32BE(4);
const depth = body[8];
const colorType = body[9] ?? -1;
const interlace = body[12];
channels = CHANNELS[colorType] ?? 0;
if (depth !== 8 || channels === 0 || interlace !== 0)
throw new Error("unsupported PNG layout");
} else if (type === "IDAT") {
data.push(body);
} else if (type === "IEND") {
break;
}
}
if (width === 0 || height === 0 || channels === 0) throw new Error("PNG has no image header");
const raw = inflateSync(Buffer.concat(data));
const stride = width * channels;
if (raw.length < (stride + 1) * height) throw new Error("PNG data is truncated");
const pixels = Buffer.alloc(stride * height);
for (let y = 0; y < height; y++) {
const filter = raw[y * (stride + 1)];
const line = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1));
const row = y * stride;
for (let x = 0; x < stride; x++) {
const left = x >= channels ? (pixels[row + x - channels] ?? 0) : 0;
const up = y > 0 ? (pixels[row - stride + x] ?? 0) : 0;
const upLeft = y > 0 && x >= channels ? (pixels[row - stride + x - channels] ?? 0) : 0;
const value = line[x] ?? 0;
let out: number;
if (filter === 0) out = value;
else if (filter === 1) out = value + left;
else if (filter === 2) out = value + up;
else if (filter === 3) out = value + ((left + up) >> 1);
else if (filter === 4) out = value + paeth(left, up, upLeft);
else throw new Error("unknown PNG filter");
pixels[row + x] = out & 0xff;
}
}
const luma = new Uint8Array(width * height);
for (let i = 0; i < width * height; i++) {
const base = i * channels;
if (channels <= 2) {
luma[i] = pixels[base] ?? 0;
} else {
const r = pixels[base] ?? 0;
const g = pixels[base + 1] ?? 0;
const b = pixels[base + 2] ?? 0;
luma[i] = Math.round(0.299 * r + 0.587 * g + 0.114 * b);
}
}
return { width, height, luma };
}
export async function readPngLuma(file: string): Promise<LumaFrame> {
return decodePngLuma(await readFile(file));
}
/** Bounding box of the non-black pixels of a mask, or null when the mask is empty. */
export function maskBounds(mask: LumaFrame): NormalizedBox | null {
let minX = mask.width;
let minY = mask.height;
let maxX = -1;
let maxY = -1;
for (let y = 0; y < mask.height; y++) {
for (let x = 0; x < mask.width; x++) {
if ((mask.luma[y * mask.width + x] ?? 0) < 128) continue;
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
}
}
if (maxX < 0) return null;
return {
x: minX / mask.width,
y: minY / mask.height,
w: (maxX + 1 - minX) / mask.width,
h: (maxY + 1 - minY) / mask.height,
};
}
/** A luma step at least this large marks a detail pixel (an edge someone could recognise). */
export const DETAIL_MIN_STEP = 12;
/** A detail pixel within this many luma levels of the source counts as unchanged. */
export const UNCHANGED_MAX_DIFF = 2;
/** Regions with fewer detail pixels show nothing identifying. */
const MIN_DETAIL_PIXELS = 16;
/**
* Residual visibility of a region (0..1): the share of the source's detail
* pixels that the treated frame left unchanged. Blur, pixelation, fills, and
* inpainting all rewrite edges, so a treated region scores near 0, an untouched
* one 1. A flat region without detail scores 0: there is nothing to recognise.
*
* `source` and `treated` are pixel-aligned at full resolution; when `treated`
* is a crop of the source, `offset` is the crop origin in source pixels.
* Optional `mask` (normalized over the source frame) limits the region to mask pixels.
*/
export function residualVisibility(input: {
source: LumaFrame;
treated: LumaFrame;
box: NormalizedBox;
offset?: { x: number; y: number };
mask?: LumaFrame;
}): number {
const { source, treated, box, mask } = input;
const offset = input.offset ?? { x: 0, y: 0 };
const x0 = Math.max(0, Math.floor(box.x * source.width));
const y0 = Math.max(0, Math.floor(box.y * source.height));
const x1 = Math.min(source.width - 1, Math.ceil((box.x + box.w) * source.width) - 1);
const y1 = Math.min(source.height - 1, Math.ceil((box.y + box.h) * source.height) - 1);
let detail = 0;
let unchanged = 0;
for (let y = y0; y < y1; y++) {
for (let x = x0; x < x1; x++) {
if (mask) {
const mx = Math.min(mask.width - 1, Math.floor((x / source.width) * mask.width));
const my = Math.min(mask.height - 1, Math.floor((y / source.height) * mask.height));
if ((mask.luma[my * mask.width + mx] ?? 0) < 128) continue;
}
const at = y * source.width + x;
const value = source.luma[at] ?? 0;
const step =
Math.abs(value - (source.luma[at + 1] ?? value)) +
Math.abs(value - (source.luma[at + source.width] ?? value));
if (step < DETAIL_MIN_STEP) continue;
const tx = x - offset.x;
const ty = y - offset.y;
if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue;
detail += 1;
if (Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= UNCHANGED_MAX_DIFF)
unchanged += 1;
}
}
return detail < MIN_DETAIL_PIXELS ? 0 : unchanged / detail;
}

391
packages/adapters/src/media-privacy-plan.ts Normal file
View File

@ -0,0 +1,391 @@
import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import type {
AnonymizeMode,
BoxKeyframe,
Edl,
EdlVideoClip,
FaceRedaction,
NormalizedBox,
} from "@rakazo/contracts";
import {
AiAlterationSchema,
AnonymizeModeSchema,
BoxKeyframeSchema,
MarkTreatmentSchema,
} from "@rakazo/contracts";
import { cosineSimilarity, decodeVectors, MAX_TRACKING_GAP_MS } from "@rakazo/media";
import { z } from "zod";
/*
* Pure planning helpers for face, mark, and region protection, plus the
* privacy record that prepareSources leaves next to the render output for the
* gate. Times are source times of the asset unless a name says otherwise.
*/
/** Tracking rate for the re-detection pass on clip sources (faces the stored analysis missed). */
export const FACE_TRACK_FPS = 10;
/**
* Minimum cosine similarity between a face embedding and a person centroid to
* count as that person. Below it the face is unassigned, which fails closed:
* unassigned faces are redacted and block the gate while a protected person
* appears in the project.
*/
export const FACE_MATCH_MIN_COSINE = 0.6;
/** Step for linear interpolation across short detection gaps (<= MAX_TRACKING_GAP_MS). */
export const INTERPOLATE_STEP_MS = 100;
/** Longer gaps hold the last box, enlarged by this share on every side (spec 7). */
export const GAP_HOLD_GROWTH = 0.25;
/** Padding around marks and regions, in percent of the box. */
export const MARK_PADDING_PCT = 10;
/** Name of the privacy record prepareSources writes into the render work directory. */
export const PRIVACY_RECORD_FILE = "privacy.json";
export type ProtectMode = Exclude<AnonymizeMode, "none">;
export type PartialMode = "partial_eyes" | "partial_upper" | "partial_lower";
export interface Protection {
mode: ProtectMode;
paddingPct: number;
featherPct: number;
minSizePct: number;
/** Redaction id from the EDL, "default" (person setting), or "unassigned". */
origin: string;
}
export interface CandidateTrack {
/** Stored track id, or null for faces found by the re-detection pass. */
storedId: string | null;
personId: string | null;
keyframes: BoxKeyframe[];
}
export interface TimeWindow {
startMs: number;
endMs: number;
}
const DEFAULT_PROTECTION = { paddingPct: 20, featherPct: 10, minSizePct: 2 };
export function isPartial(mode: AnonymizeMode): mode is PartialMode {
return mode === "partial_eyes" || mode === "partial_upper" || mode === "partial_lower";
}
function fromRedaction(redaction: FaceRedaction): Protection {
return {
mode: redaction.mode,
paddingPct: redaction.paddingPct,
featherPct: redaction.featherPct,
minSizePct: redaction.minSizePct,
origin: redaction.id,
};
}
/**
* Protection for one face track. Most specific wins: EDL track target, EDL
* person target, EDL unassigned target, the person's default, and finally a
* blur for unassigned faces while any protected person appears in the project.
*/
export function resolveProtection(
track: Pick<CandidateTrack, "storedId" | "personId">,
context: {
faces: FaceRedaction[];
defaults: Map<string, AnonymizeMode>;
protectUnassigned: boolean;
},
): Protection | null {
const byTrack = context.faces.find(
(face) => "trackId" in face.target && face.target.trackId === track.storedId,
);
if (byTrack && track.storedId) return fromRedaction(byTrack);
if (track.personId) {
const byPerson = context.faces.find(
(face) => "personId" in face.target && face.target.personId === track.personId,
);
if (byPerson) return fromRedaction(byPerson);
const mode = context.defaults.get(track.personId);
if (mode && mode !== "none") return { mode, ...DEFAULT_PROTECTION, origin: "default" };
return null;
}
const unassigned = context.faces.find((face) => "unassignedFaces" in face.target);
if (unassigned) return fromRedaction(unassigned);
if (context.protectUnassigned) return { mode: "blur", ...DEFAULT_PROTECTION, origin: "unassigned" };
return null;
}
/** Best person for an embedding by cosine to the centroids, or null below the threshold. */
export function matchPerson(
embedding: ArrayLike<number> | null,
centroids: Array<{ personId: string; centroid: Uint8Array }>,
): string | null {
if (!embedding || embedding.length === 0) return null;
let best: { personId: string; score: number } | null = null;
for (const { personId, centroid } of centroids) {
if (centroid.byteLength !== embedding.length * 4) continue;
const vector = decodeVectors(centroid, embedding.length)[0];
if (!vector) continue;
const score = cosineSimilarity(embedding, vector);
if (score >= FACE_MATCH_MIN_COSINE && (!best || score > best.score)) best = { personId, score };
}
return best?.personId ?? null;
}
const clampUnit = (value: number) => Math.min(1, Math.max(0, value));
/** Grow a box by `share` of its size on every side, clamped to the frame. */
export function growBox(box: NormalizedBox, share: number): NormalizedBox {
const x = clampUnit(box.x - box.w * share);
const y = clampUnit(box.y - box.h * share);
return {
x,
y,
w: clampUnit(box.x + box.w + box.w * share) - x,
h: clampUnit(box.y + box.h + box.h * share) - y,
};
}
/** Enlarge a box around its centre until both sides reach `minSizePct` of the frame. */
export function ensureMinSize(box: NormalizedBox, minSizePct: number): NormalizedBox {
const min = minSizePct / 100;
if (box.w >= min && box.h >= min) return box;
const w = Math.max(box.w, min);
const h = Math.max(box.h, min);
const x = clampUnit(box.x + box.w / 2 - w / 2);
const y = clampUnit(box.y + box.h / 2 - h / 2);
return { x, y, w: Math.min(w, 1 - x), h: Math.min(h, 1 - y) };
}
export function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox {
const x = Math.min(a.x, b.x);
const y = Math.min(a.y, b.y);
return {
x,
y,
w: Math.max(a.x + a.w, b.x + b.w) - x,
h: Math.max(a.y + a.h, b.y + b.h) - y,
};
}
export function padBox(box: NormalizedBox, paddingPct: number): NormalizedBox {
return growBox(box, Math.max(0, paddingPct) / 200);
}
function lerpBox(a: NormalizedBox, b: NormalizedBox, f: number): NormalizedBox {
const mix = (p: number, q: number) => p + (q - p) * f;
return { x: mix(a.x, b.x), y: mix(a.y, b.y), w: mix(a.w, b.w), h: mix(a.h, b.h) };
}
const round4 = (value: number) => Math.round(value * 10_000) / 10_000;
const roundBox = (box: NormalizedBox): NormalizedBox => ({
x: round4(box.x),
y: round4(box.y),
w: round4(box.w),
h: round4(box.h),
});
/**
* Keyframes for redaction inside the windows (clip ranges plus the bridging
* limit on both sides). Gaps up to MAX_TRACKING_GAP_MS are interpolated;
* longer ones hold the last box grown by GAP_HOLD_GROWTH until the face is
* found again and are returned so the gate blocks them (needs review).
*/
export function protectKeyframes(
keyframes: BoxKeyframe[],
windows: TimeWindow[],
minSizePct: number,
): { keyframes: BoxKeyframe[]; gaps: TimeWindow[] } {
const inside = [...keyframes]
.filter((k) =>
windows.some(
(w) => k.tMs >= w.startMs - MAX_TRACKING_GAP_MS && k.tMs <= w.endMs + MAX_TRACKING_GAP_MS,
),
)
.sort((a, b) => a.tMs - b.tMs)
.map((k) => ({ tMs: k.tMs, box: ensureMinSize(k.box, minSizePct) }));
const out: BoxKeyframe[] = [];
const gaps: TimeWindow[] = [];
inside.forEach((current, index) => {
out.push({ tMs: current.tMs, box: roundBox(current.box) });
const next = inside[index + 1];
if (!next) return;
const span = next.tMs - current.tMs;
if (span <= MAX_TRACKING_GAP_MS) {
for (let t = current.tMs + INTERPOLATE_STEP_MS; t < next.tMs; t += INTERPOLATE_STEP_MS) {
out.push({
tMs: t,
box: roundBox(lerpBox(current.box, next.box, (t - current.tMs) / span)),
});
}
return;
}
const held = roundBox(growBox(current.box, GAP_HOLD_GROWTH));
out.push({ tMs: current.tMs + 1, box: held }, { tMs: next.tMs - 1, box: held });
gaps.push({ startMs: current.tMs, endMs: next.tMs });
});
return { keyframes: out, gaps };
}
/** Source windows each clip of an asset shows. */
export function clipWindows(clips: EdlVideoClip[]): TimeWindow[] {
return clips.map((clip) => ({ startMs: clip.inMs, endMs: clip.outMs }));
}
export function overlapsWindows(keyframes: BoxKeyframe[], windows: TimeWindow[]): boolean {
return keyframes.some((k) =>
windows.some(
(w) => k.tMs >= w.startMs - MAX_TRACKING_GAP_MS && k.tMs <= w.endMs + MAX_TRACKING_GAP_MS,
),
);
}
/**
* Inner part of a face box that a partial mode must alter. It sits inside the
* band every provider covers (eyes 22-55 %, upper 0-55 %, lower 50-100 % of
* the padded box), so the gate checks only pixels the mode promised to change.
*/
export function partialCheckBand(box: NormalizedBox, mode: PartialMode): NormalizedBox {
const [top, bottom] =
mode === "partial_eyes" ? [0.3, 0.48] : mode === "partial_upper" ? [0.05, 0.5] : [0.55, 0.95];
return {
x: box.x + box.w * 0.1,
y: box.y + box.h * top,
w: box.w * 0.8,
h: box.h * (bottom - top),
};
}
/** Crop window in source pixels with the output aspect, as the compiler sizes it. */
export function cropPixels(
geometry: { sourceWidth: number; sourceHeight: number; outWidth: number; outHeight: number },
keyframe: { cx: number; cy: number; scale: number },
): { x: number; y: number; w: number; h: number } {
const outAspect = geometry.outWidth / geometry.outHeight;
let w: number;
let h: number;
if (geometry.sourceWidth / geometry.sourceHeight > outAspect) {
h = geometry.sourceHeight;
w = h * outAspect;
} else {
w = geometry.sourceWidth;
h = w / outAspect;
}
const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2);
const even = (v: number) => Math.max(0, Math.floor(v / 2) * 2);
const cw = Math.min(geometry.sourceWidth - (geometry.sourceWidth % 2), roundEven(w / keyframe.scale));
const ch = Math.min(
geometry.sourceHeight - (geometry.sourceHeight % 2),
roundEven(h / keyframe.scale),
);
const x = even(
Math.min(Math.max(0, keyframe.cx * geometry.sourceWidth - cw / 2), geometry.sourceWidth - cw),
);
const y = even(
Math.min(Math.max(0, keyframe.cy * geometry.sourceHeight - ch / 2), geometry.sourceHeight - ch),
);
return { x, y, w: cw, h: ch };
}
/** Map source gaps of one clip onto the output timeline (intro included). */
export function gapToOutput(
edl: Edl,
clip: EdlVideoClip,
gap: TimeWindow,
): { startMs: number; endMs: number } | null {
const start = Math.max(gap.startMs, clip.inMs);
const end = Math.min(gap.endMs, clip.outMs);
if (end <= start) return null;
const intro = edl.intro?.durationMs ?? 0;
const toOutput = (t: number) => Math.round(intro + clip.atMs + (t - clip.inMs) / clip.speed);
return { startMs: toOutput(start), endMs: toOutput(end) };
}
// ---------------------------------------------------------------------------
// Privacy record (prepareSources → gate)
// ---------------------------------------------------------------------------
const PixelBoxSchema = z.object({
x: z.number().int().min(0),
y: z.number().int().min(0),
w: z.number().int().min(1),
h: z.number().int().min(1),
});
export const PrivacyRecordSchema = z.object({
version: z.literal(1),
assets: z.record(
z.string(),
z.object({
basePath: z.string(),
preparedPath: z.string(),
still: z.boolean(),
/** Static crop applied for reframe_out, in base display pixels. */
crop: PixelBoxSchema.nullable(),
marks: z.array(
z.object({
markId: z.string(),
treatment: MarkTreatmentSchema,
/** Absolute mask directory (`<ms>.png` per frame) or null for box-only regions. */
maskDir: z.string().nullable(),
keyframes: z.array(BoxKeyframeSchema),
}),
),
partialFaces: z.array(
z.object({
personId: z.string().nullable(),
mode: z.enum(["partial_eyes", "partial_upper", "partial_lower"]),
keyframes: z.array(BoxKeyframeSchema),
}),
),
}),
),
/** Source-time gaps over MAX_TRACKING_GAP_MS per clip. */
trackingGaps: z.array(
z.object({
clipId: z.string(),
startMs: z.number(),
endMs: z.number(),
personId: z.string().nullable(),
}),
),
/** Person-bound marks re-found with low confidence, for review. */
reviews: z.array(z.object({ markId: z.string(), assetId: z.string(), score: z.number() })),
aiAlterations: z.array(AiAlterationSchema),
/** Effective protection per person (EDL target over person default). */
protectedPeople: z.record(z.string(), AnonymizeModeSchema),
/** People whose faces appear in the clip ranges. */
visiblePeople: z.array(z.string()),
protectUnassigned: z.boolean(),
});
export type PrivacyRecord = z.infer<typeof PrivacyRecordSchema>;
export async function writePrivacyRecord(workDir: string, record: PrivacyRecord): Promise<void> {
await writeFile(path.join(workDir, PRIVACY_RECORD_FILE), JSON.stringify(record));
}
export async function readPrivacyRecord(workDir: string): Promise<PrivacyRecord> {
let text: string;
try {
text = await readFile(path.join(workDir, PRIVACY_RECORD_FILE), "utf8");
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
throw new Error(
"privacy record is missing: sources must be prepared in the render's work directory",
);
}
throw error;
}
return PrivacyRecordSchema.parse(JSON.parse(text));
}
/** Stored face keyframe files: an array of keyframes or a tracking result with `keyframes`. */
const StoredKeyframesSchema = z.union([
z.array(BoxKeyframeSchema.passthrough()),
z.object({ keyframes: z.array(BoxKeyframeSchema.passthrough()) }).passthrough(),
]);
export function parseStoredKeyframes(json: unknown): BoxKeyframe[] {
const parsed = StoredKeyframesSchema.parse(json);
const list = Array.isArray(parsed) ? parsed : parsed.keyframes;
return list.map((k) => ({ tMs: k.tMs, box: k.box }));
}