import { readFile } from "node:fs/promises"; import { inflateSync } from "node:zlib"; import type { NormalizedBox } from "@rakazo/contracts"; import { runMediaTool } from "./media-folder-source-probe.js"; /* * Pixel checks for the privacy gate and mark quality. Video frames are decoded * by ffmpeg straight to 8-bit luma; mask PNGs from the segmenter go through the * decoder below (8-bit gray, gray+alpha, RGB, RGBA; not interlaced, not palette). */ export interface LumaFrame { width: number; height: number; /** One luma byte per pixel, row-major. */ luma: Uint8Array; } const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); const CHANNELS: Record = { 0: 1, 2: 3, 4: 2, 6: 4 }; function paeth(a: number, b: number, c: number): number { const p = a + b - c; const pa = Math.abs(p - a); const pb = Math.abs(p - b); const pc = Math.abs(p - c); if (pa <= pb && pa <= pc) return a; return pb <= pc ? b : c; } /** Decode a PNG to luma (Rec. 601 weights). Throws on anything it cannot read exactly. */ export function decodePngLuma(png: Buffer): LumaFrame { if (png.length < 8 || !png.subarray(0, 8).equals(PNG_SIGNATURE)) throw new Error("not a PNG"); let offset = 8; let width = 0; let height = 0; let channels = 0; const data: Buffer[] = []; while (offset + 8 <= png.length) { const length = png.readUInt32BE(offset); const type = png.toString("ascii", offset + 4, offset + 8); const body = png.subarray(offset + 8, offset + 8 + length); offset += 12 + length; if (type === "IHDR") { width = body.readUInt32BE(0); height = body.readUInt32BE(4); const depth = body[8]; const colorType = body[9] ?? -1; const interlace = body[12]; channels = CHANNELS[colorType] ?? 0; if (depth !== 8 || channels === 0 || interlace !== 0) throw new Error("unsupported PNG layout"); } else if (type === "IDAT") { data.push(body); } else if (type === "IEND") { break; } } if (width === 0 || height === 0 || channels === 0) throw new Error("PNG has no image header"); const raw = inflateSync(Buffer.concat(data)); const stride = width * channels; if (raw.length < (stride + 1) * height) throw new Error("PNG data is truncated"); const pixels = Buffer.alloc(stride * height); for (let y = 0; y < height; y++) { const filter = raw[y * (stride + 1)]; const line = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1)); const row = y * stride; for (let x = 0; x < stride; x++) { const left = x >= channels ? (pixels[row + x - channels] ?? 0) : 0; const up = y > 0 ? (pixels[row - stride + x] ?? 0) : 0; const upLeft = y > 0 && x >= channels ? (pixels[row - stride + x - channels] ?? 0) : 0; const value = line[x] ?? 0; let out: number; if (filter === 0) out = value; else if (filter === 1) out = value + left; else if (filter === 2) out = value + up; else if (filter === 3) out = value + ((left + up) >> 1); else if (filter === 4) out = value + paeth(left, up, upLeft); else throw new Error("unknown PNG filter"); pixels[row + x] = out & 0xff; } } const luma = new Uint8Array(width * height); for (let i = 0; i < width * height; i++) { const base = i * channels; if (channels <= 2) { luma[i] = pixels[base] ?? 0; } else { const r = pixels[base] ?? 0; const g = pixels[base + 1] ?? 0; const b = pixels[base + 2] ?? 0; luma[i] = Math.round(0.299 * r + 0.587 * g + 0.114 * b); } } return { width, height, luma }; } export async function readPngLuma(file: string): Promise { return decodePngLuma(await readFile(file)); } /** * Decode the frame shown at `tMs` as luma, scaled to `width` x `height` with * nearest-neighbour sampling (a no-op at the display size). Throws unless * ffmpeg returns a full frame. */ export async function extractLuma( file: string, input: { tMs: number; width: number; height: number; still?: boolean }, options: { ffmpegPath?: string; signal?: AbortSignal } = {}, ): Promise { const seek = input.still ? [] : ["-ss", (Math.max(0, input.tMs) / 1000).toFixed(3)]; const { stdout } = await runMediaTool( options.ffmpegPath ?? "ffmpeg", [ "-v", "error", ...seek, "-i", file, "-frames:v", "1", "-vf", `scale=${input.width}:${input.height}:flags=neighbor`, "-f", "rawvideo", "-pix_fmt", "gray", "-", ], { signal: options.signal }, ); const size = input.width * input.height; if (stdout.length < size) throw new Error(`no frame at ${input.tMs} ms`); return { width: input.width, height: input.height, luma: new Uint8Array(stdout.subarray(0, size)), }; } /** Bounding box of the bright (>= 128) pixels of a mask, or null when the mask is empty. */ export function maskBounds(mask: LumaFrame): NormalizedBox | null { let minX = mask.width; let minY = mask.height; let maxX = -1; let maxY = -1; for (let y = 0; y < mask.height; y++) { for (let x = 0; x < mask.width; x++) { if ((mask.luma[y * mask.width + x] ?? 0) < 128) continue; minX = Math.min(minX, x); minY = Math.min(minY, y); maxX = Math.max(maxX, x); maxY = Math.max(maxY, y); } } if (maxX < 0) return null; return { x: minX / mask.width, y: minY / mask.height, w: (maxX + 1 - minX) / mask.width, h: (maxY + 1 - minY) / mask.height, }; } /** A luma step at least this large marks a detail pixel (an edge someone could recognise). */ export const DETAIL_MIN_STEP = 12; /** A detail pixel within this many luma levels of the source counts as unchanged. */ export const UNCHANGED_MAX_DIFF = 2; /** Looser tolerance for the control area outside treated regions (codec noise). */ const CONTROL_MAX_DIFF = 6; /** Regions with fewer detail pixels show nothing identifying. */ const MIN_DETAIL_PIXELS = 16; function pixelRange(box: NormalizedBox, width: number, height: number) { return { x0: Math.max(0, Math.floor(box.x * width)), y0: Math.max(0, Math.floor(box.y * height)), x1: Math.min(width, Math.ceil((box.x + box.w) * width)), y1: Math.min(height, Math.ceil((box.y + box.h) * height)), }; } function detailStep(frame: LumaFrame, x: number, y: number): number { const at = y * frame.width + x; const value = frame.luma[at] ?? 0; const right = x + 1 < frame.width ? (frame.luma[at + 1] ?? value) : value; const down = y + 1 < frame.height ? (frame.luma[at + frame.width] ?? value) : value; return Math.abs(value - right) + Math.abs(value - down); } /** * Residual visibility of a region (0..1): the share of the source's detail * pixels (edges) that the treated frame left unchanged, value and edge * strength alike. Blur, pixelation, fills, and inpainting all rewrite edges, * so a treated region scores near 0, an untouched one 1. A flat region * without detail scores 0: there is nothing to recognise. * * `source` and `treated` are pixel-aligned at full resolution; when `treated` * is a crop of the source, `offset` is the crop origin in source pixels and * detail outside the crop is invisible. Optional `mask` (normalized over the * source frame) limits the region to mask pixels. */ export function residualVisibility(input: { source: LumaFrame; treated: LumaFrame; box: NormalizedBox; offset?: { x: number; y: number }; mask?: LumaFrame; }): number { const { source, treated, box, mask } = input; const offset = input.offset ?? { x: 0, y: 0 }; const { x0, y0, x1, y1 } = pixelRange(box, source.width, source.height); let detail = 0; let unchanged = 0; for (let y = y0; y < y1; y++) { for (let x = x0; x < x1; x++) { if (mask) { const mx = Math.min(mask.width - 1, Math.floor((x / source.width) * mask.width)); const my = Math.min(mask.height - 1, Math.floor((y / source.height) * mask.height)); if ((mask.luma[my * mask.width + mx] ?? 0) < 128) continue; } const step = detailStep(source, x, y); if (step < DETAIL_MIN_STEP) continue; const tx = x - offset.x; const ty = y - offset.y; if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue; detail += 1; const value = source.luma[y * source.width + x] ?? 0; // Visible means the pixel and its edge survived, not a value that happens to match. if ( Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= UNCHANGED_MAX_DIFF && Math.abs(step - detailStep(treated, tx, ty)) <= 2 * UNCHANGED_MAX_DIFF ) unchanged += 1; } } return detail < MIN_DETAIL_PIXELS ? 0 : unchanged / detail; } /** * Share of detail pixels outside every treated box that the treated frame kept * (within codec tolerance). A low value means the two frames are not the same * moment or framing, so a residual measured between them proves nothing. * Returns 1 when the control area has too little detail to judge. */ export function controlAgreement(input: { source: LumaFrame; treated: LumaFrame; boxes: NormalizedBox[]; offset?: { x: number; y: number }; }): number { const { source, treated, boxes } = input; const offset = input.offset ?? { x: 0, y: 0 }; const ranges = boxes.map((box) => pixelRange(box, source.width, source.height)); let detail = 0; let kept = 0; for (let y = 0; y < source.height; y++) { for (let x = 0; x < source.width; x++) { const tx = x - offset.x; const ty = y - offset.y; if (tx < 0 || ty < 0 || tx >= treated.width || ty >= treated.height) continue; if (ranges.some((r) => x >= r.x0 - 2 && x < r.x1 + 2 && y >= r.y0 - 2 && y < r.y1 + 2)) continue; if (detailStep(source, x, y) < DETAIL_MIN_STEP) continue; detail += 1; const value = source.luma[y * source.width + x] ?? 0; if (Math.abs(value - (treated.luma[ty * treated.width + tx] ?? 0)) <= CONTROL_MAX_DIFF) kept += 1; } } return detail < MIN_DETAIL_PIXELS ? 1 : kept / detail; } /** * Temporal stability of a treated region over consecutive frames (0..1): * 1 when the region changes no more between neighbours than the rest of the * frame, falling to 0 as it flickers 24+ luma levels more on average. */ export function temporalStability(frames: LumaFrame[], box: NormalizedBox): number { if (frames.length < 2) return 1; let excess = 0; for (let i = 1; i < frames.length; i++) { const a = frames[i - 1]!; const b = frames[i]!; if (a.width !== b.width || a.height !== b.height) throw new Error("frame sizes differ"); const r = pixelRange(box, a.width, a.height); let inside = 0; let insideCount = 0; let outside = 0; let outsideCount = 0; for (let y = 0; y < a.height; y++) { for (let x = 0; x < a.width; x++) { const at = y * a.width + x; const diff = Math.abs((a.luma[at] ?? 0) - (b.luma[at] ?? 0)); if (x >= r.x0 && x < r.x1 && y >= r.y0 && y < r.y1) { inside += diff; insideCount += 1; } else { outside += diff; outsideCount += 1; } } } const meanInside = insideCount ? inside / insideCount : 0; const meanOutside = outsideCount ? outside / outsideCount : 0; excess += Math.max(0, meanInside - meanOutside); } return Math.max(0, Math.min(1, 1 - excess / (frames.length - 1) / 24)); }