"""redact: frame-accurate face and region redaction (OpenCV + ffmpeg pipes). Geometry per instruction and frame: - Boxes interpolate linearly between keyframes. A single keyframe is a static region for the whole video; otherwise the region applies from the first to the last keyframe (plus half a frame either side). - `paddingPct` grows the box by that percentage of its width/height (half on each side). Mask regions are dilated by the same share of their extent. - `featherPct` fades the effect outward over that percentage of the region's shorter side. The region itself is always fully covered. Fails closed: an unreadable frame, a mask that cannot be read, or an encoder error is a 500 and no file is left at outPath. """ from __future__ import annotations import bisect import os import threading from dataclasses import dataclass from typing import Any import cv2 import numpy as np from .. import media_io, paths from ..errors import BadRequest from ..schemas import RedactionInstruction, RedactRequest from .common import Ctx, open_input from .masks import MaskIndex OUTPUT_EXTENSIONS = (".mkv", ".mp4", ".mov") DEFAULT_FILL = (0, 0, 0) def hex_to_bgr(color: str | None) -> tuple[int, int, int] | None: if not color: return None value = color.lstrip("#") r, g, b = int(value[0:2], 16), int(value[2:4], 16), int(value[4:6], 16) return (b, g, r) @dataclass class Plan: mode: str padding: float feather: float color: tuple[int, int, int] | None times: list[int] boxes: list[tuple[float, float, float, float]] landmark_times: list[int] landmarks: list[list[tuple[float, float]]] masks: MaskIndex | None def build_plan(ctx: Ctx, instruction: RedactionInstruction, frame_ms: float) -> Plan: keyframes = sorted(instruction.keyframes, key=lambda k: k.tMs) landmarks = sorted(instruction.landmarks or [], key=lambda frame: frame.tMs) masks = None if instruction.maskDir: masks = MaskIndex(paths.input_dir(ctx.settings.media_root, instruction.maskDir), frame_ms) return Plan( mode=instruction.mode, padding=instruction.paddingPct / 100.0, feather=instruction.featherPct / 100.0, color=hex_to_bgr(instruction.color), times=[k.tMs for k in keyframes], boxes=[(k.box.x, k.box.y, k.box.w, k.box.h) for k in keyframes], landmark_times=[frame.tMs for frame in landmarks], landmarks=[[(p.x, p.y) for p in frame.points] for frame in landmarks], masks=masks, ) def interpolate(times: list[int], values: list[Any], t_ms: float, frame_ms: float, hold_single: bool) -> Any: """Linear interpolation of tuples/lists over keyframe times; None outside the span.""" if not times: return None if len(times) == 1: if hold_single or abs(t_ms - times[0]) <= frame_ms / 2: return values[0] return None if t_ms < times[0] - frame_ms / 2 or t_ms > times[-1] + frame_ms / 2: return None if t_ms <= times[0]: return values[0] if t_ms >= times[-1]: return values[-1] i = bisect.bisect_right(times, t_ms) t0, t1 = times[i - 1], times[i] a, b = values[i - 1], values[i] if t1 == t0: return b f = (t_ms - t0) / (t1 - t0) if isinstance(a, list): if len(a) != len(b): return a if f < 0.5 else b return [tuple(pa[k] + (pb[k] - pa[k]) * f for k in range(2)) for pa, pb in zip(a, b, strict=True)] return tuple(a[k] + (b[k] - a[k]) * f for k in range(len(a))) def _pixel_box( box: tuple[float, float, float, float], padding: float, width: int, height: int ) -> tuple[int, int, int, int]: x, y, w, h = box x -= w * padding / 2 y -= h * padding / 2 w *= 1 + padding h *= 1 + padding x0 = int(np.floor(max(0.0, x) * width)) y0 = int(np.floor(max(0.0, y) * height)) x1 = int(np.ceil(min(1.0, x + w) * width)) y1 = int(np.ceil(min(1.0, y + h) * height)) return x0, y0, max(x1, x0), max(y1, y0) def blur_effect(roi: np.ndarray, size: int) -> np.ndarray: """Strong, size-scaled blur: downsample, upsample, Gaussian.""" h, w = roi.shape[:2] factor = max(4, size // 6) small = cv2.resize(roi, (max(1, w // factor), max(1, h // factor)), interpolation=cv2.INTER_AREA) up = cv2.resize(small, (w, h), interpolation=cv2.INTER_LINEAR) k = max(3, (size // 3) | 1) return cv2.GaussianBlur(up, (k, k), 0) def pixelate_effect(roi: np.ndarray, size: int) -> np.ndarray: h, w = roi.shape[:2] block = max(4, size // 10) small = cv2.resize(roi, (max(1, -(-w // block)), max(1, -(-h // block))), interpolation=cv2.INTER_AREA) return cv2.resize(small, (w, h), interpolation=cv2.INTER_NEAREST) def _ring_color(frame: np.ndarray, region: np.ndarray, margin: int) -> tuple[int, int, int]: """Median colour just outside the region: the 'skin tone patch' cover fill.""" kernel = np.ones((2 * margin + 1, 2 * margin + 1), np.uint8) ring = (cv2.dilate(region, kernel) > 0) & (region == 0) if not ring.any(): return DEFAULT_FILL b, g, r = np.median(frame[ring], axis=0) return (int(b), int(g), int(r)) def _region_for( plan: Plan, t_ms: int, frame_ms: float, width: int, height: int ) -> tuple[np.ndarray, int] | None: """Binary region mask (uint8 0/255) and its characteristic size in px.""" region = np.zeros((height, width), np.uint8) size = 0 hold = plan.masks is None box = interpolate(plan.times, plan.boxes, t_ms, frame_ms, hold_single=hold) if box is not None: x0, y0, x1, y1 = _pixel_box(box, plan.padding, width, height) raw_w, raw_h = box[2] * width, box[3] * height if plan.mode in ("partial_eyes", "partial_upper", "partial_lower"): _partial_region(plan, region, box, (x0, y0, x1, y1), t_ms, frame_ms, width, height) else: region[y0:y1, x0:x1] = 255 size = max(size, int(min(raw_w, raw_h))) if plan.masks is not None: mask = plan.masks.lookup(t_ms, width, height) if mask is not None: ys, xs = np.nonzero(mask) if xs.size: extent = int(min(xs.max() - xs.min() + 1, ys.max() - ys.min() + 1)) grow = int(round(extent * plan.padding / 2)) if grow > 0: mask = cv2.dilate(mask, np.ones((2 * grow + 1, 2 * grow + 1), np.uint8)) region |= mask size = max(size, extent) if not region.any(): return None return region, max(size, 1) def _partial_region( plan: Plan, region: np.ndarray, box: tuple[float, float, float, float], padded: tuple[int, int, int, int], t_ms: int, frame_ms: float, width: int, height: int, ) -> None: x0, y0, x1, y1 = padded bx, by, bw, bh = box points = interpolate(plan.landmark_times, plan.landmarks, t_ms, frame_ms, hold_single=False) # Five-point landmarks: right eye, left eye, nose, mouth corners. eyes = points[:2] if points and len(points) >= 2 else None nose_y = points[2][1] if points and len(points) >= 3 else by + bh * 0.55 if plan.mode == "partial_eyes": if eyes: (ex1, ey1), (ex2, ey2) = eyes else: ex1, ey1, ex2, ey2 = bx + bw * 0.3, by + bh * 0.4, bx + bw * 0.7, by + bh * 0.4 cx, cy = (ex1 + ex2) / 2 * width, (ey1 + ey2) / 2 * height angle = float(np.degrees(np.arctan2((ey2 - ey1) * height, (ex2 - ex1) * width))) band_w = (x1 - x0) * 1.0 band_h = max(bh * height * 0.32 * (1 + plan.padding), 2.0) rect = cv2.boxPoints(((cx, cy), (band_w, band_h), angle)).astype(np.int32) cv2.fillConvexPoly(region, rect, 255) elif plan.mode == "partial_upper": split = int(round(min(max(nose_y, by), by + bh) * height)) region[y0 : max(split, y0), x0:x1] = 255 else: split = int(round(min(max(nose_y, by), by + bh) * height)) region[min(split, y1) : y1, x0:x1] = 255 def apply_plan(frame: np.ndarray, plan: Plan, t_ms: int, frame_ms: float) -> None: height, width = frame.shape[:2] found = _region_for(plan, t_ms, frame_ms, width, height) if found is None: return region, size = found radius = int(round(plan.feather * size)) ys, xs = np.nonzero(region) margin = radius + 2 x0, x1 = max(0, int(xs.min()) - margin), min(width, int(xs.max()) + margin + 1) y0, y1 = max(0, int(ys.min()) - margin), min(height, int(ys.max()) + margin + 1) roi = frame[y0:y1, x0:x1] roi_region = region[y0:y1, x0:x1] mode = plan.mode if mode in ("blur", "partial_upper", "partial_lower"): effected = blur_effect(roi, size) elif mode == "pixelate": effected = pixelate_effect(roi, size) else: if mode == "cover" and plan.color is None: fill = _ring_color(frame, region, max(4, size // 20)) else: fill = plan.color or DEFAULT_FILL effected = np.empty_like(roi) effected[:] = fill alpha = roi_region.astype(np.float32) / 255.0 if radius > 0: kernel = np.ones((2 * radius + 1, 2 * radius + 1), np.uint8) grown = cv2.dilate(roi_region, kernel).astype(np.float32) / 255.0 k = 2 * radius + 1 soft = cv2.GaussianBlur(grown, (k, k), 0) alpha = np.maximum(alpha, soft) alpha = alpha[..., None] blended = roi.astype(np.float32) * (1.0 - alpha) + effected.astype(np.float32) * alpha roi[:] = np.clip(np.rint(blended), 0, 255).astype(np.uint8) def apply_redactions(frame: np.ndarray, t_ms: int, plans: list[Plan], frame_ms: float) -> np.ndarray: for plan in plans: apply_plan(frame, plan, t_ms, frame_ms) return frame def run(ctx: Ctx, body: RedactRequest, cancel: threading.Event) -> dict[str, Any]: real, info = open_input(ctx, body.path, cancel) ctx.registry.runtime("redact") out_real = paths.output_file(ctx.settings.media_root, body.outPath) if os.path.splitext(out_real)[1].lower() not in OUTPUT_EXTENSIONS: raise BadRequest("outPath must end in .mkv, .mp4 or .mov") if out_real == real: raise BadRequest("outPath must differ from path") frame_ms = info.frame_ms() plans = [build_plan(ctx, r, frame_ms) for r in body.redactions] def per_frame(_index: int, t_ms: int, frame: np.ndarray) -> np.ndarray: return apply_redactions(frame, t_ms, plans, frame_ms) count = media_io.transform_video(real, out_real, info, cancel, per_frame) return {"outPath": body.outPath, "framesProcessed": count}