5e2094f2ad
FastAPI service in infra/media-worker implementing every MediaInferenceProvider operation behind HTTP API v1, plus capabilities. Bearer auth, MEDIA_ROOT realpath enforcement, strict request schemas, cancellation on disconnect, GPU/CPU gates with 503 overflow, input size/duration limits, lazy SHA256-pinned model downloads, fail-closed redaction with atomic output, offline pytest suite with fake backends, CPU/CUDA/ROCm Dockerfile and a license review of every model. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj
272 lines
10 KiB
Python
272 lines
10 KiB
Python
"""redact: frame-accurate face and region redaction (OpenCV + ffmpeg pipes).
|
|
|
|
Geometry per instruction and frame:
|
|
- Boxes interpolate linearly between keyframes. A single keyframe is a static
|
|
region for the whole video; otherwise the region applies from the first to
|
|
the last keyframe (plus half a frame either side).
|
|
- `paddingPct` grows the box by that percentage of its width/height (half on
|
|
each side). Mask regions are dilated by the same share of their extent.
|
|
- `featherPct` fades the effect outward over that percentage of the region's
|
|
shorter side. The region itself is always fully covered.
|
|
|
|
Fails closed: an unreadable frame, a mask that cannot be read, or an encoder
|
|
error is a 500 and no file is left at outPath.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import bisect
|
|
import os
|
|
import threading
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import cv2
|
|
import numpy as np
|
|
|
|
from .. import media_io, paths
|
|
from ..errors import BadRequest
|
|
from ..schemas import RedactionInstruction, RedactRequest
|
|
from .common import Ctx, open_input
|
|
from .masks import MaskIndex
|
|
|
|
OUTPUT_EXTENSIONS = (".mkv", ".mp4", ".mov")
|
|
DEFAULT_FILL = (0, 0, 0)
|
|
|
|
|
|
def hex_to_bgr(color: str | None) -> tuple[int, int, int] | None:
|
|
if not color:
|
|
return None
|
|
value = color.lstrip("#")
|
|
r, g, b = int(value[0:2], 16), int(value[2:4], 16), int(value[4:6], 16)
|
|
return (b, g, r)
|
|
|
|
|
|
@dataclass
|
|
class Plan:
|
|
mode: str
|
|
padding: float
|
|
feather: float
|
|
color: tuple[int, int, int] | None
|
|
times: list[int]
|
|
boxes: list[tuple[float, float, float, float]]
|
|
landmark_times: list[int]
|
|
landmarks: list[list[tuple[float, float]]]
|
|
masks: MaskIndex | None
|
|
|
|
|
|
def build_plan(ctx: Ctx, instruction: RedactionInstruction, frame_ms: float) -> Plan:
|
|
keyframes = sorted(instruction.keyframes, key=lambda k: k.tMs)
|
|
landmarks = sorted(instruction.landmarks or [], key=lambda frame: frame.tMs)
|
|
masks = None
|
|
if instruction.maskDir:
|
|
masks = MaskIndex(paths.input_dir(ctx.settings.media_root, instruction.maskDir), frame_ms)
|
|
return Plan(
|
|
mode=instruction.mode,
|
|
padding=instruction.paddingPct / 100.0,
|
|
feather=instruction.featherPct / 100.0,
|
|
color=hex_to_bgr(instruction.color),
|
|
times=[k.tMs for k in keyframes],
|
|
boxes=[(k.box.x, k.box.y, k.box.w, k.box.h) for k in keyframes],
|
|
landmark_times=[frame.tMs for frame in landmarks],
|
|
landmarks=[[(p.x, p.y) for p in frame.points] for frame in landmarks],
|
|
masks=masks,
|
|
)
|
|
|
|
|
|
def interpolate(times: list[int], values: list[Any], t_ms: float, frame_ms: float, hold_single: bool) -> Any:
|
|
"""Linear interpolation of tuples/lists over keyframe times; None outside the span."""
|
|
if not times:
|
|
return None
|
|
if len(times) == 1:
|
|
if hold_single or abs(t_ms - times[0]) <= frame_ms / 2:
|
|
return values[0]
|
|
return None
|
|
if t_ms < times[0] - frame_ms / 2 or t_ms > times[-1] + frame_ms / 2:
|
|
return None
|
|
if t_ms <= times[0]:
|
|
return values[0]
|
|
if t_ms >= times[-1]:
|
|
return values[-1]
|
|
i = bisect.bisect_right(times, t_ms)
|
|
t0, t1 = times[i - 1], times[i]
|
|
a, b = values[i - 1], values[i]
|
|
if t1 == t0:
|
|
return b
|
|
f = (t_ms - t0) / (t1 - t0)
|
|
if isinstance(a, list):
|
|
if len(a) != len(b):
|
|
return a if f < 0.5 else b
|
|
return [tuple(pa[k] + (pb[k] - pa[k]) * f for k in range(2)) for pa, pb in zip(a, b, strict=True)]
|
|
return tuple(a[k] + (b[k] - a[k]) * f for k in range(len(a)))
|
|
|
|
|
|
def _pixel_box(
|
|
box: tuple[float, float, float, float], padding: float, width: int, height: int
|
|
) -> tuple[int, int, int, int]:
|
|
x, y, w, h = box
|
|
x -= w * padding / 2
|
|
y -= h * padding / 2
|
|
w *= 1 + padding
|
|
h *= 1 + padding
|
|
x0 = int(np.floor(max(0.0, x) * width))
|
|
y0 = int(np.floor(max(0.0, y) * height))
|
|
x1 = int(np.ceil(min(1.0, x + w) * width))
|
|
y1 = int(np.ceil(min(1.0, y + h) * height))
|
|
return x0, y0, max(x1, x0), max(y1, y0)
|
|
|
|
|
|
def blur_effect(roi: np.ndarray, size: int) -> np.ndarray:
|
|
"""Strong, size-scaled blur: downsample, upsample, Gaussian."""
|
|
h, w = roi.shape[:2]
|
|
factor = max(4, size // 6)
|
|
small = cv2.resize(roi, (max(1, w // factor), max(1, h // factor)), interpolation=cv2.INTER_AREA)
|
|
up = cv2.resize(small, (w, h), interpolation=cv2.INTER_LINEAR)
|
|
k = max(3, (size // 3) | 1)
|
|
return cv2.GaussianBlur(up, (k, k), 0)
|
|
|
|
|
|
def pixelate_effect(roi: np.ndarray, size: int) -> np.ndarray:
|
|
h, w = roi.shape[:2]
|
|
block = max(4, size // 10)
|
|
small = cv2.resize(roi, (max(1, -(-w // block)), max(1, -(-h // block))), interpolation=cv2.INTER_AREA)
|
|
return cv2.resize(small, (w, h), interpolation=cv2.INTER_NEAREST)
|
|
|
|
|
|
def _ring_color(frame: np.ndarray, region: np.ndarray, margin: int) -> tuple[int, int, int]:
|
|
"""Median colour just outside the region: the 'skin tone patch' cover fill."""
|
|
kernel = np.ones((2 * margin + 1, 2 * margin + 1), np.uint8)
|
|
ring = (cv2.dilate(region, kernel) > 0) & (region == 0)
|
|
if not ring.any():
|
|
return DEFAULT_FILL
|
|
b, g, r = np.median(frame[ring], axis=0)
|
|
return (int(b), int(g), int(r))
|
|
|
|
|
|
def _region_for(
|
|
plan: Plan, t_ms: int, frame_ms: float, width: int, height: int
|
|
) -> tuple[np.ndarray, int] | None:
|
|
"""Binary region mask (uint8 0/255) and its characteristic size in px."""
|
|
region = np.zeros((height, width), np.uint8)
|
|
size = 0
|
|
hold = plan.masks is None
|
|
box = interpolate(plan.times, plan.boxes, t_ms, frame_ms, hold_single=hold)
|
|
if box is not None:
|
|
x0, y0, x1, y1 = _pixel_box(box, plan.padding, width, height)
|
|
raw_w, raw_h = box[2] * width, box[3] * height
|
|
if plan.mode in ("partial_eyes", "partial_upper", "partial_lower"):
|
|
_partial_region(plan, region, box, (x0, y0, x1, y1), t_ms, frame_ms, width, height)
|
|
else:
|
|
region[y0:y1, x0:x1] = 255
|
|
size = max(size, int(min(raw_w, raw_h)))
|
|
if plan.masks is not None:
|
|
mask = plan.masks.lookup(t_ms, width, height)
|
|
if mask is not None:
|
|
ys, xs = np.nonzero(mask)
|
|
if xs.size:
|
|
extent = int(min(xs.max() - xs.min() + 1, ys.max() - ys.min() + 1))
|
|
grow = int(round(extent * plan.padding / 2))
|
|
if grow > 0:
|
|
mask = cv2.dilate(mask, np.ones((2 * grow + 1, 2 * grow + 1), np.uint8))
|
|
region |= mask
|
|
size = max(size, extent)
|
|
if not region.any():
|
|
return None
|
|
return region, max(size, 1)
|
|
|
|
|
|
def _partial_region(
|
|
plan: Plan,
|
|
region: np.ndarray,
|
|
box: tuple[float, float, float, float],
|
|
padded: tuple[int, int, int, int],
|
|
t_ms: int,
|
|
frame_ms: float,
|
|
width: int,
|
|
height: int,
|
|
) -> None:
|
|
x0, y0, x1, y1 = padded
|
|
bx, by, bw, bh = box
|
|
points = interpolate(plan.landmark_times, plan.landmarks, t_ms, frame_ms, hold_single=False)
|
|
# Five-point landmarks: right eye, left eye, nose, mouth corners.
|
|
eyes = points[:2] if points and len(points) >= 2 else None
|
|
nose_y = points[2][1] if points and len(points) >= 3 else by + bh * 0.55
|
|
if plan.mode == "partial_eyes":
|
|
if eyes:
|
|
(ex1, ey1), (ex2, ey2) = eyes
|
|
else:
|
|
ex1, ey1, ex2, ey2 = bx + bw * 0.3, by + bh * 0.4, bx + bw * 0.7, by + bh * 0.4
|
|
cx, cy = (ex1 + ex2) / 2 * width, (ey1 + ey2) / 2 * height
|
|
angle = float(np.degrees(np.arctan2((ey2 - ey1) * height, (ex2 - ex1) * width)))
|
|
band_w = (x1 - x0) * 1.0
|
|
band_h = max(bh * height * 0.32 * (1 + plan.padding), 2.0)
|
|
rect = cv2.boxPoints(((cx, cy), (band_w, band_h), angle)).astype(np.int32)
|
|
cv2.fillConvexPoly(region, rect, 255)
|
|
elif plan.mode == "partial_upper":
|
|
split = int(round(min(max(nose_y, by), by + bh) * height))
|
|
region[y0 : max(split, y0), x0:x1] = 255
|
|
else:
|
|
split = int(round(min(max(nose_y, by), by + bh) * height))
|
|
region[min(split, y1) : y1, x0:x1] = 255
|
|
|
|
|
|
def apply_plan(frame: np.ndarray, plan: Plan, t_ms: int, frame_ms: float) -> None:
|
|
height, width = frame.shape[:2]
|
|
found = _region_for(plan, t_ms, frame_ms, width, height)
|
|
if found is None:
|
|
return
|
|
region, size = found
|
|
radius = int(round(plan.feather * size))
|
|
ys, xs = np.nonzero(region)
|
|
margin = radius + 2
|
|
x0, x1 = max(0, int(xs.min()) - margin), min(width, int(xs.max()) + margin + 1)
|
|
y0, y1 = max(0, int(ys.min()) - margin), min(height, int(ys.max()) + margin + 1)
|
|
roi = frame[y0:y1, x0:x1]
|
|
roi_region = region[y0:y1, x0:x1]
|
|
mode = plan.mode
|
|
if mode in ("blur", "partial_upper", "partial_lower"):
|
|
effected = blur_effect(roi, size)
|
|
elif mode == "pixelate":
|
|
effected = pixelate_effect(roi, size)
|
|
else:
|
|
if mode == "cover" and plan.color is None:
|
|
fill = _ring_color(frame, region, max(4, size // 20))
|
|
else:
|
|
fill = plan.color or DEFAULT_FILL
|
|
effected = np.empty_like(roi)
|
|
effected[:] = fill
|
|
alpha = roi_region.astype(np.float32) / 255.0
|
|
if radius > 0:
|
|
kernel = np.ones((2 * radius + 1, 2 * radius + 1), np.uint8)
|
|
grown = cv2.dilate(roi_region, kernel).astype(np.float32) / 255.0
|
|
k = 2 * radius + 1
|
|
soft = cv2.GaussianBlur(grown, (k, k), 0)
|
|
alpha = np.maximum(alpha, soft)
|
|
alpha = alpha[..., None]
|
|
blended = roi.astype(np.float32) * (1.0 - alpha) + effected.astype(np.float32) * alpha
|
|
roi[:] = np.clip(np.rint(blended), 0, 255).astype(np.uint8)
|
|
|
|
|
|
def apply_redactions(frame: np.ndarray, t_ms: int, plans: list[Plan], frame_ms: float) -> np.ndarray:
|
|
for plan in plans:
|
|
apply_plan(frame, plan, t_ms, frame_ms)
|
|
return frame
|
|
|
|
|
|
def run(ctx: Ctx, body: RedactRequest, cancel: threading.Event) -> dict[str, Any]:
|
|
real, info = open_input(ctx, body.path, cancel)
|
|
ctx.registry.runtime("redact")
|
|
out_real = paths.output_file(ctx.settings.media_root, body.outPath)
|
|
if os.path.splitext(out_real)[1].lower() not in OUTPUT_EXTENSIONS:
|
|
raise BadRequest("outPath must end in .mkv, .mp4 or .mov")
|
|
if out_real == real:
|
|
raise BadRequest("outPath must differ from path")
|
|
frame_ms = info.frame_ms()
|
|
plans = [build_plan(ctx, r, frame_ms) for r in body.redactions]
|
|
|
|
def per_frame(_index: int, t_ms: int, frame: np.ndarray) -> np.ndarray:
|
|
return apply_redactions(frame, t_ms, plans, frame_ms)
|
|
|
|
count = media_io.transform_video(real, out_real, info, cancel, per_frame)
|
|
return {"outPath": body.outPath, "framesProcessed": count}
|