"""saliency: spectral residual (Hou & Zhang, 2007), implemented with numpy/OpenCV. The point is the saliency-weighted centroid of the top 5 % most salient pixels; strength is how much those pixels stand out against the frame mean. """ from __future__ import annotations import threading from typing import Any import cv2 import numpy as np from .. import media_io from ..schemas import TimesRequest from .common import Ctx, clamp01, open_input, unique_sorted WORK_SIZE = 64 def saliency_map(frame: np.ndarray) -> np.ndarray: gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY).astype(np.float32) small = cv2.resize(gray, (WORK_SIZE, WORK_SIZE), interpolation=cv2.INTER_AREA) spectrum = np.fft.fft2(small) log_amp = np.log(np.abs(spectrum) + 1e-8) phase = np.angle(spectrum) residual = log_amp - cv2.blur(log_amp, (3, 3)) sal = np.abs(np.fft.ifft2(np.exp(residual + 1j * phase))) ** 2 sal = cv2.GaussianBlur(sal.astype(np.float32), (0, 0), 2.5) peak = float(sal.max()) return sal / peak if peak > 0 else sal def salient_point(frame: np.ndarray) -> tuple[float, float, float]: sal = saliency_map(frame) threshold = float(np.quantile(sal, 0.95)) top = np.where(sal >= threshold, sal, 0.0) total = float(top.sum()) if total <= 0: return 0.5, 0.5, 0.0 ys, xs = np.mgrid[0 : sal.shape[0], 0 : sal.shape[1]] cx = float((xs * top).sum() / total + 0.5) / sal.shape[1] cy = float((ys * top).sum() / total + 0.5) / sal.shape[0] mean = float(sal.mean()) top_mean = float(sal[sal >= threshold].mean()) strength = (top_mean - mean) / top_mean if top_mean > 0 else 0.0 return clamp01(cx), clamp01(cy), clamp01(strength) def run(ctx: Ctx, body: TimesRequest, cancel: threading.Event) -> dict[str, Any]: real, info = open_input(ctx, body.path, cancel) ctx.registry.runtime("saliency") points = [] for t_ms, frame in media_io.frames_at(real, info, unique_sorted(body.timesMs), cancel, 512): cx, cy, strength = salient_point(frame) points.append({"tMs": t_ms, "cx": cx, "cy": cy, "strength": strength}) return {"points": points}