From 270ea8068a06fb8d81103533d1e8c9980e66f2b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 17:52:17 +0000 Subject: [PATCH] feat(media): inference emulator, sidecar client, folder source and factory Add the Media Studio adapters behind the frozen provider contracts: a deterministic offline inference emulator driven by file-name markers that renders real redactions with ffmpeg, a synthetic emulator library, the bearer-auth HTTP client for the local media sidecar with validated responses, typed errors, busy retries and timeouts, a lazily paginated folder source that never follows symlinks, and the configuration factory. A shared conformance suite runs against the emulator and, in the Docker integration job, against the sidecar. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- .../adapters/src/media-conformance.test.ts | 281 +++++ packages/adapters/src/media-emulator-clips.ts | 66 ++ .../adapters/src/media-emulator-source.ts | 208 ++++ packages/adapters/src/media-emulator.test.ts | 335 ++++++ packages/adapters/src/media-emulator.ts | 1003 +++++++++++++++++ packages/adapters/src/media-factory.test.ts | 53 + packages/adapters/src/media-factory.ts | 90 ++ .../adapters/src/media-folder-source-probe.ts | 194 ++++ .../adapters/src/media-folder-source.test.ts | 177 +++ packages/adapters/src/media-folder-source.ts | 221 ++++ packages/adapters/src/media-sidecar.test.ts | 222 ++++ packages/adapters/src/media-sidecar.ts | 410 +++++++ 12 files changed, 3260 insertions(+) create mode 100644 packages/adapters/src/media-conformance.test.ts create mode 100644 packages/adapters/src/media-emulator-clips.ts create mode 100644 packages/adapters/src/media-emulator-source.ts create mode 100644 packages/adapters/src/media-emulator.test.ts create mode 100644 packages/adapters/src/media-emulator.ts create mode 100644 packages/adapters/src/media-factory.test.ts create mode 100644 packages/adapters/src/media-factory.ts create mode 100644 packages/adapters/src/media-folder-source-probe.ts create mode 100644 packages/adapters/src/media-folder-source.test.ts create mode 100644 packages/adapters/src/media-folder-source.ts create mode 100644 packages/adapters/src/media-sidecar.test.ts create mode 100644 packages/adapters/src/media-sidecar.ts diff --git a/packages/adapters/src/media-conformance.test.ts b/packages/adapters/src/media-conformance.test.ts new file mode 100644 index 0000000..5d28ce6 --- /dev/null +++ b/packages/adapters/src/media-conformance.test.ts @@ -0,0 +1,281 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { MediaInferenceProvider } from "@rakazo/adapter-kit"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import type { MediaInferenceOperation } from "@rakazo/contracts"; +import { MediaCapabilitiesSchema, MediaInferenceOperationSchema } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { EmulatorMediaInferenceProvider } from "./media-emulator.js"; +import { extractRgbFrame, ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js"; +import { MEDIA_SIDECAR_RESPONSE_SCHEMAS, SidecarMediaInferenceProvider } from "./media-sidecar.js"; + +/* + * One contract for every MediaInferenceProvider. The emulator always runs; the + * sidecar runs only in the Docker integration job, which sets + * MEDIA_CONFORMANCE_URL, MEDIA_CONFORMANCE_TOKEN and MEDIA_CONFORMANCE_ROOT (a + * directory inside the sidecar's MEDIA_ROOT, mounted at the same path here). + */ + +const WIDTH = 320; +const HEIGHT = 180; +const hasFfmpeg = ffmpegAvailable(); + +interface Target { + name: string; + /** Directory inside the provider's root where the suite writes fixtures. */ + root: () => Promise; + provider: (root: string) => MediaInferenceProvider; +} + +const targets: Target[] = [ + { + name: "emulator", + root: () => mkdtemp(path.join(tmpdir(), "media-conformance-")), + provider: (root) => new EmulatorMediaInferenceProvider({ root }), + }, +]; + +const sidecarUrl = process.env.MEDIA_CONFORMANCE_URL; +const sidecarToken = process.env.MEDIA_CONFORMANCE_TOKEN; +const sidecarRoot = process.env.MEDIA_CONFORMANCE_ROOT; +if (sidecarUrl && sidecarToken && sidecarRoot) { + targets.push({ + name: "sidecar", + root: async () => { + await mkdir(sidecarRoot, { recursive: true }); + return mkdtemp(path.join(sidecarRoot, "conformance-")); + }, + provider: () => new SidecarMediaInferenceProvider({ baseUrl: sidecarUrl, token: sidecarToken }), + }); +} + +function requestFor(operation: MediaInferenceOperation, clip: string, root: string) { + const box = { x: 0.25, y: 0.25, w: 0.25, h: 0.25 }; + switch (operation) { + case "detectScenes": + case "audioFeatures": + return { path: clip }; + case "transcribe": + return { path: clip, language: "en" }; + case "detectFaces": + case "trackFaces": + return { path: clip, sampleFps: 2 }; + case "faceLandmarks": + case "embedFrames": + case "saliency": + return { path: clip, timesMs: [0, 500, 1000] }; + case "classifyContent": + return { path: clip, sampleFps: 2, extraTimesMs: [750] }; + case "segmentPrompted": + return { + path: clip, + prompt: { tMs: 500, box }, + propagate: true, + outDir: path.join(root, "masks"), + }; + case "inpaintVideo": + return { + path: clip, + maskDir: path.join(root, "masks"), + outPath: path.join(root, "inpaint.mp4"), + }; + case "redact": + return { + path: clip, + redactions: [ + { + keyframes: [ + { tMs: 0, box }, + { tMs: 2000, box }, + ], + mode: "blur" as const, + paddingPct: 0, + featherPct: 0, + }, + ], + outPath: path.join(root, "redacted-any.mp4"), + }; + case "synthesizeIdentity": + return { + path: clip, + track: [{ tMs: 0, box }], + seed: 7, + outPath: path.join(root, "synthetic.mp4"), + }; + } +} + +/** Mean absolute RGB difference inside and outside a pixel box (with a guard margin). */ +function regionDiff( + a: Buffer, + b: Buffer, + box: { x: number; y: number; w: number; h: number }, + margin = 4, +) { + let inside = 0; + let insideCount = 0; + let outside = 0; + let outsideCount = 0; + for (let y = 0; y < HEIGHT; y++) { + for (let x = 0; x < WIDTH; x++) { + const within = x >= box.x && x < box.x + box.w && y >= box.y && y < box.y + box.h; + const near = + x >= box.x - margin && + x < box.x + box.w + margin && + y >= box.y - margin && + y < box.y + box.h + margin; + let diff = 0; + for (let c = 0; c < 3; c++) { + const i = (y * WIDTH + x) * 3 + c; + diff += Math.abs((a[i] ?? 0) - (b[i] ?? 0)); + } + if (within) { + inside += diff / 3; + insideCount++; + } else if (!near) { + outside += diff / 3; + outsideCount++; + } + } + } + return { inside: inside / insideCount, outside: outside / outsideCount }; +} + +for (const target of targets) { + describe.skipIf(!hasFfmpeg)(`MediaInferenceProvider conformance: ${target.name}`, () => { + let root: string; + let clip: string; + let provider: MediaInferenceProvider; + + beforeAll(async () => { + root = await target.root(); + clip = await renderTestClip(path.join(root, "clip.mp4"), { + seconds: 2, + width: WIDTH, + height: HEIGHT, + }); + provider = target.provider(root); + }); + + afterAll(async () => { + if (root) await rm(root, { recursive: true, force: true }); + }); + + it("reports a capabilities document for every operation", async () => { + const capabilities = MediaCapabilitiesSchema.parse(await provider.capabilities()); + expect(Object.keys(capabilities.operations).sort()).toEqual( + [...MediaInferenceOperationSchema.options].sort(), + ); + }); + + it("returns schema-valid output or Unsupported for every operation", async () => { + const capabilities = await provider.capabilities(); + // segmentPrompted runs first so inpaintVideo finds its mask directory. + for (const operation of MediaInferenceOperationSchema.options) { + const request = requestFor(operation, clip, root); + const call = provider[operation] as (input: unknown) => Promise; + try { + const result = await call.call(provider, request); + const parsed = MEDIA_SIDECAR_RESPONSE_SCHEMAS[operation].safeParse(result); + expect(parsed.success, `${operation}: ${parsed.error?.message}`).toBe(true); + expect(capabilities.operations[operation]?.available, operation).toBe(true); + } catch (error) { + if (!(error instanceof UnsupportedMediaOperationError)) throw error; + expect(error.operation).toBe(operation); + expect(capabilities.operations[operation]?.available, operation).toBe(false); + } + } + }); + + it("rejects paths outside the media root", async () => { + const outside = await mkdtemp(path.join(tmpdir(), "media-outside-")); + try { + const stray = path.join(outside, "stray.mp4"); + await writeFile(stray, "not inside the root"); + await expect(provider.detectScenes({ path: stray })).rejects.toThrow(); + await expect( + provider.detectScenes({ + path: path.join(root, "..", path.basename(outside), "stray.mp4"), + }), + ).rejects.toThrow(); + await expect( + provider.redact({ + path: clip, + redactions: [], + outPath: path.join(outside, "escaped.mp4"), + }), + ).rejects.toThrow(); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + + it("redacts only inside the box", async () => { + const box = { x: 0.25, y: 0.25, w: 0.25, h: 0.25 }; + const pixels = { x: 80, y: 44, w: 80, h: 46 }; + for (const mode of ["solid", "blur"] as const) { + const outPath = path.join(root, `redacted-${mode}.mp4`); + const result = await provider.redact({ + path: clip, + outPath, + redactions: [ + { + keyframes: [ + { tMs: 0, box }, + { tMs: 2000, box }, + ], + mode, + paddingPct: 0, + featherPct: 0, + color: "#102030", + }, + ], + }); + expect(result.framesProcessed).toBeGreaterThan(0); + const source = await extractRgbFrame(clip, 1000); + const redacted = await extractRgbFrame(result.outPath, 1000); + expect(redacted.length).toBe(WIDTH * HEIGHT * 3); + const diff = regionDiff(source, redacted, pixels); + expect(diff.inside, mode).toBeGreaterThan(8); + expect(diff.outside, mode).toBeLessThan(1.5); + } + }); + + it("honours AbortSignal", async () => { + const aborted = new AbortController(); + aborted.abort(); + await expect( + provider.detectScenes({ path: clip, signal: aborted.signal }), + ).rejects.toMatchObject({ name: "AbortError" }); + await expect(provider.capabilities(aborted.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + + const long = await renderTestClip(path.join(root, "long.mp4"), { + seconds: 20, + width: 1280, + height: 720, + }); + const controller = new AbortController(); + const pending = provider.redact({ + path: long, + outPath: path.join(root, "aborted.mp4"), + redactions: [ + { + keyframes: [ + { tMs: 0, box: { x: 0.1, y: 0.1, w: 0.3, h: 0.3 } }, + { tMs: 20_000, box: { x: 0.5, y: 0.5, w: 0.3, h: 0.3 } }, + ], + mode: "blur", + paddingPct: 10, + featherPct: 0, + }, + ], + signal: controller.signal, + }); + setTimeout(() => controller.abort(), 150); + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + }); + }); +} diff --git a/packages/adapters/src/media-emulator-clips.ts b/packages/adapters/src/media-emulator-clips.ts new file mode 100644 index 0000000..95a3cd7 --- /dev/null +++ b/packages/adapters/src/media-emulator-clips.ts @@ -0,0 +1,66 @@ +import { spawnSync } from "node:child_process"; +import { runMediaTool } from "./media-folder-source-probe.js"; + +/* + * Synthetic test clips (ffmpeg testsrc2 + sine) for media tests across the + * codebase. Name the output with media-emulator markers to steer emulator + * results, e.g. `clip__faces-1__.mp4`. + */ + +export function ffmpegAvailable(ffmpegPath = "ffmpeg"): boolean { + return spawnSync(ffmpegPath, ["-version"], { stdio: "ignore" }).status === 0; +} + +export interface TestClipOptions { + seconds?: number; + width?: number; + height?: number; + fps?: number; + audio?: boolean; + ffmpegPath?: string; + signal?: AbortSignal; +} + +export async function renderTestClip(outPath: string, options: TestClipOptions = {}) { + const { seconds = 2, width = 320, height = 180, fps = 25, audio = true } = options; + const args = [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + `testsrc2=s=${width}x${height}:r=${fps}:d=${seconds}`, + ]; + if (audio) + args.push("-f", "lavfi", "-i", `sine=f=440:sample_rate=48000:d=${seconds}`, "-shortest"); + args.push("-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"); + if (audio) args.push("-c:a", "aac"); + args.push(outPath); + await runMediaTool(options.ffmpegPath ?? "ffmpeg", args, { signal: options.signal }); + return outPath; +} + +/** Decode one frame at `tMs` as packed RGB24 bytes. */ +export async function extractRgbFrame( + file: string, + tMs: number, + options: { ffmpegPath?: string } = {}, +): Promise { + const { stdout } = await runMediaTool(options.ffmpegPath ?? "ffmpeg", [ + "-v", + "error", + "-i", + file, + "-ss", + (tMs / 1000).toFixed(3), + "-frames:v", + "1", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "-", + ]); + return stdout; +} diff --git a/packages/adapters/src/media-emulator-source.ts b/packages/adapters/src/media-emulator-source.ts new file mode 100644 index 0000000..4d30b7e --- /dev/null +++ b/packages/adapters/src/media-emulator-source.ts @@ -0,0 +1,208 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, rename, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import type { + MediaSourceItem, + MediaSourceListRequest, + MediaSourceListResult, + MediaSourceProvider, +} from "@rakazo/adapter-kit"; +import { PathOutsideRootError, resolveInside } from "@rakazo/media/node"; +import { z } from "zod"; +import type { MediaToolPaths } from "./media-folder-source-probe.js"; +import { runMediaTool } from "./media-folder-source-probe.js"; + +/* + * Deterministic synthetic library for tests and the scripted runtime. Config + * `{ count, seed }` describes `count` assets split into three capture sessions + * on separate days in three cities (more than 2 km apart), alternating two + * visual themes (`__theme-a__` testsrc2, `__theme-b__` SMPTE bars). Every fifth + * asset is a photo. File names carry media-emulator markers: + * index % 10 == 0 __faces-1____person-a__ + * index % 10 == 1 __faces-1____person-b__ + * index == 2 __nsfw-explicit_nudity-0.9__ + * index % 4 == 3 __speech__ (videos only) + * index % 10 == 6 __cut-2000__ (videos only) + * `resolve` renders the clip on first use (4 s, 320x180, ffmpeg testsrc2 or + * smptehdbars plus a sine tone) and caches it under `fixturesDir//`. + */ + +const ConfigSchema = z.object({ + count: z.number().int().min(1).max(100_000), + seed: z.number().int().min(0).max(2_147_483_647), +}); + +const CLIP_MS = 4000; +const CLIP_SIZE = { width: 320, height: 180, fps: 25 }; +const SESSIONS = [ + { lat: 52.52, lon: 13.405 }, + { lat: 53.5511, lon: 9.9937 }, + { lat: 48.1374, lon: 11.5755 }, +] as const; + +export interface EmulatorAsset { + index: number; + fileName: string; + kind: "video" | "photo"; + theme: "a" | "b"; + session: number; + capturedAt: Date; + gpsLat: number; + gpsLon: number; +} + +/** The deterministic description of asset `index` in an emulator library. */ +export function emulatorAsset(seed: number, count: number, index: number): EmulatorAsset { + const kind = index % 5 === 4 ? "photo" : "video"; + const theme = index % 2 === 0 ? "a" : "b"; + const session = Math.min(SESSIONS.length - 1, Math.floor((index * SESSIONS.length) / count)); + const firstInSession = Math.ceil((session * count) / SESSIONS.length); + const day = Date.UTC(2026, 4, 2 + (seed % 28) + session * 2, 8 + session); + const capturedAt = new Date(day + (index - firstInSession) * 3 * 60_000); + const place = SESSIONS[session] ?? SESSIONS[0]; + const jitter = ((index * 7919 + seed) % 100) / 100_000; + const markers = [`__theme-${theme}__`]; + if (index % 10 === 0) markers.push("__faces-1____person-a__"); + if (index % 10 === 1) markers.push("__faces-1____person-b__"); + if (index === 2) markers.push("__nsfw-explicit_nudity-0.9__"); + if (kind === "video" && index % 4 === 3) markers.push("__speech__"); + if (kind === "video" && index % 10 === 6) markers.push("__cut-2000__"); + const ext = kind === "photo" ? "jpg" : "mp4"; + return { + index, + fileName: `emu-${seed}-${String(index).padStart(5, "0")}${markers.join("")}.${ext}`, + kind, + theme, + session, + capturedAt, + gpsLat: Math.round((place.lat + jitter) * 1e5) / 1e5, + gpsLon: Math.round((place.lon + jitter) * 1e5) / 1e5, + }; +} + +function iso6709(lat: number, lon: number): string { + const sign = (value: number, width: number) => + `${value < 0 ? "-" : "+"}${Math.abs(value).toFixed(4).padStart(width, "0")}`; + return `${sign(lat, 7)}${sign(lon, 8)}/`; +} + +export interface EmulatorMediaSourceOptions extends MediaToolPaths { + /** Cache for rendered clips; must lie inside the media root to be analysable. */ + fixturesDir: string; +} + +export class EmulatorMediaSourceProvider implements MediaSourceProvider { + readonly kind = "emulator" as const; + + constructor(private readonly options: EmulatorMediaSourceOptions) {} + + async list(request: MediaSourceListRequest): Promise { + request.signal?.throwIfAborted(); + const { count, seed } = ConfigSchema.parse(request.config); + const start = request.cursor ? decodeIndex(request.cursor) : 0; + const end = Math.min(count, start + Math.max(1, Math.floor(request.limit))); + const items: MediaSourceItem[] = []; + for (let index = start; index < end; index++) { + const asset = emulatorAsset(seed, count, index); + items.push({ + externalId: asset.fileName, + kind: asset.kind, + capturedAt: asset.capturedAt, + durationMs: asset.kind === "video" ? CLIP_MS : null, + width: CLIP_SIZE.width, + height: CLIP_SIZE.height, + fps: asset.kind === "video" ? CLIP_SIZE.fps : null, + gpsLat: asset.gpsLat, + gpsLon: asset.gpsLon, + deviceModel: "Emulator", + albumNames: [], + isFavorite: index % 9 === 0, + fileName: asset.fileName, + sizeBytes: null, + revision: `emulator-v1-${seed}`, + }); + } + return { + items, + nextCursor: end < count ? Buffer.from(String(end)).toString("base64url") : null, + }; + } + + async resolve(config: Record, externalId: string): Promise { + const { count, seed } = ConfigSchema.parse(config); + const match = /^emu-(\d+)-(\d{5})/.exec(externalId); + const index = match ? Number(match[2]) : -1; + if (!match || Number(match[1]) !== seed || index >= count) + throw new PathOutsideRootError(externalId); + const asset = emulatorAsset(seed, count, index); + if (asset.fileName !== externalId) throw new PathOutsideRootError(externalId); + await mkdir(path.join(this.options.fixturesDir, String(seed)), { recursive: true }); + const target = await resolveInside(this.options.fixturesDir, `${seed}/${asset.fileName}`); + if (await exists(target)) return target; + const partial = path.join( + path.dirname(target), + `.partial-${randomUUID()}.${asset.kind === "photo" ? "jpg" : "mp4"}`, + ); + try { + await runMediaTool(this.options.ffmpegPath ?? "ffmpeg", renderArgs(asset, partial)); + await rename(partial, target); + } finally { + await rm(partial, { force: true }); + } + return target; + } +} + +function renderArgs(asset: EmulatorAsset, out: string): string[] { + const { width, height, fps } = CLIP_SIZE; + const pattern = asset.theme === "a" ? "testsrc2" : "smptehdbars"; + const video = `${pattern}=s=${width}x${height}:r=${fps}:d=${CLIP_MS / 1000}`; + const metadata = [ + "-metadata", + `creation_time=${asset.capturedAt.toISOString()}`, + "-metadata", + `location=${iso6709(asset.gpsLat, asset.gpsLon)}`, + ]; + if (asset.kind === "photo") + return ["-v", "error", "-y", "-f", "lavfi", "-i", video, "-frames:v", "1", out]; + return [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + video, + "-f", + "lavfi", + "-i", + `sine=f=${220 + (asset.index % 12) * 40}:sample_rate=48000:d=${CLIP_MS / 1000}`, + "-shortest", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + ...metadata, + "-movflags", + "+faststart", + out, + ]; +} + +function decodeIndex(cursor: string): number { + const value = Number(Buffer.from(cursor, "base64url").toString("utf8")); + if (!Number.isInteger(value) || value < 0) throw new Error("invalid emulator cursor"); + return value; +} + +async function exists(file: string): Promise { + try { + return (await stat(file)).isFile(); + } catch { + return false; + } +} diff --git a/packages/adapters/src/media-emulator.test.ts b/packages/adapters/src/media-emulator.test.ts new file mode 100644 index 0000000..8343a20 --- /dev/null +++ b/packages/adapters/src/media-emulator.test.ts @@ -0,0 +1,335 @@ +import { copyFile, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import { cosineSimilarity, haversineKm } from "@rakazo/media"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + buildRedactionGraph, + EmulatorMediaInferenceProvider, + parseEmulatorMarkers, + redactionRegions, +} from "./media-emulator.js"; +import { ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js"; +import { EmulatorMediaSourceProvider, emulatorAsset } from "./media-emulator-source.js"; +import { probeMedia } from "./media-folder-source-probe.js"; + +const hasFfmpeg = ffmpegAvailable(); +let root: string; +let provider: EmulatorMediaInferenceProvider; + +beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), "media-emulator-")); + provider = new EmulatorMediaInferenceProvider({ root }); +}); + +afterAll(async () => { + await rm(root, { recursive: true, force: true }); +}); + +/** A non-media file: the emulator falls back to a 10 s clip, so no ffmpeg is needed. */ +async function fixture(name: string, content = "synthetic bytes") { + const file = path.join(root, name); + await writeFile(file, content); + return file; +} + +describe("parseEmulatorMarkers", () => { + it("reads every documented marker", () => { + expect( + parseEmulatorMarkers( + "/x/clip__faces-2____person-a____noface-gap-1000-2500____nsfw-explicit_nudity-0.9____nsfw-violence-0.4____cut-1500____cut-3000____speech____theme-b____inpaintq-0.3__.mp4", + ), + ).toEqual({ + faces: 2, + person: "a", + noFaceGaps: [{ startMs: 1000, endMs: 2500 }], + nsfw: { explicit_nudity: 0.9, violence: 0.4 }, + cutsMs: [1500, 3000], + speech: true, + theme: "b", + inpaintQuality: 0.3, + }); + expect(parseEmulatorMarkers("__nsfw-unknown-0.5____nsfw-violence-7__.mp4").nsfw).toEqual({}); + }); +}); + +describe("redaction regions", () => { + it("holds the union of neighbouring keyframes and bounds the graph", () => { + const regions = redactionRegions( + { + keyframes: [ + { tMs: 1000, box: { x: 0.5, y: 0.5, w: 0.1, h: 0.1 } }, + { tMs: 0, box: { x: 0.1, y: 0.1, w: 0.1, h: 0.1 } }, + ], + mode: "blur", + paddingPct: 0, + featherPct: 0, + }, + 1000, + 1000, + ); + expect(regions).toEqual([ + { startMs: 0, endMs: 1000, box: { x: 100, y: 100, w: 500, h: 500 } }, + { startMs: 1000, endMs: 1100, box: { x: 500, y: 500, w: 100, h: 100 } }, + ]); + const keyframes = Array.from({ length: 5000 }, (_, i) => ({ + tMs: i * 40, + box: { x: (i % 50) / 100, y: 0.2, w: 0.1, h: 0.1 }, + })); + const many = redactionRegions( + { keyframes, mode: "pixelate", paddingPct: 25, featherPct: 0 }, + 1920, + 1080, + ); + expect(many.length).toBeLessThanOrEqual(400); + expect(many[0]?.startMs).toBe(0); + expect(many.at(-1)?.endMs).toBe(4999 * 40 + 100); + const graph = buildRedactionGraph( + [{ keyframes, mode: "solid", paddingPct: 0, featherPct: 0, color: "#ff0000" }], + 1920, + 1080, + ); + expect(graph.filter).toContain("color=0xff0000@1:t=fill"); + }); +}); + +describe("EmulatorMediaInferenceProvider", () => { + it("is quiet without markers", async () => { + const file = await fixture("plain.mp4"); + const scenes = await provider.detectScenes({ path: file }); + expect(scenes.scenes).toEqual([{ startMs: 0, endMs: 10_000 }]); + expect(scenes.shots).toHaveLength(1); + expect((await provider.trackFaces({ path: file, sampleFps: 2 })).tracks).toEqual([]); + const faces = await provider.detectFaces({ path: file, sampleFps: 1 }); + expect(faces.frames).toHaveLength(10); + expect(faces.frames.every((frame) => frame.faces.length === 0)).toBe(true); + const classified = await provider.classifyContent({ path: file, sampleFps: 2 }); + expect(classified.frames).toHaveLength(20); + for (const frame of classified.frames) + expect(Object.values(frame.scores).every((score) => score === 0.01)).toBe(true); + expect(await provider.transcribe({ path: file })).toEqual({ language: null, words: [] }); + }); + + it("forces faces, gaps, cuts, classifier scores and speech from the file name", async () => { + const file = await fixture( + "rich__faces-2____noface-gap-2000-3000____cut-4000____nsfw-explicit_nudity-0.9____speech__.mp4", + ); + const tracks = await provider.trackFaces({ path: file, sampleFps: 2 }); + expect(tracks.tracks.map((track) => track.trackId)).toEqual(["track-1", "track-2"]); + const times = tracks.tracks[0]?.keyframes.map((keyframe) => keyframe.tMs) ?? []; + expect(times).toContain(1500); + expect(times).not.toContain(2000); + expect(times).not.toContain(2500); + expect(times).toContain(3000); + const gapFrame = await provider.detectFaces({ + path: file, + sampleFps: 1, + timesMs: [2500, 3500], + }); + expect(gapFrame.frames.map((frame) => frame.faces.length)).toEqual([0, 2]); + + expect((await provider.detectScenes({ path: file })).scenes).toEqual([ + { startMs: 0, endMs: 4000 }, + { startMs: 4000, endMs: 10_000 }, + ]); + const classified = await provider.classifyContent({ + path: file, + sampleFps: 0.5, + extraTimesMs: [4000, 4001], + }); + expect(classified.frames.map((frame) => frame.tMs)).toEqual([0, 2000, 4000, 4001, 6000, 8000]); + for (const frame of classified.frames) { + expect(frame.scores.explicit_nudity).toBe(0.9); + expect(frame.scores.violence).toBe(0.01); + } + const english = await provider.transcribe({ path: file }); + const german = await provider.transcribe({ path: file, language: "de-DE" }); + expect(english.language).toBe("en"); + expect(english.words.map((word) => word.text).slice(0, 2)).toEqual(["welcome", "back"]); + expect(german.language).toBe("de"); + expect(german.words[0]).toEqual({ + startMs: 400, + endMs: 720, + text: "willkommen", + confidence: 0.95, + }); + }); + + it("is deterministic per file content and keeps boxes stable", async () => { + const file = await fixture("same__faces-1__.mp4"); + const copy = path.join(root, "copy__faces-1__.mp4"); + await copyFile(file, copy); + const a = await provider.detectFaces({ path: file, sampleFps: 4 }); + const b = await provider.detectFaces({ path: file, sampleFps: 4 }); + const c = await provider.detectFaces({ path: copy, sampleFps: 4 }); + expect(a).toEqual(b); + expect(a).toEqual(c); + const boxes = a.frames.map((frame) => frame.faces[0]?.box); + expect(Math.abs((boxes.at(-1)?.x ?? 0) - (boxes[0]?.x ?? 0))).toBeLessThan(0.05); + }); + + it("embeds people and themes consistently", async () => { + const personA1 = await fixture("one__faces-1____person-a__.mp4", "first"); + const personA2 = await fixture("two__faces-1____person-a__.mp4", "second"); + const personB = await fixture("three__faces-1____person-b__.mp4", "third"); + const embed = async (file: string) => + (await provider.trackFaces({ path: file, sampleFps: 1 })).tracks[0]?.embedding ?? []; + expect(cosineSimilarity(await embed(personA1), await embed(personA2))).toBeCloseTo(1, 5); + expect(cosineSimilarity(await embed(personA1), await embed(personB))).toBeLessThan(0.8); + + const themeA1 = await fixture("x__theme-a__.mp4", "x"); + const themeA2 = await fixture("y__theme-a__.mp4", "y"); + const themeB = await fixture("z__theme-b__.mp4", "z"); + const mean = async (file: string) => + (await provider.embedFrames({ path: file, timesMs: [0, 1000] })).vectors[0] ?? []; + expect(cosineSimilarity(await mean(themeA1), await mean(themeA2))).toBeGreaterThan(0.95); + expect(cosineSimilarity(await mean(themeA1), await mean(themeB))).toBeLessThan(0.85); + }); + + it("never synthesizes identities", async () => { + const file = await fixture("synth.mp4"); + const capabilities = await provider.capabilities(); + expect(capabilities.operations.synthesizeIdentity).toMatchObject({ + available: false, + license: null, + }); + expect(capabilities.operations.inpaintVideo).toMatchObject({ + available: true, + experimental: true, + }); + await expect( + provider.synthesizeIdentity({ + path: file, + track: [], + seed: 1, + outPath: path.join(root, "s.mp4"), + }), + ).rejects.toBeInstanceOf(UnsupportedMediaOperationError); + }); + + it("rejects paths outside its root", async () => { + await expect(provider.detectScenes({ path: "/etc/hostname" })).rejects.toThrow( + "outside the allowed root", + ); + await expect(provider.detectScenes({ path: "relative.mp4" })).rejects.toThrow(); + }); + + it.skipIf(!hasFfmpeg)("writes masks and inpaints with the reported quality", async () => { + const clip = await renderTestClip(path.join(root, "mark__inpaintq-0.4__.mp4"), { + seconds: 1, + audio: false, + }); + const masks = await provider.segmentPrompted({ + path: clip, + prompt: { tMs: 0, point: { x: 0.5, y: 0.5 } }, + propagate: true, + outDir: path.join(root, "marks/m1"), + }); + expect((await readdir(masks.maskDir)).sort()).toEqual(["0.png", "500.png"]); + expect(masks.keyframes[0]?.box).toEqual({ x: 0.44, y: 0.44, w: 0.12, h: 0.12 }); + const inpainted = await provider.inpaintVideo({ + path: clip, + maskDir: masks.maskDir, + outPath: path.join(root, "marks/m1.mp4"), + }); + expect(inpainted.quality.score).toBe(0.4); + expect((await probeMedia(inpainted.outPath)).durationMs).toBeGreaterThan(900); + }); + + it.skipIf(!hasFfmpeg)("renders very large redaction graphs through a script file", async () => { + const clip = await renderTestClip(path.join(root, "busy.mp4"), { seconds: 1, audio: false }); + const instruction = (offset: number) => ({ + keyframes: Array.from({ length: 2000 }, (_, i) => ({ + tMs: i, + box: { x: offset + (i % 40) / 400, y: 0.1, w: 0.1, h: 0.1 }, + })), + mode: "blur" as const, + paddingPct: 0, + featherPct: 0, + }); + const instructions = [instruction(0), instruction(0.3), instruction(0.6)]; + expect(buildRedactionGraph(instructions, 320, 180).filter.length).toBeGreaterThan(100_000); + const result = await provider.redact({ + path: clip, + redactions: instructions, + outPath: path.join(root, "busy-redacted.mp4"), + }); + expect(result.framesProcessed).toBe(25); + }); + + it.skipIf(!hasFfmpeg)("measures loudness with ebur128", async () => { + const clip = await renderTestClip(path.join(root, "tone.mp4"), { seconds: 2 }); + const features = await provider.audioFeatures({ path: clip }); + expect(features.integratedLufs).toBeLessThan(-10); + expect(features.integratedLufs).toBeGreaterThan(-40); + expect(features.energy.length).toBeGreaterThan(0); + expect(features.beatsMs.length).toBeGreaterThan(0); + const silent = await renderTestClip(path.join(root, "mute.mp4"), { seconds: 1, audio: false }); + expect(await provider.audioFeatures({ path: silent })).toMatchObject({ + integratedLufs: null, + silences: [{ startMs: 0, endMs: 1000 }], + }); + }); +}); + +describe("EmulatorMediaSourceProvider", () => { + const config = { count: 30, seed: 4 }; + + it("describes three sessions, two themes and the marker assets", async () => { + const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") }); + const first = await source.list({ config, cursor: null, limit: 12 }); + const second = await source.list({ config, cursor: first.nextCursor, limit: 100 }); + expect(second.nextCursor).toBeNull(); + const items = [...first.items, ...second.items]; + expect(items).toHaveLength(30); + expect(new Set(items.map((item) => item.externalId)).size).toBe(30); + expect(await source.list({ config, cursor: null, limit: 12 })).toEqual(first); + + const assets = items.map((_, index) => emulatorAsset(config.seed, config.count, index)); + expect(new Set(assets.map((asset) => asset.session))).toEqual(new Set([0, 1, 2])); + expect(new Set(assets.map((asset) => asset.theme))).toEqual(new Set(["a", "b"])); + const sessionStarts = [0, 1, 2].map( + (session) => assets.find((asset) => asset.session === session)!, + ); + expect( + haversineKm( + { lat: sessionStarts[0]!.gpsLat, lon: sessionStarts[0]!.gpsLon }, + { lat: sessionStarts[1]!.gpsLat, lon: sessionStarts[1]!.gpsLon }, + ), + ).toBeGreaterThan(2); + const names = items.map((item) => item.externalId).join(" "); + expect(names).toContain("__faces-1____person-a__"); + expect(names).toContain("__faces-1____person-b__"); + expect(names).toContain("__nsfw-explicit_nudity-0.9__"); + expect(names).toContain("__speech__"); + expect(items.some((item) => item.kind === "photo")).toBe(true); + }); + + it("rejects unknown external ids", async () => { + const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") }); + for (const bad of [ + "../etc/passwd", + "emu-4-00099__theme-b__.mp4", + "emu-5-00000__theme-a__.mp4", + "emu-4-00000.mp4", + ]) + await expect(source.resolve(config, bad), bad).rejects.toThrow(); + }); + + it.skipIf(!hasFfmpeg)("renders and caches clips on demand", async () => { + const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") }); + const asset = emulatorAsset(config.seed, config.count, 0); + const file = await source.resolve(config, asset.fileName); + expect(path.basename(file)).toBe(asset.fileName); + const probed = await probeMedia(file); + expect(probed).toMatchObject({ durationMs: 4000, width: 320, height: 180, hasAudio: true }); + expect(probed.capturedAt?.toISOString()).toBe(asset.capturedAt.toISOString()); + expect(await source.resolve(config, asset.fileName)).toBe(file); + const tracks = await provider.trackFaces({ path: file, sampleFps: 1 }); + expect(tracks.tracks).toHaveLength(1); + const photo = emulatorAsset(config.seed, config.count, 4); + expect(photo.kind).toBe("photo"); + expect((await probeMedia(await source.resolve(config, photo.fileName))).width).toBe(320); + }); +}); diff --git a/packages/adapters/src/media-emulator.ts b/packages/adapters/src/media-emulator.ts new file mode 100644 index 0000000..a773840 --- /dev/null +++ b/packages/adapters/src/media-emulator.ts @@ -0,0 +1,1003 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, open, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { crc32, deflateSync } from "node:zlib"; +import type { + AudioFeaturesResult, + ClassifyContentResult, + DetectFacesResult, + DetectScenesResult, + EmbedFramesResult, + FaceLandmarksResult, + FaceObservation, + InpaintVideoResult, + MediaInferenceProvider, + MediaTimeSpan, + RedactionInstruction, + SaliencyResult, + SegmentPromptedResult, + TrackFacesResult, + TranscribeResult, + TranscriptWord, +} from "@rakazo/adapter-kit"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import type { + BoxKeyframe, + ContentCategory, + MediaCapabilities, + MediaMarkPrompt, + NormalizedBox, + NormalizedPoint, +} from "@rakazo/contracts"; +import { CONTENT_CATEGORIES, MediaInferenceOperationSchema } from "@rakazo/contracts"; +import { assertInside } from "@rakazo/media/node"; +import type { MediaToolPaths } from "./media-folder-source-probe.js"; +import { MediaToolMissingError, probeMedia, runMediaTool } from "./media-folder-source-probe.js"; + +/* + * Deterministic, offline MediaInferenceProvider. No ML: every result is derived + * from the file itself, so the same file always yields the same output. + * + * Seed: SHA-256 over the first 64 KiB of the file plus its byte size. Duration + * and frame size come from ffprobe; a file ffprobe cannot read counts as a + * 10 s, 1280x720 clip. + * + * Fixture markers in the file name force results (they combine freely, e.g. + * `clip__faces-2____speech____cut-1500__.mp4`): + * __faces-N__ N tracked faces with stable, slowly drifting + * boxes (track i keeps its box across calls). + * __person-__ identity seed for the face embeddings, so + * the same person matches across files. + * Without it, track i embeds as "track i". + * __noface-gap--__ no detection for any face inside that window + * (track keyframes are missing there). + * __nsfw--__ classifyContent returns `score` for that + * ContentCategory on every sampled frame. + * __cut-__ a shot and scene cut at that time. + * __speech__ transcribe returns a fixed word sequence + * (German when the request language starts + * with "de", English otherwise) with timings. + * __theme-__ frame embeddings share a theme vector + * (cosine >= 0.95 within a theme). + * __inpaintq-__ inpaintVideo reports this quality score. + * Without markers: zero faces, every classifier score 0.01, one shot and one + * scene, no words, embeddings from the file seed. + * + * redact and inpaintVideo write real files with ffmpeg (argv only): drawbox + * fills for solid/cover/sticker and boxblur or pixelation for blur, pixelate + * and partial modes, over keyframe boxes held between keyframes as their union + * (fail closed) and gated with enable='between(t,a,b)'. featherPct and + * landmarks are ignored; partial modes use fixed bands of the box. Output is lossless + * H.264, so pixels outside the boxes match the source. Without ffmpeg both + * throw MediaToolMissingError. synthesizeIdentity is always unsupported. + * + * audioFeatures measures loudness with ffmpeg ebur128 when available; tempo + * and beats are seeded numbers. Every path must lie inside the configured root. + */ + +const HEAD_BYTES = 64 * 1024; +const FALLBACK_DURATION_MS = 10_000; +const FALLBACK_SIZE = { width: 1280, height: 720 }; +const MAX_SAMPLES = 20_000; +/** How long the last keyframe box is held so the final frames stay covered. */ +const HOLD_AFTER_LAST_MS = 100; +const MAX_REGIONS_PER_INSTRUCTION = 400; +const INLINE_GRAPH_LIMIT = 100_000; +const EMBED_DIM = 32; +const IDENTITY_DIM = 16; +const CLASSIFIER_FLOOR = 0.01; + +const WORDS = { + en: "welcome back today we are filming at the lake and the light is perfect".split(" "), + de: "willkommen zurück heute drehen wir am see und das licht ist perfekt".split(" "), +}; + +export interface EmulatorMarkers { + faces: number; + person: string | null; + noFaceGaps: MediaTimeSpan[]; + nsfw: Partial>; + cutsMs: number[]; + speech: boolean; + theme: string | null; + inpaintQuality: number | null; +} + +/** Parse the fixture markers documented above from a file name. */ +export function parseEmulatorMarkers(fileName: string): EmulatorMarkers { + const name = path.basename(fileName); + const markers: EmulatorMarkers = { + faces: 0, + person: null, + noFaceGaps: [], + nsfw: {}, + cutsMs: [], + speech: name.includes("__speech__"), + theme: null, + inpaintQuality: null, + }; + for (const match of name.matchAll(/__faces-(\d{1,2})__/g)) markers.faces = Number(match[1]); + for (const match of name.matchAll(/__person-([a-z0-9]+)__/g)) markers.person = match[1] ?? null; + for (const match of name.matchAll(/__theme-([a-z0-9]+)__/g)) markers.theme = match[1] ?? null; + for (const match of name.matchAll(/__cut-(\d+)__/g)) markers.cutsMs.push(Number(match[1])); + for (const match of name.matchAll(/__noface-gap-(\d+)-(\d+)__/g)) { + const startMs = Number(match[1]); + const endMs = Number(match[2]); + if (endMs > startMs) markers.noFaceGaps.push({ startMs, endMs }); + } + for (const match of name.matchAll(/__nsfw-([a-z_]+)-(\d+(?:\.\d+)?)__/g)) { + const category = match[1] as ContentCategory; + const score = Number(match[2]); + if (CONTENT_CATEGORIES.includes(category) && score >= 0 && score <= 1) + markers.nsfw[category] = score; + } + for (const match of name.matchAll(/__inpaintq-(\d+(?:\.\d+)?)__/g)) { + const score = Number(match[1]); + if (score >= 0 && score <= 1) markers.inpaintQuality = score; + } + return markers; +} + +// --------------------------------------------------------------------------- +// Deterministic numbers +// --------------------------------------------------------------------------- + +/** mulberry32 seeded from SHA-256(seed + label). */ +function prng(seed: string, label: string): () => number { + let state = createHash("sha256").update(`${seed}\0${label}`).digest().readUInt32LE(0); + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function round4(value: number): number { + return Math.round(value * 10_000) / 10_000; +} + +function clampUnit(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +function unitVector(seed: string, label: string, dim: number): number[] { + const next = prng(seed, label); + const raw = Array.from({ length: dim }, () => next() * 2 - 1); + const norm = Math.hypot(...raw) || 1; + return raw.map((value) => round4(value / norm)); +} + +function sampleTimes(durationMs: number, sampleFps: number): number[] { + if (!(sampleFps > 0)) throw new RangeError("sampleFps must be positive"); + let step = Math.max(1, Math.round(1000 / sampleFps)); + while (durationMs / step > MAX_SAMPLES) step *= 2; + const times: number[] = []; + for (let t = 0; t < Math.max(durationMs, 1); t += step) times.push(t); + return times; +} + +// --------------------------------------------------------------------------- +// Provider +// --------------------------------------------------------------------------- + +interface Inspected { + absolute: string; + seed: string; + markers: EmulatorMarkers; + durationMs: number; + width: number; + height: number; + hasAudio: boolean; +} + +export interface EmulatorMediaInferenceOptions extends MediaToolPaths { + /** Media root; every input and output path must lie inside it. */ + root: string; +} + +export class EmulatorMediaInferenceProvider implements MediaInferenceProvider { + readonly id = "emulator"; + + constructor(private readonly options: EmulatorMediaInferenceOptions) {} + + async capabilities(signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const operations = {} as MediaCapabilities["operations"]; + for (const operation of MediaInferenceOperationSchema.options) { + operations[operation] = + operation === "synthesizeIdentity" + ? { + available: false, + experimental: true, + model: null, + license: null, + commercialUse: null, + } + : { + available: true, + experimental: operation === "inpaintVideo", + model: "emulator", + license: "Apache-2.0", + commercialUse: true, + }; + } + return { configured: true, operations, accelerator: "none", hardwareEncoders: [] }; + } + + async detectScenes(request: { path: string; signal?: AbortSignal }): Promise { + const file = await this.inspect(request.path, request.signal); + const cuts = [...new Set(file.markers.cutsMs)] + .filter((cut) => cut > 0 && cut < file.durationMs) + .sort((a, b) => a - b); + const bounds = [0, ...cuts, file.durationMs]; + const spans = bounds.slice(1).map((endMs, index) => ({ startMs: bounds[index] ?? 0, endMs })); + return { shots: spans.map((span) => ({ ...span, score: 1 })), scenes: spans }; + } + + async transcribe(request: { + path: string; + language?: string; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + if (!file.markers.speech) return { language: null, words: [] }; + const language = request.language?.toLowerCase().startsWith("de") ? "de" : "en"; + const words: TranscriptWord[] = []; + let cursor = 400; + for (const text of WORDS[language]) { + const endMs = cursor + 320; + if (endMs > file.durationMs) break; + words.push({ startMs: cursor, endMs, text, confidence: 0.95 }); + cursor = endMs + 80; + } + return { language, words }; + } + + async detectFaces(request: { + path: string; + sampleFps: number; + timesMs?: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const times = request.timesMs ?? sampleTimes(file.durationMs, request.sampleFps); + return { + frames: times.map((tMs) => ({ + tMs, + faces: this.facesAt(file, tMs).map(({ box, confidence }) => ({ + box, + confidence, + landmarks: landmarksFor(box), + })), + })), + }; + } + + async trackFaces(request: { + path: string; + sampleFps: number; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const times = sampleTimes(file.durationMs, request.sampleFps); + const tracks = Array.from({ length: file.markers.faces }, (_, index) => ({ + trackId: `track-${index + 1}`, + keyframes: times + .filter((tMs) => !inGap(file, tMs)) + .map((tMs) => ({ tMs, box: faceBox(file, index, tMs), confidence: 0.93 })), + embedding: unitVector( + "identity", + file.markers.person ? `${file.markers.person}:${index}` : `track:${index}`, + IDENTITY_DIM, + ), + })); + return { sampleFps: request.sampleFps, tracks }; + } + + async faceLandmarks(request: { + path: string; + timesMs: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + return { + frames: request.timesMs.map((tMs) => ({ + tMs, + faces: this.facesAt(file, tMs).map(({ box }) => ({ box, landmarks: landmarksFor(box) })), + })), + }; + } + + async embedFrames(request: { + path: string; + timesMs: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const base = unitVector( + "embed", + file.markers.theme ? `theme:${file.markers.theme}` : `asset:${file.seed}`, + EMBED_DIM, + ); + const vectors = request.timesMs.map((tMs) => { + const noise = prng(file.seed, `frame:${tMs}`); + const raw = base.map((value) => value + (noise() * 2 - 1) * 0.04); + const norm = Math.hypot(...raw) || 1; + return raw.map((value) => round4(value / norm)); + }); + return { model: "emulator-embed-v1", dim: EMBED_DIM, vectors }; + } + + async classifyContent(request: { + path: string; + sampleFps: number; + extraTimesMs?: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const times = [ + ...new Set([ + ...sampleTimes(file.durationMs, request.sampleFps), + ...(request.extraTimesMs ?? []), + ]), + ].sort((a, b) => a - b); + const scores = Object.fromEntries( + CONTENT_CATEGORIES.map((category) => [ + category, + file.markers.nsfw[category] ?? CLASSIFIER_FLOOR, + ]), + ) as Record; + return { + model: "emulator-classifier-v1", + frames: times.map((tMs) => ({ tMs, scores: { ...scores } })), + }; + } + + async segmentPrompted(request: { + path: string; + prompt: MediaMarkPrompt; + propagate: boolean; + outDir: string; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const outDir = await assertInside(this.options.root, request.outDir); + const box = promptBox(request.prompt); + const times = request.propagate + ? [...new Set([...sampleTimes(file.durationMs, 2), request.prompt.tMs])].sort((a, b) => a - b) + : [request.prompt.tMs]; + await mkdir(outDir, { recursive: true }); + const scale = Math.min(1, 640 / file.width); + const width = Math.max(2, Math.round(file.width * scale)); + const height = Math.max(2, Math.round(file.height * scale)); + const png = encodeMaskPng(width, height, box); + for (const tMs of times) { + request.signal?.throwIfAborted(); + await writeFile(path.join(outDir, `${tMs}.png`), png); + } + return { + maskDir: outDir, + keyframes: times.map((tMs) => ({ tMs, box, confidence: 0.9 })), + score: 0.9, + }; + } + + async inpaintVideo(request: { + path: string; + maskDir: string; + outPath: string; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const maskDir = await assertInside(this.options.root, request.maskDir); + const keyframes = await this.boxesFromMaskDir(maskDir, request.signal); + const outPath = await this.renderRedactions( + file, + [{ keyframes, mode: "blur", paddingPct: 0, featherPct: 0 }], + request.outPath, + request.signal, + ); + const score = file.markers.inpaintQuality ?? 0.82; + return { + outPath, + quality: { + temporalStability: file.markers.inpaintQuality ?? 0.85, + edgeQuality: file.markers.inpaintQuality ?? 0.8, + score, + }, + }; + } + + async redact(request: { + path: string; + redactions: RedactionInstruction[]; + outPath: string; + signal?: AbortSignal; + }): Promise<{ outPath: string; framesProcessed: number }> { + const file = await this.inspect(request.path, request.signal); + const instructions: RedactionInstruction[] = []; + for (const instruction of request.redactions) { + if (!instruction.maskDir) { + instructions.push(instruction); + continue; + } + const maskDir = await assertInside(this.options.root, instruction.maskDir); + const fromMasks = await this.boxesFromMaskDir(maskDir, request.signal); + instructions.push({ ...instruction, keyframes: [...instruction.keyframes, ...fromMasks] }); + } + const outPath = await this.renderRedactions( + file, + instructions, + request.outPath, + request.signal, + ); + const { stdout } = await runMediaTool( + this.ffprobe, + [ + "-v", + "error", + "-select_streams", + "v:0", + "-count_packets", + "-show_entries", + "stream=nb_read_packets", + "-of", + "csv=p=0", + "--", + outPath, + ], + { signal: request.signal }, + ); + const framesProcessed = Number.parseInt(stdout.toString("utf8").trim(), 10); + if (!Number.isFinite(framesProcessed) || framesProcessed <= 0) + throw new Error("redact produced no video frames"); + return { outPath, framesProcessed }; + } + + async synthesizeIdentity(request: { + path: string; + track: BoxKeyframe[]; + seed: number; + outPath: string; + signal?: AbortSignal; + }): Promise<{ outPath: string }> { + request.signal?.throwIfAborted(); + throw new UnsupportedMediaOperationError( + "synthesizeIdentity", + "no commercially licensed identity synthesis model is installed", + ); + } + + async audioFeatures(request: { + path: string; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + const next = prng(file.seed, "audio"); + const tempoBpm = 90 + Math.floor(next() * 50); + const beatsMs: number[] = []; + const beatStep = 60_000 / tempoBpm; + for (let t = Math.round(next() * beatStep); t < file.durationMs; t += beatStep) + beatsMs.push(Math.round(t)); + const seeded = { + tempoBpm, + beatsMs, + energy: sampleTimes(file.durationMs, 2).map((tMs) => ({ + tMs, + value: round4(0.3 + next() * 0.5), + })), + silences: [] as MediaTimeSpan[], + }; + if (!file.hasAudio) + return { + integratedLufs: null, + truePeakDb: null, + tempoBpm: null, + beatsMs: [], + energy: [], + silences: [{ startMs: 0, endMs: file.durationMs }], + }; + let stderr: string; + try { + ({ stderr } = await runMediaTool( + this.ffmpeg, + [ + "-nostats", + "-hide_banner", + "-i", + file.absolute, + "-map", + "0:a:0", + "-af", + "ebur128=peak=true", + "-f", + "null", + "-", + ], + { signal: request.signal }, + )); + } catch (error) { + if (!(error instanceof MediaToolMissingError)) throw error; + return { + integratedLufs: round1(-24 + next() * 12), + truePeakDb: round1(-6 + next() * 5), + ...seeded, + }; + } + return { ...parseEbur128(stderr), tempoBpm, beatsMs }; + } + + async saliency(request: { + path: string; + timesMs: number[]; + signal?: AbortSignal; + }): Promise { + const file = await this.inspect(request.path, request.signal); + return { + points: request.timesMs.map((tMs) => { + const face = this.facesAt(file, tMs)[0]; + if (face) + return { + tMs, + cx: round4(face.box.x + face.box.w / 2), + cy: round4(face.box.y + face.box.h / 2), + strength: 0.9, + }; + const next = prng(file.seed, `saliency:${tMs}`); + return { + tMs, + cx: round4(0.4 + next() * 0.2), + cy: round4(0.4 + next() * 0.2), + strength: 0.5, + }; + }), + }; + } + + // ------------------------------------------------------------------------- + + private get ffmpeg(): string { + return this.options.ffmpegPath ?? "ffmpeg"; + } + + private get ffprobe(): string { + return this.options.ffprobePath ?? "ffprobe"; + } + + private async inspect(requested: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const absolute = await assertInside(this.options.root, requested); + const stats = await stat(absolute); + if (!stats.isFile()) throw new Error("media path is not a file"); + const handle = await open(absolute, "r"); + let head: Buffer; + try { + const buffer = Buffer.alloc(Math.min(HEAD_BYTES, stats.size)); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + head = buffer.subarray(0, bytesRead); + } finally { + await handle.close(); + } + const seed = createHash("sha256").update(head).update(String(stats.size)).digest("hex"); + let probed: Awaited> | null = null; + try { + probed = await probeMedia(absolute, { ffprobePath: this.ffprobe, signal }); + } catch { + signal?.throwIfAborted(); + } + return { + absolute, + seed, + markers: parseEmulatorMarkers(absolute), + durationMs: probed?.durationMs ?? FALLBACK_DURATION_MS, + width: probed?.width ?? FALLBACK_SIZE.width, + height: probed?.height ?? FALLBACK_SIZE.height, + hasAudio: probed?.hasAudio ?? false, + }; + } + + private facesAt(file: Inspected, tMs: number): FaceObservation[] { + if (inGap(file, tMs)) return []; + return Array.from({ length: file.markers.faces }, (_, index) => ({ + box: faceBox(file, index, tMs), + confidence: 0.93, + })); + } + + /** Bounding box per `.png` mask, measured at 128x128 and rounded outward. */ + private async boxesFromMaskDir(maskDir: string, signal?: AbortSignal): Promise { + const names = (await readdir(maskDir)).filter((name) => /^\d+\.png$/.test(name)); + const keyframes: BoxKeyframe[] = []; + for (const name of names.sort((a, b) => Number.parseInt(a, 10) - Number.parseInt(b, 10))) { + const { stdout } = await runMediaTool( + this.ffmpeg, + [ + "-v", + "error", + "-i", + path.join(maskDir, name), + "-vf", + "scale=128:128:flags=area", + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "-", + ], + { signal }, + ); + const box = maskBounds(stdout, 128, 128); + if (box) keyframes.push({ tMs: Number.parseInt(name, 10), box }); + } + return keyframes; + } + + private async renderRedactions( + file: Inspected, + instructions: RedactionInstruction[], + requestedOut: string, + signal?: AbortSignal, + ): Promise { + const outPath = await assertInside(this.options.root, requestedOut); + if (path.resolve(outPath) === path.resolve(file.absolute)) + throw new Error("redact output must differ from the input"); + await mkdir(path.dirname(outPath), { recursive: true }); + const graph = buildRedactionGraph(instructions, file.width, file.height); + // Linux caps one argv string at 128 KiB; larger graphs go through a script file. + const scratch = + graph.filter.length > INLINE_GRAPH_LIMIT + ? await mkdtemp(path.join(tmpdir(), "media-emulator-")) + : null; + try { + let filterArgs = ["-filter_complex", graph.filter]; + if (scratch) { + const script = path.join(scratch, "graph.txt"); + await writeFile(script, graph.filter); + filterArgs = ["-filter_complex_script", script]; + } + await runMediaTool( + this.ffmpeg, + [ + "-v", + "error", + "-y", + "-i", + file.absolute, + ...filterArgs, + "-map", + `[${graph.output}]`, + "-map", + "0:a?", + "-c:v", + "libx264", + "-qp", + "0", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-b:a", + "192k", + "-map_metadata", + "-1", + outPath, + ], + { signal }, + ); + } catch (error) { + await rm(outPath, { force: true }); + throw error; + } finally { + if (scratch) await rm(scratch, { recursive: true, force: true }); + } + return outPath; + } +} + +function round1(value: number): number { + return Math.round(value * 10) / 10; +} + +function inGap(file: Inspected, tMs: number): boolean { + return file.markers.noFaceGaps.some((gap) => tMs >= gap.startMs && tMs < gap.endMs); +} + +function faceBox(file: Inspected, index: number, tMs: number): NormalizedBox { + const next = prng(file.seed, `face:${index}`); + const column = index % 3; + const row = Math.floor(index / 3) % 2; + const drift = 0.04 * Math.min(1, tMs / Math.max(1, file.durationMs)); + return { + x: round4(0.06 + column * 0.3 + next() * 0.06 + drift), + y: round4(0.12 + row * 0.4 + next() * 0.1), + w: 0.16, + h: 0.22, + }; +} + +function landmarksFor(box: NormalizedBox): NormalizedPoint[] { + const at = (fx: number, fy: number) => ({ + x: round4(box.x + box.w * fx), + y: round4(box.y + box.h * fy), + }); + return [at(0.3, 0.38), at(0.7, 0.38), at(0.5, 0.58), at(0.35, 0.78), at(0.65, 0.78)]; +} + +function promptBox(prompt: MediaMarkPrompt): NormalizedBox { + if (prompt.box) return prompt.box; + if (!prompt.point) throw new Error("segment prompt needs a box or a point"); + const size = 0.12; + const x = clampUnit(Math.min(prompt.point.x - size / 2, 1 - size)); + const y = clampUnit(Math.min(prompt.point.y - size / 2, 1 - size)); + return { x: round4(x), y: round4(y), w: size, h: size }; +} + +function maskBounds(gray: Buffer, width: number, height: number): NormalizedBox | null { + let minX = width; + let minY = height; + let maxX = -1; + let maxY = -1; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + if ((gray[y * width + x] ?? 0) === 0) continue; + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + if (maxX < 0) return null; + return { + x: minX / width, + y: minY / height, + w: (maxX + 1 - minX) / width, + h: (maxY + 1 - minY) / height, + }; +} + +/** Grayscale 8-bit PNG: white inside the box, black elsewhere. */ +export function encodeMaskPng(width: number, height: number, box: NormalizedBox): Buffer { + const x0 = Math.floor(box.x * width); + const y0 = Math.floor(box.y * height); + const x1 = Math.ceil((box.x + box.w) * width); + const y1 = Math.ceil((box.y + box.h) * height); + const raw = Buffer.alloc((width + 1) * height); + for (let y = 0; y < height; y++) { + const row = y * (width + 1); + if (y < y0 || y >= y1) continue; + raw.fill(255, row + 1 + Math.max(0, x0), row + 1 + Math.min(width, x1)); + } + const chunk = (type: string, data: Buffer) => { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type, "ascii"), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(body) >>> 0); + return Buffer.concat([length, body, crc]); + }; + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; // bit depth + header[9] = 0; // grayscale + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk("IHDR", header), + chunk("IDAT", deflateSync(raw)), + chunk("IEND", Buffer.alloc(0)), + ]); +} + +// --------------------------------------------------------------------------- +// Redaction filtergraph +// --------------------------------------------------------------------------- + +interface PixelBox { + x: number; + y: number; + w: number; + h: number; +} + +interface Region { + startMs: number; + endMs: number; + box: PixelBox; +} + +function unionBox(a: NormalizedBox, b: NormalizedBox): NormalizedBox { + const x = Math.min(a.x, b.x); + const y = Math.min(a.y, b.y); + return { + x, + y, + w: Math.max(a.x + a.w, b.x + b.w) - x, + h: Math.max(a.y + a.h, b.y + b.h) - y, + }; +} + +function padBox(box: NormalizedBox, paddingPct: number): NormalizedBox { + const padX = (box.w * Math.max(0, paddingPct)) / 200; + const padY = (box.h * Math.max(0, paddingPct)) / 200; + const x = clampUnit(box.x - padX); + const y = clampUnit(box.y - padY); + return { + x, + y, + w: clampUnit(box.x + box.w + padX) - x, + h: clampUnit(box.y + box.h + padY) - y, + }; +} + +/** Sub-box a partial mode covers, relative to the face box. */ +function modeBox(box: NormalizedBox, mode: RedactionInstruction["mode"]): NormalizedBox { + const band = (top: number, bottom: number) => ({ + x: box.x, + y: box.y + box.h * top, + w: box.w, + h: box.h * (bottom - top), + }); + if (mode === "partial_eyes") return band(0.22, 0.55); + if (mode === "partial_upper") return band(0, 0.55); + if (mode === "partial_lower") return band(0.5, 1); + return box; +} + +/** Normalized box → even pixel box (outward), at least 8x8, inside the frame. */ +function toPixels(box: NormalizedBox, width: number, height: number): PixelBox { + const even = (value: number) => value - (value % 2); + let x = even(Math.max(0, Math.floor(box.x * width))); + let y = even(Math.max(0, Math.floor(box.y * height))); + const right = Math.min(width, Math.ceil((box.x + box.w) * width)); + const bottom = Math.min(height, Math.ceil((box.y + box.h) * height)); + let w = Math.max(8, right - x + ((right - x) % 2)); + let h = Math.max(8, bottom - y + ((bottom - y) % 2)); + w = Math.min(w, even(width)); + h = Math.min(h, even(height)); + x = Math.min(x, even(width - w)); + y = Math.min(y, even(height - h)); + return { x, y, w, h }; +} + +/** Consecutive keyframes become one region holding the union of both boxes. */ +export function redactionRegions( + instruction: RedactionInstruction, + width: number, + height: number, +): Region[] { + const keyframes = [...instruction.keyframes] + .sort((a, b) => a.tMs - b.tMs) + .map((keyframe) => ({ + tMs: keyframe.tMs, + box: modeBox(padBox(keyframe.box, instruction.paddingPct), instruction.mode), + })); + const spans: Array<{ startMs: number; endMs: number; box: NormalizedBox }> = []; + keyframes.forEach((keyframe, index) => { + const following = keyframes[index + 1]; + spans.push( + following + ? { + startMs: keyframe.tMs, + endMs: following.tMs, + box: unionBox(keyframe.box, following.box), + } + : { startMs: keyframe.tMs, endMs: keyframe.tMs + HOLD_AFTER_LAST_MS, box: keyframe.box }, + ); + }); + // Bound graph size: merge neighbours (union, fail closed) until few enough remain. + let merged = spans; + while (merged.length > MAX_REGIONS_PER_INSTRUCTION) { + const next: typeof spans = []; + for (let index = 0; index < merged.length; index += 2) { + const a = merged[index]!; + const b = merged[index + 1]; + next.push(b ? { startMs: a.startMs, endMs: b.endMs, box: unionBox(a.box, b.box) } : a); + } + merged = next; + } + const regions: Region[] = []; + for (const span of merged) { + const box = toPixels(span.box, width, height); + const previous = regions.at(-1); + if ( + previous && + previous.endMs >= span.startMs && + previous.box.x === box.x && + previous.box.y === box.y && + previous.box.w === box.w && + previous.box.h === box.h + ) { + previous.endMs = Math.max(previous.endMs, span.endMs); + } else { + regions.push({ startMs: span.startMs, endMs: span.endMs, box }); + } + } + return regions; +} + +function drawColor(color: string | undefined): string { + return color && /^#[0-9a-fA-F]{6}$/.test(color) ? `0x${color.slice(1)}@1` : "black@1"; +} + +export function buildRedactionGraph( + instructions: RedactionInstruction[], + width: number, + height: number, +): { filter: string; output: string } { + const lines = ["[0:v:0]format=yuv420p[v0]"]; + let current = "v0"; + let counter = 0; + const label = () => `v${++counter}`; + for (const instruction of instructions) { + for (const region of redactionRegions(instruction, width, height)) { + const { x, y, w, h } = region.box; + const enable = `enable='between(t,${(region.startMs / 1000).toFixed(3)},${(region.endMs / 1000).toFixed(3)})'`; + const out = label(); + if ( + instruction.mode === "solid" || + instruction.mode === "cover" || + instruction.mode === "sticker" + ) { + lines.push( + `[${current}]drawbox=x=${x}:y=${y}:w=${w}:h=${h}:color=${drawColor(instruction.color)}:t=fill:${enable}[${out}]`, + ); + } else { + const base = label(); + const patch = label(); + const effected = label(); + let effect: string; + if (instruction.mode === "pixelate") { + const block = Math.max(4, Math.round(Math.min(w, h) / 8)); + effect = `scale=${Math.max(1, Math.ceil(w / block))}:${Math.max(1, Math.ceil(h / block))}:flags=area,scale=${w}:${h}:flags=neighbor`; + } else { + const radius = Math.max(1, Math.min(24, Math.floor(Math.min(w, h) / 4))); + effect = `boxblur=luma_radius=${radius}:luma_power=3:chroma_radius=${Math.max(1, Math.floor(radius / 2))}:chroma_power=3`; + } + lines.push(`[${current}]split=2[${base}][${patch}]`); + lines.push(`[${patch}]crop=${w}:${h}:${x}:${y},${effect}[${effected}]`); + lines.push(`[${base}][${effected}]overlay=${x}:${y}:${enable}[${out}]`); + } + current = out; + } + } + return { filter: lines.join(";\n"), output: current }; +} + +function parseEbur128( + stderr: string, +): Pick { + const summary = stderr.slice(stderr.lastIndexOf("Summary:")); + const integrated = /I:\s+(-?\d+(?:\.\d+)?) LUFS/.exec(summary); + const peak = /Peak:\s+(-?\d+(?:\.\d+)?) dBFS/.exec(summary); + const energy: Array<{ tMs: number; value: number }> = []; + const silences: MediaTimeSpan[] = []; + let silentSince: number | null = null; + let lastMs = 0; + for (const match of stderr.matchAll( + /t:\s*(\d+(?:\.\d+)?)\s+TARGET:\S+ LUFS\s+M:\s*(-?\d+(?:\.\d+)?)/g, + )) { + const tMs = Math.round(Number(match[1]) * 1000); + const momentary = Number(match[2]); + lastMs = tMs; + if (tMs % 500 < 100) energy.push({ tMs, value: round4(clampUnit((momentary + 70) / 70)) }); + if (momentary < -50) silentSince ??= tMs; + else if (silentSince !== null) { + if (tMs - silentSince >= 500) silences.push({ startMs: silentSince, endMs: tMs }); + silentSince = null; + } + } + if (silentSince !== null && lastMs - silentSince >= 500) + silences.push({ startMs: silentSince, endMs: lastMs }); + const integratedLufs = integrated ? Number(integrated[1]) : null; + return { + integratedLufs: integratedLufs !== null && integratedLufs > -70 ? integratedLufs : null, + truePeakDb: peak ? Number(peak[1]) : null, + energy, + silences, + }; +} diff --git a/packages/adapters/src/media-factory.test.ts b/packages/adapters/src/media-factory.test.ts new file mode 100644 index 0000000..f27c359 --- /dev/null +++ b/packages/adapters/src/media-factory.test.ts @@ -0,0 +1,53 @@ +import { homedir } from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + createMediaInferenceProvider, + createMediaSourceProvider, + mediaConfigFromEnv, + resolveMediaInferenceKind, +} from "./media-factory.js"; + +const worker = { + MEDIA_WORKER_URL: "http://127.0.0.1:8790", + MEDIA_WORKER_TOKEN: "placeholder-token-placeholder-token-00", +}; + +describe("media factory", () => { + it("reads one generic worker connection and derives paths", () => { + expect(mediaConfigFromEnv({ DATA_DIR: "/srv/evoo", ...worker })).toEqual({ + enabled: true, + workerUrl: worker.MEDIA_WORKER_URL, + workerToken: worker.MEDIA_WORKER_TOKEN, + profilesDir: path.join(homedir(), ".evoo/media-profiles"), + mediaRoot: "/srv/evoo/media", + }); + expect( + mediaConfigFromEnv({ DATA_DIR: "/srv/evoo", MEDIA_PROFILES_DIR: "/etc/evoo/profiles" }), + ).toMatchObject({ enabled: false, workerUrl: null, profilesDir: "/etc/evoo/profiles" }); + }); + + it("selects the sidecar only with URL and token", () => { + expect(resolveMediaInferenceKind(worker)).toBe("sidecar"); + expect(createMediaInferenceProvider(worker)?.id).toBe("sidecar"); + expect(resolveMediaInferenceKind({ MEDIA_WORKER_URL: worker.MEDIA_WORKER_URL })).toBeNull(); + expect(createMediaInferenceProvider({})).toBeNull(); + }); + + it("pins the emulator under the scripted runtime", () => { + expect(resolveMediaInferenceKind({ ...worker, AGENT_RUNTIME: "scripted" })).toBe("emulator"); + expect(createMediaInferenceProvider({ AGENT_RUNTIME: "scripted" })?.id).toBe("emulator"); + expect(mediaConfigFromEnv({ AGENT_RUNTIME: "scripted" }).enabled).toBe(true); + // pin-test-env sets AGENT_RUNTIME=scripted for every unit test. + expect(createMediaInferenceProvider()?.id).toBe("emulator"); + }); + + it("creates server-side sources only", () => { + expect(createMediaSourceProvider("folder", {}).kind).toBe("folder"); + expect(createMediaSourceProvider("emulator", { AGENT_RUNTIME: "scripted" }).kind).toBe( + "emulator", + ); + expect(() => createMediaSourceProvider("emulator", {})).toThrow(); + expect(() => createMediaSourceProvider("apple_photos", {})).toThrow(); + }); +}); diff --git a/packages/adapters/src/media-factory.ts b/packages/adapters/src/media-factory.ts new file mode 100644 index 0000000..e6453f1 --- /dev/null +++ b/packages/adapters/src/media-factory.ts @@ -0,0 +1,90 @@ +import { homedir } from "node:os"; +import path from "node:path"; +import type { MediaInferenceProvider, MediaSourceProvider } from "@rakazo/adapter-kit"; +import { expandHome } from "@rakazo/media/node"; +import { EmulatorMediaInferenceProvider } from "./media-emulator.js"; +import { EmulatorMediaSourceProvider } from "./media-emulator-source.js"; +import { FolderMediaSourceProvider } from "./media-folder-source.js"; +import { SidecarMediaInferenceProvider } from "./media-sidecar.js"; + +export interface MediaConfig { + /** Media Studio runs: a sidecar is configured or the scripted runtime uses the emulator. */ + enabled: boolean; + workerUrl: string | null; + workerToken: string | null; + profilesDir: string; + /** `${DATA_DIR}/media`, absolute. Every derived file and sidecar path lives here. */ + mediaRoot: string; +} + +export type MediaInferenceKind = "sidecar" | "emulator"; + +/** Emulators replace the sidecar whenever the scripted runtime is pinned (tests, e2e). */ +export function mediaEmulatorEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return env.AGENT_RUNTIME === "scripted"; +} + +export function mediaConfigFromEnv(env: NodeJS.ProcessEnv = process.env): MediaConfig { + const workerUrl = env.MEDIA_WORKER_URL?.trim() || null; + const workerToken = env.MEDIA_WORKER_TOKEN?.trim() || null; + return { + enabled: mediaEmulatorEnabled(env) || Boolean(workerUrl && workerToken), + workerUrl, + workerToken, + profilesDir: path.resolve( + expandHome(env.MEDIA_PROFILES_DIR?.trim() || "~/.evoo/media-profiles", homedir()), + ), + mediaRoot: path.resolve(env.DATA_DIR ?? "./data", "media"), + }; +} + +export function resolveMediaInferenceKind( + env: NodeJS.ProcessEnv = process.env, +): MediaInferenceKind | null { + if (mediaEmulatorEnabled(env)) return "emulator"; + const config = mediaConfigFromEnv(env); + return config.workerUrl && config.workerToken ? "sidecar" : null; +} + +/** The configured inference provider, or null when Media Studio is not set up. */ +export function createMediaInferenceProvider( + env: NodeJS.ProcessEnv = process.env, +): MediaInferenceProvider | null { + const config = mediaConfigFromEnv(env); + switch (resolveMediaInferenceKind(env)) { + case "emulator": + return new EmulatorMediaInferenceProvider({ root: config.mediaRoot }); + case "sidecar": + return new SidecarMediaInferenceProvider({ + baseUrl: config.workerUrl as string, + token: config.workerToken as string, + }); + default: + return null; + } +} + +/** Emulator clips are cached inside the media root so inference can read them. */ +export function emulatorFixturesDir(env: NodeJS.ProcessEnv = process.env): string { + return path.join(mediaConfigFromEnv(env).mediaRoot, "emulator-library"); +} + +/** + * Server-side source adapter for a persisted MediaSource kind. Client-pushed + * kinds (apple_photos, ios_library) have no server adapter. + */ +export function createMediaSourceProvider( + kind: string, + env: NodeJS.ProcessEnv = process.env, +): MediaSourceProvider { + switch (kind) { + case "folder": + return new FolderMediaSourceProvider(); + case "emulator": + if (!mediaEmulatorEnabled(env)) break; + return new EmulatorMediaSourceProvider({ fixturesDir: emulatorFixturesDir(env) }); + default: + break; + } + throw new Error(`No server-side media source for "${kind}". Use folder.`); +} diff --git a/packages/adapters/src/media-folder-source-probe.ts b/packages/adapters/src/media-folder-source-probe.ts new file mode 100644 index 0000000..c53a787 --- /dev/null +++ b/packages/adapters/src/media-folder-source-probe.ts @@ -0,0 +1,194 @@ +import { execFile } from "node:child_process"; + +/* + * Local ffmpeg/ffprobe helpers shared by the folder source and the media + * emulator. Every call is an argv array without a shell; nothing here builds a + * command string. + */ + +export class MediaToolMissingError extends Error { + constructor(readonly tool: string) { + super(`${tool} is not installed or not on PATH; install ffmpeg to use local media processing`); + this.name = "MediaToolMissingError"; + } +} + +export interface MediaToolPaths { + ffmpegPath?: string; + ffprobePath?: string; +} + +/** Run a media tool and collect stdout (binary) and stderr. Rejects on non-zero exit. */ +export function runMediaTool( + bin: string, + args: string[], + options: { signal?: AbortSignal; maxBuffer?: number } = {}, +): Promise<{ stdout: Buffer; stderr: string }> { + options.signal?.throwIfAborted(); + return new Promise((resolve, reject) => { + execFile( + bin, + args, + { + encoding: "buffer", + maxBuffer: options.maxBuffer ?? 64 * 1024 * 1024, + signal: options.signal, + shell: false, + windowsHide: true, + }, + (error, stdout, stderr) => { + if (options.signal?.aborted) { + reject(options.signal.reason); + return; + } + if (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + reject(new MediaToolMissingError(bin)); + return; + } + const tail = stderr.toString("utf8").trim().split("\n").slice(-3).join(" "); + reject(new Error(`${bin} failed: ${tail || error.message}`)); + return; + } + resolve({ stdout, stderr: stderr.toString("utf8") }); + }, + ); + }); +} + +export interface ProbedMedia { + durationMs: number | null; + width: number | null; + height: number | null; + fps: number | null; + codec: string | null; + rotation: number; + hasAudio: boolean; + hasVideo: boolean; + capturedAt: Date | null; + gpsLat: number | null; + gpsLon: number | null; + deviceModel: string | null; +} + +interface FfprobeStream { + codec_type?: string; + codec_name?: string; + width?: number; + height?: number; + avg_frame_rate?: string; + r_frame_rate?: string; + duration?: string; + tags?: Record; + side_data_list?: Array<{ rotation?: number }>; +} + +interface FfprobeOutput { + streams?: FfprobeStream[]; + format?: { duration?: string; tags?: Record }; + frames?: Array<{ tags?: Record }>; +} + +/** Probe a local file. Throws when ffprobe is missing or cannot read the file. */ +export async function probeMedia( + file: string, + options: { ffprobePath?: string; signal?: AbortSignal; frameTags?: boolean } = {}, +): Promise { + const args = ["-v", "error", "-print_format", "json", "-show_format", "-show_streams"]; + if (options.frameTags) args.push("-show_frames", "-read_intervals", "%+#1"); + args.push("--", file); + const { stdout } = await runMediaTool(options.ffprobePath ?? "ffprobe", args, { + signal: options.signal, + }); + return parseFfprobe(JSON.parse(stdout.toString("utf8")) as FfprobeOutput); +} + +export function parseFfprobe(output: FfprobeOutput): ProbedMedia { + const streams = output.streams ?? []; + const video = streams.find((stream) => stream.codec_type === "video"); + const tags = collectTags(output); + const rotation = video ? streamRotation(video) : 0; + const quarterTurn = Math.abs(rotation) % 180 === 90; + const durationSeconds = Number(output.format?.duration ?? video?.duration); + const gps = parseIso6709( + tags.get("com.apple.quicktime.location.iso6709") ?? tags.get("location"), + ); + return { + durationMs: + Number.isFinite(durationSeconds) && durationSeconds > 0 + ? Math.round(durationSeconds * 1000) + : null, + width: (quarterTurn ? video?.height : video?.width) ?? null, + height: (quarterTurn ? video?.width : video?.height) ?? null, + fps: video ? (parseRate(video.avg_frame_rate) ?? parseRate(video.r_frame_rate)) : null, + codec: video?.codec_name ?? null, + rotation, + hasAudio: streams.some((stream) => stream.codec_type === "audio"), + hasVideo: Boolean(video), + capturedAt: parseCaptureDate( + tags.get("com.apple.quicktime.creationdate") ?? + tags.get("datetimeoriginal") ?? + tags.get("creation_time") ?? + tags.get("datetime"), + ), + gpsLat: gps?.lat ?? null, + gpsLon: gps?.lon ?? null, + deviceModel: tags.get("com.apple.quicktime.model") ?? tags.get("model") ?? null, + }; +} + +/** Format, stream and first-frame tags merged with lowercase keys; format wins. */ +function collectTags(output: FfprobeOutput): Map { + const merged = new Map(); + const sources = [ + output.format?.tags, + ...(output.streams ?? []).map((stream) => stream.tags), + ...(output.frames ?? []).map((frame) => frame.tags), + ]; + for (const source of sources) { + for (const [key, value] of Object.entries(source ?? {})) { + const normalized = key.toLowerCase(); + if (!merged.has(normalized) && typeof value === "string" && value.trim()) + merged.set(normalized, value.trim()); + } + } + return merged; +} + +function streamRotation(stream: FfprobeStream): number { + const side = stream.side_data_list?.find((entry) => typeof entry.rotation === "number"); + if (side?.rotation !== undefined) return side.rotation; + const tagged = Number(stream.tags?.rotate); + return Number.isFinite(tagged) ? tagged : 0; +} + +function parseRate(rate: string | undefined): number | null { + if (!rate) return null; + const [num, den] = rate.split("/").map(Number); + if (!num || !Number.isFinite(num)) return null; + const value = den ? num / den : num; + return Number.isFinite(value) && value > 0 ? Math.round(value * 1000) / 1000 : null; +} + +/** ISO 8601 or EXIF ("2026:05:01 09:00:00") timestamps; null when unparseable. */ +export function parseCaptureDate(value: string | undefined): Date | null { + if (!value) return null; + const exif = /^(\d{4}):(\d{2}):(\d{2})[ T](\d{2}):(\d{2}):(\d{2})/.exec(value); + const date = exif + ? new Date(`${exif[1]}-${exif[2]}-${exif[3]}T${exif[4]}:${exif[5]}:${exif[6]}Z`) + : new Date(value); + const time = date.getTime(); + return Number.isFinite(time) && time > 0 ? date : null; +} + +/** "+52.5200+013.4050+034.000/" → { lat, lon }; null when absent or out of range. */ +export function parseIso6709(value: string | undefined): { lat: number; lon: number } | null { + if (!value) return null; + const match = /^([+-]\d+(?:\.\d+)?)([+-]\d+(?:\.\d+)?)/.exec(value.trim()); + if (!match) return null; + const lat = Number(match[1]); + const lon = Number(match[2]); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) return null; + if (Math.abs(lat) > 90 || Math.abs(lon) > 180) return null; + return { lat, lon }; +} diff --git a/packages/adapters/src/media-folder-source.test.ts b/packages/adapters/src/media-folder-source.test.ts new file mode 100644 index 0000000..dbb4ff6 --- /dev/null +++ b/packages/adapters/src/media-folder-source.test.ts @@ -0,0 +1,177 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { MediaSourceItem } from "@rakazo/adapter-kit"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { ffmpegAvailable } from "./media-emulator-clips.js"; +import { FolderMediaSourceProvider } from "./media-folder-source.js"; +import { parseIso6709, runMediaTool } from "./media-folder-source-probe.js"; + +let root: string; +const provider = new FolderMediaSourceProvider(); + +async function touch(relative: string, content = "") { + const file = path.join(root, relative); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, content); +} + +async function listAll(limit: number, cursor: string | null = null) { + const pages: MediaSourceItem[][] = []; + let next = cursor; + do { + const page = await provider.list({ config: { path: root }, cursor: next, limit }); + expect(page.items.length).toBeLessThanOrEqual(limit); + pages.push(page.items); + next = page.nextCursor; + } while (next); + return pages.flat().map((item) => item.externalId); +} + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "media-folder-")); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +describe("FolderMediaSourceProvider", () => { + it("lists media in stable code-unit order across pages", async () => { + for (const name of ["b/c.mp4", "b-d.jpg", "a.MOV", "b/a/z.webm", "c.png", "notes.txt"]) + await touch(name); + const ids = await listAll(2); + expect(ids).toEqual(["a.MOV", "b-d.jpg", "b/a/z.webm", "b/c.mp4", "c.png"]); + }); + + it("keeps the cursor stable when files are added mid-sync", async () => { + for (const name of ["a.mp4", "c.mp4", "e.mp4", "g.mp4"]) await touch(name); + const first = await provider.list({ config: { path: root }, cursor: null, limit: 2 }); + expect(first.items.map((item) => item.externalId)).toEqual(["a.mp4", "c.mp4"]); + await touch("b.mp4"); // before the cursor: picked up by the next full sync + await touch("d.mp4"); // after the cursor: part of this sync + const rest = await listAll(2, first.nextCursor); + expect(rest).toEqual(["d.mp4", "e.mp4", "g.mp4"]); + expect(await listAll(10)).toEqual(["a.mp4", "b.mp4", "c.mp4", "d.mp4", "e.mp4", "g.mp4"]); + }); + + it("skips hidden entries and symlinks without following them", async () => { + await touch("keep/clip.mp4"); + await touch(".hidden.mp4"); + await touch(".cache/inside.mp4"); + await touch("keep/.DS_Store.jpg"); + const outside = await mkdtemp(path.join(tmpdir(), "media-folder-outside-")); + try { + await writeFile(path.join(outside, "secret.mp4"), ""); + await symlink(path.join(outside, "secret.mp4"), path.join(root, "link.mp4")); + await symlink(outside, path.join(root, "linked-dir")); + await symlink(path.join(root, "keep"), path.join(root, "keep-alias")); + expect(await listAll(10)).toEqual(["keep/clip.mp4"]); + await expect(provider.resolve({ path: root }, "link.mp4")).rejects.toThrow(); + await expect(provider.resolve({ path: root }, "linked-dir/secret.mp4")).rejects.toThrow(); + await expect(provider.resolve({ path: root }, "keep-alias/clip.mp4")).rejects.toThrow(); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + + it("resolves inside the root and rejects traversal", async () => { + await touch("keep/clip.mp4"); + await touch(".hidden.mp4"); + await expect(provider.resolve({ path: root }, "keep/clip.mp4")).resolves.toBe( + path.join(root, "keep/clip.mp4"), + ); + for (const bad of [ + "../x", + "../x.mp4", + "keep/../../x.mp4", + "/etc/passwd", + ".hidden.mp4", + "keep", + ]) + await expect(provider.resolve({ path: root }, bad), bad).rejects.toThrow(); + }); + + it("rejects relative roots and forged cursors", async () => { + await expect( + provider.list({ config: { path: "media" }, cursor: null, limit: 5 }), + ).rejects.toThrow(); + await expect( + provider.list({ config: { path: root }, cursor: "not base64url!", limit: 5 }), + ).rejects.toThrow(); + }); + + it("pages a 1,000-file folder without duplicates or gaps", async () => { + const expected: string[] = []; + for (let dir = 0; dir < 10; dir++) { + for (let file = 0; file < 100; file++) { + const relative = `d${dir}/f${String(file).padStart(3, "0")}.${file % 3 === 0 ? "jpg" : "mp4"}`; + // Unreadable media must not abort a page; most files are empty and skip probing. + await touch(relative, file % 50 === 7 ? "garbage, not a video" : ""); + expected.push(relative); + } + } + await touch("d3/ignore.txt"); + const pages: MediaSourceItem[][] = []; + let cursor: string | null = null; + do { + const page = await provider.list({ config: { path: root }, cursor, limit: 137 }); + pages.push(page.items); + cursor = page.nextCursor; + } while (cursor); + const items = pages.flat(); + expect(pages.length).toBe(Math.ceil(1000 / 137)); + expect(items.map((item) => item.externalId)).toEqual(expected.sort()); + expect(new Set(items.map((item) => item.externalId)).size).toBe(1000); + const garbage = items.find((item) => item.externalId === "d0/f007.mp4"); + expect(garbage).toMatchObject({ durationMs: null, width: null, sizeBytes: 20, kind: "video" }); + expect(garbage?.capturedAt).toBeInstanceOf(Date); + expect(garbage?.revision).toMatch(/^\d+-20$/); + expect(items.find((item) => item.externalId === "d0/f000.jpg")?.kind).toBe("photo"); + }); + + it.skipIf(!ffmpegAvailable())( + "reads duration, size, capture time and GPS with ffprobe", + async () => { + await mkdir(path.join(root, "trip"), { recursive: true }); + await runMediaTool("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:r=25:d=1", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-metadata", + "creation_time=2026-05-02T08:00:00Z", + "-metadata", + "location=+52.5200+013.4050/", + path.join(root, "trip/clip.mp4"), + ]); + const [item] = (await provider.list({ config: { path: root }, cursor: null, limit: 5 })) + .items; + expect(item).toMatchObject({ + externalId: "trip/clip.mp4", + kind: "video", + durationMs: 1000, + width: 320, + height: 180, + fps: 25, + gpsLat: 52.52, + gpsLon: 13.405, + albumNames: ["trip"], + }); + expect(item?.capturedAt?.toISOString()).toBe("2026-05-02T08:00:00.000Z"); + }, + ); + + it("parses ISO 6709 locations", () => { + expect(parseIso6709("+52.5200+013.4050+034.000/")).toEqual({ lat: 52.52, lon: 13.405 }); + expect(parseIso6709("-33.8688+151.2093/")).toEqual({ lat: -33.8688, lon: 151.2093 }); + expect(parseIso6709("+95.0+013.0/")).toBeNull(); + expect(parseIso6709("nowhere")).toBeNull(); + }); +}); diff --git a/packages/adapters/src/media-folder-source.ts b/packages/adapters/src/media-folder-source.ts new file mode 100644 index 0000000..7fa1d85 --- /dev/null +++ b/packages/adapters/src/media-folder-source.ts @@ -0,0 +1,221 @@ +import type { Dirent } from "node:fs"; +import { lstat, readdir } from "node:fs/promises"; +import path from "node:path"; +import type { + MediaSourceItem, + MediaSourceListRequest, + MediaSourceListResult, + MediaSourceProvider, +} from "@rakazo/adapter-kit"; +import type { MediaAssetKind } from "@rakazo/contracts"; +import { PathOutsideRootError, resolveInside } from "@rakazo/media/node"; +import { z } from "zod"; +import { probeMedia } from "./media-folder-source-probe.js"; + +/* + * Server-side folder source. The deployment owner picks an absolute root; the + * walk lists media files below it in a stable order (relative POSIX paths, + * code-unit sorted) so the opaque cursor (the last relative path) stays valid + * while files are added. Hidden entries and symlinks are skipped and never + * followed. Only the directories needed for one page are read. + */ + +export const FOLDER_MEDIA_EXTENSIONS: Readonly> = { + mp4: "video", + mov: "video", + m4v: "video", + mkv: "video", + webm: "video", + avi: "video", + jpg: "photo", + jpeg: "photo", + png: "photo", + heic: "photo", + heif: "photo", + webp: "photo", +}; + +const FolderConfigSchema = z.object({ + path: z.string().min(1).refine(path.isAbsolute, "folder path must be absolute"), +}); + +const MAX_PAGE = 1000; +const PROBE_CONCURRENCY = 4; + +export interface FolderMediaSourceOptions { + ffprobePath?: string; +} + +export function encodeFolderCursor(relativePath: string): string { + return Buffer.from(relativePath, "utf8").toString("base64url"); +} + +export function decodeFolderCursor(cursor: string | null): string | null { + if (!cursor) return null; + const decoded = Buffer.from(cursor, "base64url").toString("utf8"); + if (!decoded || decoded.includes("\0") || encodeFolderCursor(decoded) !== cursor) + throw new Error("invalid folder cursor"); + return decoded; +} + +function mediaKindOf(name: string): MediaAssetKind | null { + const ext = path.extname(name).slice(1).toLowerCase(); + return FOLDER_MEDIA_EXTENSIONS[ext] ?? null; +} + +/** Sort key that makes a depth-first walk yield relative paths in code-unit order. */ +function entryKey(entry: Dirent): string { + return entry.isDirectory() ? `${entry.name}/` : entry.name; +} + +function compareKeys(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** + * Yield relative paths of media files strictly after `after`, in order. + * Directories whose whole subtree sorts before the cursor are not read. + */ +async function* walkAfter( + root: string, + relativeDir: string, + after: string | null, + signal?: AbortSignal, +): AsyncGenerator { + signal?.throwIfAborted(); + let entries: Dirent[]; + try { + entries = await readdir(path.join(root, relativeDir), { withFileTypes: true }); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + // A directory removed or locked mid-walk is skipped; the root itself must exist. + if (relativeDir && (code === "ENOENT" || code === "EACCES" || code === "ENOTDIR")) return; + throw error; + } + const visible = entries + .filter((entry) => !entry.name.startsWith(".") && !entry.isSymbolicLink()) + .filter((entry) => entry.isDirectory() || (entry.isFile() && mediaKindOf(entry.name))) + .map((entry) => ({ entry, key: entryKey(entry) })) + .sort((a, b) => compareKeys(a.key, b.key)); + for (const { entry, key } of visible) { + const relative = relativeDir ? `${relativeDir}/${key}` : key; + if (entry.isDirectory()) { + // Subtree entirely before the cursor: every path in it starts with `relative`. + if (after !== null && relative < after && !after.startsWith(relative)) continue; + yield* walkAfter(root, relative.slice(0, -1), after, signal); + } else if (after === null || relative > after) { + yield relative; + } + } +} + +async function mapLimited( + items: T[], + limit: number, + fn: (item: T) => Promise, +): Promise { + const out = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const index = next++; + out[index] = await fn(items[index] as T); + } + }); + await Promise.all(workers); + return out; +} + +export class FolderMediaSourceProvider implements MediaSourceProvider { + readonly kind = "folder" as const; + + constructor(private readonly options: FolderMediaSourceOptions = {}) {} + + async list(request: MediaSourceListRequest): Promise { + const { path: root } = FolderConfigSchema.parse(request.config); + const limit = Math.max(1, Math.min(MAX_PAGE, Math.floor(request.limit))); + const after = decodeFolderCursor(request.cursor); + const relatives: string[] = []; + let more = false; + for await (const relative of walkAfter(root, "", after, request.signal)) { + if (relatives.length === limit) { + more = true; + break; + } + relatives.push(relative); + } + const items = await mapLimited(relatives, PROBE_CONCURRENCY, (relative) => + this.describe(root, relative, request.signal), + ); + const present = items.filter((item): item is MediaSourceItem => item !== null); + const last = relatives.at(-1); + return { items: present, nextCursor: more && last ? encodeFolderCursor(last) : null }; + } + + async resolve(config: Record, externalId: string): Promise { + const { path: root } = FolderConfigSchema.parse(config); + const segments = externalId.split("/"); + if (segments.some((segment) => segment.startsWith(".")) || !mediaKindOf(externalId)) + throw new PathOutsideRootError(externalId); + const absolute = await resolveInside(root, externalId); + // Symlinks are never followed, at any depth below the root. + let current = await resolveInside(root, "."); + for (const segment of segments) { + current = path.join(current, segment); + if ((await lstat(current)).isSymbolicLink()) throw new PathOutsideRootError(externalId); + } + if (!(await lstat(absolute)).isFile()) throw new PathOutsideRootError(externalId); + return absolute; + } + + /** + * Stat and probe one file. Probe failures yield null metadata, never a failed + * page. The parent folder name doubles as an album for rule-based labels. + */ + private async describe( + root: string, + relative: string, + signal?: AbortSignal, + ): Promise { + const absolute = path.join(root, relative); + let stats: Awaited>; + try { + stats = await lstat(absolute); + } catch { + return null; // Removed between walk and stat. + } + if (!stats.isFile()) return null; + const kind = mediaKindOf(relative) ?? "video"; + let probed: Awaited> | null = null; + if (stats.size > 0) { + try { + probed = await probeMedia(absolute, { + ffprobePath: this.options.ffprobePath, + signal, + frameTags: kind === "photo", + }); + } catch { + // Unreadable, unsupported, or no ffprobe on this host: keep the file with null metadata. + signal?.throwIfAborted(); + probed = null; + } + } + return { + externalId: relative, + kind, + capturedAt: probed?.capturedAt ?? new Date(Math.trunc(stats.mtimeMs)), + durationMs: kind === "video" ? (probed?.durationMs ?? null) : null, + width: probed?.width ?? null, + height: probed?.height ?? null, + fps: kind === "video" ? (probed?.fps ?? null) : null, + gpsLat: probed?.gpsLat ?? null, + gpsLon: probed?.gpsLon ?? null, + deviceModel: probed?.deviceModel ?? null, + albumNames: relative.includes("/") ? [path.posix.basename(path.posix.dirname(relative))] : [], + isFavorite: false, + fileName: path.posix.basename(relative), + sizeBytes: stats.size, + revision: `${Math.trunc(stats.mtimeMs)}-${stats.size}`, + }; + } +} diff --git a/packages/adapters/src/media-sidecar.test.ts b/packages/adapters/src/media-sidecar.test.ts new file mode 100644 index 0000000..ea502f8 --- /dev/null +++ b/packages/adapters/src/media-sidecar.test.ts @@ -0,0 +1,222 @@ +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import { MediaInferenceOperationSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { MediaSidecarError, SidecarMediaInferenceProvider } from "./media-sidecar.js"; + +const TOKEN = "test-token-0123456789abcdef0123456789abcdef"; +const SCENES = { + shots: [{ startMs: 0, endMs: 1000, score: 1 }], + scenes: [{ startMs: 0, endMs: 1000 }], +}; + +interface Call { + url: string; + init: RequestInit; +} + +function json(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function fakeFetch(responses: Array Promise)>) { + const calls: Call[] = []; + const impl = (async (input: string | URL | Request, init?: RequestInit) => { + calls.push({ url: String(input), init: init ?? {} }); + const next = responses.shift(); + if (!next) throw new Error("unexpected request"); + return typeof next === "function" ? next(init ?? {}) : next; + }) as typeof fetch; + return { impl, calls }; +} + +function client(fetchImpl: typeof fetch, extra: { retryBaseMs?: number } = {}) { + return new SidecarMediaInferenceProvider({ + baseUrl: "http://127.0.0.1:8800/", + token: TOKEN, + fetch: fetchImpl, + retryBaseMs: 0, + ...extra, + }); +} + +describe("SidecarMediaInferenceProvider", () => { + it("posts camelCase JSON with bearer auth and without the signal", async () => { + const { impl, calls } = fakeFetch([json(200, SCENES)]); + const controller = new AbortController(); + const result = await client(impl).detectScenes({ + path: "/data/media/a.mp4", + signal: controller.signal, + }); + expect(result).toEqual(SCENES); + expect(calls).toHaveLength(1); + const [call] = calls; + expect(call?.url).toBe("http://127.0.0.1:8800/v1/detectScenes"); + expect(call?.init.method).toBe("POST"); + const headers = call?.init.headers as Record; + expect(headers.authorization).toBe(`Bearer ${TOKEN}`); + expect(headers["content-type"]).toBe("application/json"); + expect(JSON.parse(String(call?.init.body))).toEqual({ path: "/data/media/a.mp4" }); + expect(call?.init.signal).toBeInstanceOf(AbortSignal); + }); + + it("reads capabilities with GET", async () => { + const capabilities = { + configured: true, + operations: Object.fromEntries( + MediaInferenceOperationSchema.options.map((operation) => [ + operation, + { + available: operation !== "synthesizeIdentity", + experimental: false, + model: "m", + license: "Apache-2.0", + commercialUse: true, + }, + ]), + ), + accelerator: "cpu", + hardwareEncoders: [], + }; + const { impl, calls } = fakeFetch([json(200, capabilities)]); + await expect(client(impl).capabilities()).resolves.toEqual(capabilities); + expect(calls[0]?.url).toBe("http://127.0.0.1:8800/v1/capabilities"); + expect(calls[0]?.init.method).toBe("GET"); + }); + + it("maps 501 to UnsupportedMediaOperationError", async () => { + const { impl } = fakeFetch([ + json(501, { + error: { code: "unsupported", message: "no licensed model", operation: "inpaintVideo" }, + }), + ]); + const error = await client(impl) + .inpaintVideo({ path: "/m/a.mp4", maskDir: "/m/masks", outPath: "/m/out.mp4" }) + .catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(UnsupportedMediaOperationError); + expect((error as UnsupportedMediaOperationError).operation).toBe("inpaintVideo"); + expect((error as Error).message).toContain("no licensed model"); + }); + + it("retries 503 busy with backoff, at most three times", async () => { + const busy = () => json(503, { error: { code: "busy", message: "queue full" } }); + const recovered = fakeFetch([busy(), busy(), json(200, SCENES)]); + await expect(client(recovered.impl).detectScenes({ path: "/m/a.mp4" })).resolves.toEqual( + SCENES, + ); + expect(recovered.calls).toHaveLength(3); + + const exhausted = fakeFetch([busy(), busy(), busy(), busy(), busy()]); + const error = await client(exhausted.impl) + .detectScenes({ path: "/m/a.mp4" }) + .catch((caught: unknown) => caught); + expect(exhausted.calls).toHaveLength(4); + expect(error).toBeInstanceOf(MediaSidecarError); + expect((error as MediaSidecarError).code).toBe("busy"); + expect((error as MediaSidecarError).status).toBe(503); + }); + + it("maps typed sidecar errors", async () => { + const { impl } = fakeFetch([ + json(400, { error: { code: "invalid_path", message: "path is outside MEDIA_ROOT" } }), + json(500, "not json-shaped"), + ]); + const provider = client(impl); + await expect(provider.detectScenes({ path: "/etc/passwd" })).rejects.toMatchObject({ + name: "MediaSidecarError", + code: "invalid_path", + status: 400, + operation: "detectScenes", + }); + await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({ + code: "http_error", + status: 500, + }); + }); + + it("passes caller aborts through as AbortError", async () => { + const controller = new AbortController(); + const { impl } = fakeFetch([ + (init) => + new Promise((_, reject) => { + init.signal?.addEventListener("abort", () => reject(init.signal?.reason)); + setTimeout(() => controller.abort(), 10); + }), + ]); + await expect( + client(impl).trackFaces({ path: "/m/a.mp4", sampleFps: 5, signal: controller.signal }), + ).rejects.toMatchObject({ name: "AbortError" }); + + const already = new AbortController(); + already.abort(); + const idle = fakeFetch([]); + await expect( + client(idle.impl).detectScenes({ path: "/m/a.mp4", signal: already.signal }), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(idle.calls).toHaveLength(0); + }); + + it("reports a timeout as a typed error", async () => { + const { impl } = fakeFetch([ + (init) => + new Promise((_, reject) => { + init.signal?.addEventListener("abort", () => reject(init.signal?.reason)); + }), + ]); + const provider = new SidecarMediaInferenceProvider({ + baseUrl: "http://127.0.0.1:8800", + token: TOKEN, + fetch: impl, + timeoutsMs: { detectScenes: 20 }, + }); + await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({ + code: "timeout", + }); + }); + + it("rejects malformed responses", async () => { + const { impl } = fakeFetch([ + json(200, { shots: [{ startMs: -1, endMs: 10, score: 1 }], scenes: [] }), + json(200, { + frames: [{ tMs: 0, faces: [{ box: { x: 2, y: 0, w: 0.1, h: 0.1 }, confidence: 1 }] }], + }), + json(200, { model: "m", dim: 3, vectors: [[1, 2]] }), + new Response("", { status: 200 }), + ]); + const provider = client(impl); + await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({ + code: "malformed_response", + }); + await expect(provider.detectFaces({ path: "/m/a.mp4", sampleFps: 1 })).rejects.toMatchObject({ + code: "malformed_response", + }); + await expect(provider.embedFrames({ path: "/m/a.mp4", timesMs: [0] })).rejects.toMatchObject({ + code: "malformed_response", + }); + await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({ + code: "malformed_response", + }); + }); + + it("never puts the token into error messages", async () => { + const { impl } = fakeFetch([ + json(401, { error: { code: "unauthorized", message: `bad token ${TOKEN}` } }), + json(500, { error: { code: "internal", message: `crashed with Bearer ${TOKEN}` } }), + async () => { + throw new TypeError(`connect ECONNREFUSED (auth ${TOKEN})`); + }, + ]); + const provider = client(impl); + for (let i = 0; i < 3; i++) { + const error = (await provider + .detectScenes({ path: "/m/a.mp4" }) + .catch((caught: unknown) => caught)) as Error; + expect(error).toBeInstanceOf(MediaSidecarError); + expect(error.message).not.toContain(TOKEN); + expect(JSON.stringify(error)).not.toContain(TOKEN); + expect(String(error.stack)).not.toContain(TOKEN); + } + }); +}); diff --git a/packages/adapters/src/media-sidecar.ts b/packages/adapters/src/media-sidecar.ts new file mode 100644 index 0000000..8ca7202 --- /dev/null +++ b/packages/adapters/src/media-sidecar.ts @@ -0,0 +1,410 @@ +import type { + AudioFeaturesResult, + ClassifyContentResult, + DetectFacesResult, + DetectScenesResult, + EmbedFramesResult, + FaceLandmarksResult, + InpaintVideoResult, + MediaInferenceProvider, + SaliencyResult, + SegmentPromptedResult, + TrackFacesResult, + TranscribeResult, +} from "@rakazo/adapter-kit"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import type { MediaCapabilities, MediaInferenceOperation } from "@rakazo/contracts"; +import { + BoxKeyframeSchema, + ContentCategorySchema, + MediaCapabilitiesSchema, + NormalizedBoxSchema, + NormalizedPointSchema, +} from "@rakazo/contracts"; +import { z } from "zod"; + +/* + * HTTP client for the local media sidecar (infra/media-worker, API v1). + * Bearer auth, JSON bodies, per-operation timeouts, bounded retry on 503 busy, + * 501 → UnsupportedMediaOperationError, every other failure → MediaSidecarError. + * Responses are validated before they reach the caller. The token never + * appears in an error message. + */ + +const DEFAULT_TIMEOUT_MS = 30 * 60_000; +const LONG_TIMEOUT_MS = 4 * 60 * 60_000; +const MAX_BUSY_RETRIES = 3; + +export type MediaSidecarErrorCode = + | "bad_request" + | "invalid_path" + | "unauthorized" + | "too_large" + | "busy" + | "internal" + | "timeout" + | "unreachable" + | "malformed_response" + | "http_error"; + +export class MediaSidecarError extends Error { + constructor( + readonly code: MediaSidecarErrorCode, + message: string, + readonly status: number | null, + readonly operation: MediaInferenceOperation | "capabilities", + ) { + super(message); + this.name = "MediaSidecarError"; + } +} + +// --------------------------------------------------------------------------- +// Response schemas (shapes mirror the adapter-kit result types) +// --------------------------------------------------------------------------- + +const Ms = z.number().int().min(0); +const Unit = z.number().min(0).max(1); +const Span = z.object({ startMs: Ms, endMs: Ms }); +const ScoredKeyframe = BoxKeyframeSchema.extend({ confidence: Unit }); + +const schemas = { + detectScenes: z.object({ + shots: z.array(Span.extend({ score: z.number() })), + scenes: z.array(Span), + }) satisfies z.ZodType, + transcribe: z.object({ + language: z.string().nullable(), + words: z.array(Span.extend({ text: z.string(), confidence: Unit })), + }) satisfies z.ZodType, + detectFaces: z.object({ + frames: z.array( + z.object({ + tMs: Ms, + faces: z.array( + z.object({ + box: NormalizedBoxSchema, + confidence: Unit, + landmarks: z.array(NormalizedPointSchema).optional(), + }), + ), + }), + ), + }) satisfies z.ZodType, + trackFaces: z.object({ + sampleFps: z.number().positive(), + tracks: z.array( + z.object({ + trackId: z.string().min(1), + keyframes: z.array(ScoredKeyframe), + embedding: z.array(z.number()).nullable(), + }), + ), + }) satisfies z.ZodType, + faceLandmarks: z.object({ + frames: z.array( + z.object({ + tMs: Ms, + faces: z.array( + z.object({ box: NormalizedBoxSchema, landmarks: z.array(NormalizedPointSchema) }), + ), + }), + ), + }) satisfies z.ZodType, + embedFrames: z + .object({ + model: z.string(), + dim: z.number().int().positive(), + vectors: z.array(z.array(z.number())), + }) + .refine((value) => value.vectors.every((vector) => vector.length === value.dim), { + message: "vector length does not match dim", + }) satisfies z.ZodType, + classifyContent: z.object({ + model: z.string(), + frames: z.array(z.object({ tMs: Ms, scores: z.partialRecord(ContentCategorySchema, Unit) })), + }) satisfies z.ZodType, + segmentPrompted: z.object({ + maskDir: z.string().min(1), + keyframes: z.array(ScoredKeyframe), + score: Unit, + }) satisfies z.ZodType, + inpaintVideo: z.object({ + outPath: z.string().min(1), + quality: z.object({ temporalStability: Unit, edgeQuality: Unit, score: Unit }), + }) satisfies z.ZodType, + redact: z.object({ + outPath: z.string().min(1), + framesProcessed: z.number().int().min(0), + }) satisfies z.ZodType<{ outPath: string; framesProcessed: number }>, + synthesizeIdentity: z.object({ outPath: z.string().min(1) }) satisfies z.ZodType<{ + outPath: string; + }>, + audioFeatures: z.object({ + integratedLufs: z.number().nullable(), + truePeakDb: z.number().nullable(), + tempoBpm: z.number().nullable(), + beatsMs: z.array(Ms), + energy: z.array(z.object({ tMs: Ms, value: z.number() })), + silences: z.array(Span), + }) satisfies z.ZodType, + saliency: z.object({ + points: z.array(z.object({ tMs: Ms, cx: Unit, cy: Unit, strength: z.number() })), + }) satisfies z.ZodType, +} as const satisfies Record; + +export const MEDIA_SIDECAR_RESPONSE_SCHEMAS = schemas; + +const ErrorBodySchema = z.object({ + error: z.object({ code: z.string(), message: z.string().optional() }), +}); + +// --------------------------------------------------------------------------- +// Client +// --------------------------------------------------------------------------- + +export interface SidecarMediaInferenceOptions { + baseUrl: string; + token: string; + fetch?: typeof fetch; + /** Per-operation timeout overrides in ms. */ + timeoutsMs?: Partial>; + /** First busy-retry delay; doubles per attempt. */ + retryBaseMs?: number; +} + +type Request = Parameters[0]; +type Result = Awaited>; + +const KNOWN_CODES = new Set([ + "bad_request", + "invalid_path", + "unauthorized", + "too_large", + "busy", + "internal", +]); + +export class SidecarMediaInferenceProvider implements MediaInferenceProvider { + readonly id = "sidecar"; + private readonly baseUrl: string; + private readonly fetchImpl: typeof fetch; + + constructor(private readonly options: SidecarMediaInferenceOptions) { + this.baseUrl = options.baseUrl.replace(/\/+$/, ""); + this.fetchImpl = options.fetch ?? fetch; + } + + async capabilities(signal?: AbortSignal): Promise { + return this.call("capabilities", "GET", "/v1/capabilities", undefined, signal, (body) => + MediaCapabilitiesSchema.parse(body), + ); + } + + detectScenes(request: Request<"detectScenes">) { + return this.operation("detectScenes", request); + } + transcribe(request: Request<"transcribe">) { + return this.operation("transcribe", request); + } + detectFaces(request: Request<"detectFaces">) { + return this.operation("detectFaces", request); + } + trackFaces(request: Request<"trackFaces">) { + return this.operation("trackFaces", request); + } + faceLandmarks(request: Request<"faceLandmarks">) { + return this.operation("faceLandmarks", request); + } + embedFrames(request: Request<"embedFrames">) { + return this.operation("embedFrames", request); + } + classifyContent(request: Request<"classifyContent">) { + return this.operation("classifyContent", request); + } + segmentPrompted(request: Request<"segmentPrompted">) { + return this.operation("segmentPrompted", request); + } + inpaintVideo(request: Request<"inpaintVideo">) { + return this.operation("inpaintVideo", request); + } + redact(request: Request<"redact">) { + return this.operation("redact", request); + } + synthesizeIdentity(request: Request<"synthesizeIdentity">) { + return this.operation("synthesizeIdentity", request); + } + audioFeatures(request: Request<"audioFeatures">) { + return this.operation("audioFeatures", request); + } + saliency(request: Request<"saliency">) { + return this.operation("saliency", request); + } + + private operation( + operation: K, + request: Request, + ): Promise> { + const { signal, ...body } = request; + const schema = schemas[operation] as z.ZodType; + return this.call(operation, "POST", `/v1/${operation}`, body, signal, (json) => { + return schema.parse(json) as Result; + }); + } + + private timeoutFor(operation: MediaInferenceOperation | "capabilities"): number { + const override = this.options.timeoutsMs?.[operation]; + if (override !== undefined) return override; + return operation === "inpaintVideo" || operation === "synthesizeIdentity" + ? LONG_TIMEOUT_MS + : DEFAULT_TIMEOUT_MS; + } + + private async call( + operation: MediaInferenceOperation | "capabilities", + method: "GET" | "POST", + route: string, + body: unknown, + signal: AbortSignal | undefined, + parse: (json: unknown) => T, + ): Promise { + signal?.throwIfAborted(); + const timeout = AbortSignal.timeout(this.timeoutFor(operation)); + const combined = signal ? AbortSignal.any([signal, timeout]) : timeout; + const retryBase = this.options.retryBaseMs ?? 500; + for (let attempt = 0; ; attempt++) { + let response: Response; + try { + response = await this.fetchImpl(`${this.baseUrl}${route}`, { + method, + headers: { + authorization: `Bearer ${this.options.token}`, + accept: "application/json", + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: combined, + }); + } catch (error) { + throw this.transportError(error, operation, signal, timeout); + } + if (response.status === 503 && attempt < MAX_BUSY_RETRIES) { + await response.body?.cancel().catch(() => undefined); + try { + await delay(retryBase * 2 ** attempt, combined); + } catch (error) { + throw this.transportError(error, operation, signal, timeout); + } + continue; + } + let json: unknown; + let text: string; + try { + text = await response.text(); + } catch (error) { + throw this.transportError(error, operation, signal, timeout); + } + try { + json = text ? JSON.parse(text) : null; + } catch { + json = undefined; + } + if (!response.ok) throw this.httpError(response.status, json, operation); + if (json === undefined) + throw new MediaSidecarError( + "malformed_response", + `media sidecar returned invalid JSON for ${operation}`, + response.status, + operation, + ); + try { + return parse(json); + } catch (error) { + const detail = + error instanceof z.ZodError + ? error.issues + .slice(0, 3) + .map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) + .join("; ") + : "unexpected shape"; + throw new MediaSidecarError( + "malformed_response", + this.scrub(`media sidecar returned a malformed ${operation} response: ${detail}`), + response.status, + operation, + ); + } + } + } + + private httpError( + status: number, + json: unknown, + operation: MediaInferenceOperation | "capabilities", + ): Error { + const parsed = ErrorBodySchema.safeParse(json); + const message = this.scrub( + parsed.success && parsed.data.error.message + ? parsed.data.error.message + : `media sidecar responded ${status}`, + ); + if (status === 501 && operation !== "capabilities") + return new UnsupportedMediaOperationError(operation, message); + const serverCode = parsed.success ? parsed.data.error.code : ""; + const code: MediaSidecarErrorCode = KNOWN_CODES.has(serverCode) + ? (serverCode as MediaSidecarErrorCode) + : status === 401 + ? "unauthorized" + : status === 503 + ? "busy" + : "http_error"; + return new MediaSidecarError(code, `${operation}: ${message}`, status, operation); + } + + private transportError( + error: unknown, + operation: MediaInferenceOperation | "capabilities", + signal: AbortSignal | undefined, + timeout: AbortSignal, + ): unknown { + if (signal?.aborted) return signal.reason; + if (timeout.aborted) + return new MediaSidecarError( + "timeout", + `media sidecar ${operation} timed out`, + null, + operation, + ); + const detail = error instanceof Error ? error.message : String(error); + return new MediaSidecarError( + "unreachable", + this.scrub(`media sidecar is unreachable: ${detail}`), + null, + operation, + ); + } + + /** Strip the bearer token from any text that may end up in logs or errors. */ + private scrub(text: string): string { + const token = this.options.token; + return token ? text.split(token).join("[redacted]") : text; + } +} + +function delay(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal.aborted) { + reject(signal.reason); + return; + } + const timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + reject(signal.reason); + }; + signal.addEventListener("abort", onAbort, { once: true }); + }); +}