feat(media): inference emulator, sidecar client, folder source and factory
Add the Media Studio adapters behind the frozen provider contracts: a deterministic offline inference emulator driven by file-name markers that renders real redactions with ffmpeg, a synthetic emulator library, the bearer-auth HTTP client for the local media sidecar with validated responses, typed errors, busy retries and timeouts, a lazily paginated folder source that never follows symlinks, and the configuration factory. A shared conformance suite runs against the emulator and, in the Docker integration job, against the sidecar. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj
This commit is contained in:
Claude
parent
7ccc96ac11
commit
270ea8068a
281
packages/adapters/src/media-conformance.test.ts
Normal file
281
packages/adapters/src/media-conformance.test.ts
Normal file
@ -0,0 +1,281 @@
|
|||||||
|
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
|
||||||
|
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||||
|
import type { MediaInferenceOperation } from "@rakazo/contracts";
|
||||||
|
import { MediaCapabilitiesSchema, MediaInferenceOperationSchema } from "@rakazo/contracts";
|
||||||
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import { EmulatorMediaInferenceProvider } from "./media-emulator.js";
|
||||||
|
import { extractRgbFrame, ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js";
|
||||||
|
import { MEDIA_SIDECAR_RESPONSE_SCHEMAS, SidecarMediaInferenceProvider } from "./media-sidecar.js";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* One contract for every MediaInferenceProvider. The emulator always runs; the
|
||||||
|
* sidecar runs only in the Docker integration job, which sets
|
||||||
|
* MEDIA_CONFORMANCE_URL, MEDIA_CONFORMANCE_TOKEN and MEDIA_CONFORMANCE_ROOT (a
|
||||||
|
* directory inside the sidecar's MEDIA_ROOT, mounted at the same path here).
|
||||||
|
*/
|
||||||
|
|
||||||
|
const WIDTH = 320;
|
||||||
|
const HEIGHT = 180;
|
||||||
|
const hasFfmpeg = ffmpegAvailable();
|
||||||
|
|
||||||
|
interface Target {
|
||||||
|
name: string;
|
||||||
|
/** Directory inside the provider's root where the suite writes fixtures. */
|
||||||
|
root: () => Promise<string>;
|
||||||
|
provider: (root: string) => MediaInferenceProvider;
|
||||||
|
}
|
||||||
|
|
||||||
|
const targets: Target[] = [
|
||||||
|
{
|
||||||
|
name: "emulator",
|
||||||
|
root: () => mkdtemp(path.join(tmpdir(), "media-conformance-")),
|
||||||
|
provider: (root) => new EmulatorMediaInferenceProvider({ root }),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const sidecarUrl = process.env.MEDIA_CONFORMANCE_URL;
|
||||||
|
const sidecarToken = process.env.MEDIA_CONFORMANCE_TOKEN;
|
||||||
|
const sidecarRoot = process.env.MEDIA_CONFORMANCE_ROOT;
|
||||||
|
if (sidecarUrl && sidecarToken && sidecarRoot) {
|
||||||
|
targets.push({
|
||||||
|
name: "sidecar",
|
||||||
|
root: async () => {
|
||||||
|
await mkdir(sidecarRoot, { recursive: true });
|
||||||
|
return mkdtemp(path.join(sidecarRoot, "conformance-"));
|
||||||
|
},
|
||||||
|
provider: () => new SidecarMediaInferenceProvider({ baseUrl: sidecarUrl, token: sidecarToken }),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function requestFor(operation: MediaInferenceOperation, clip: string, root: string) {
|
||||||
|
const box = { x: 0.25, y: 0.25, w: 0.25, h: 0.25 };
|
||||||
|
switch (operation) {
|
||||||
|
case "detectScenes":
|
||||||
|
case "audioFeatures":
|
||||||
|
return { path: clip };
|
||||||
|
case "transcribe":
|
||||||
|
return { path: clip, language: "en" };
|
||||||
|
case "detectFaces":
|
||||||
|
case "trackFaces":
|
||||||
|
return { path: clip, sampleFps: 2 };
|
||||||
|
case "faceLandmarks":
|
||||||
|
case "embedFrames":
|
||||||
|
case "saliency":
|
||||||
|
return { path: clip, timesMs: [0, 500, 1000] };
|
||||||
|
case "classifyContent":
|
||||||
|
return { path: clip, sampleFps: 2, extraTimesMs: [750] };
|
||||||
|
case "segmentPrompted":
|
||||||
|
return {
|
||||||
|
path: clip,
|
||||||
|
prompt: { tMs: 500, box },
|
||||||
|
propagate: true,
|
||||||
|
outDir: path.join(root, "masks"),
|
||||||
|
};
|
||||||
|
case "inpaintVideo":
|
||||||
|
return {
|
||||||
|
path: clip,
|
||||||
|
maskDir: path.join(root, "masks"),
|
||||||
|
outPath: path.join(root, "inpaint.mp4"),
|
||||||
|
};
|
||||||
|
case "redact":
|
||||||
|
return {
|
||||||
|
path: clip,
|
||||||
|
redactions: [
|
||||||
|
{
|
||||||
|
keyframes: [
|
||||||
|
{ tMs: 0, box },
|
||||||
|
{ tMs: 2000, box },
|
||||||
|
],
|
||||||
|
mode: "blur" as const,
|
||||||
|
paddingPct: 0,
|
||||||
|
featherPct: 0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
outPath: path.join(root, "redacted-any.mp4"),
|
||||||
|
};
|
||||||
|
case "synthesizeIdentity":
|
||||||
|
return {
|
||||||
|
path: clip,
|
||||||
|
track: [{ tMs: 0, box }],
|
||||||
|
seed: 7,
|
||||||
|
outPath: path.join(root, "synthetic.mp4"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Mean absolute RGB difference inside and outside a pixel box (with a guard margin). */
|
||||||
|
function regionDiff(
|
||||||
|
a: Buffer,
|
||||||
|
b: Buffer,
|
||||||
|
box: { x: number; y: number; w: number; h: number },
|
||||||
|
margin = 4,
|
||||||
|
) {
|
||||||
|
let inside = 0;
|
||||||
|
let insideCount = 0;
|
||||||
|
let outside = 0;
|
||||||
|
let outsideCount = 0;
|
||||||
|
for (let y = 0; y < HEIGHT; y++) {
|
||||||
|
for (let x = 0; x < WIDTH; x++) {
|
||||||
|
const within = x >= box.x && x < box.x + box.w && y >= box.y && y < box.y + box.h;
|
||||||
|
const near =
|
||||||
|
x >= box.x - margin &&
|
||||||
|
x < box.x + box.w + margin &&
|
||||||
|
y >= box.y - margin &&
|
||||||
|
y < box.y + box.h + margin;
|
||||||
|
let diff = 0;
|
||||||
|
for (let c = 0; c < 3; c++) {
|
||||||
|
const i = (y * WIDTH + x) * 3 + c;
|
||||||
|
diff += Math.abs((a[i] ?? 0) - (b[i] ?? 0));
|
||||||
|
}
|
||||||
|
if (within) {
|
||||||
|
inside += diff / 3;
|
||||||
|
insideCount++;
|
||||||
|
} else if (!near) {
|
||||||
|
outside += diff / 3;
|
||||||
|
outsideCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { inside: inside / insideCount, outside: outside / outsideCount };
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const target of targets) {
|
||||||
|
describe.skipIf(!hasFfmpeg)(`MediaInferenceProvider conformance: ${target.name}`, () => {
|
||||||
|
let root: string;
|
||||||
|
let clip: string;
|
||||||
|
let provider: MediaInferenceProvider;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
root = await target.root();
|
||||||
|
clip = await renderTestClip(path.join(root, "clip.mp4"), {
|
||||||
|
seconds: 2,
|
||||||
|
width: WIDTH,
|
||||||
|
height: HEIGHT,
|
||||||
|
});
|
||||||
|
provider = target.provider(root);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
if (root) await rm(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a capabilities document for every operation", async () => {
|
||||||
|
const capabilities = MediaCapabilitiesSchema.parse(await provider.capabilities());
|
||||||
|
expect(Object.keys(capabilities.operations).sort()).toEqual(
|
||||||
|
[...MediaInferenceOperationSchema.options].sort(),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns schema-valid output or Unsupported for every operation", async () => {
|
||||||
|
const capabilities = await provider.capabilities();
|
||||||
|
// segmentPrompted runs first so inpaintVideo finds its mask directory.
|
||||||
|
for (const operation of MediaInferenceOperationSchema.options) {
|
||||||
|
const request = requestFor(operation, clip, root);
|
||||||
|
const call = provider[operation] as (input: unknown) => Promise<unknown>;
|
||||||
|
try {
|
||||||
|
const result = await call.call(provider, request);
|
||||||
|
const parsed = MEDIA_SIDECAR_RESPONSE_SCHEMAS[operation].safeParse(result);
|
||||||
|
expect(parsed.success, `${operation}: ${parsed.error?.message}`).toBe(true);
|
||||||
|
expect(capabilities.operations[operation]?.available, operation).toBe(true);
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof UnsupportedMediaOperationError)) throw error;
|
||||||
|
expect(error.operation).toBe(operation);
|
||||||
|
expect(capabilities.operations[operation]?.available, operation).toBe(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects paths outside the media root", async () => {
|
||||||
|
const outside = await mkdtemp(path.join(tmpdir(), "media-outside-"));
|
||||||
|
try {
|
||||||
|
const stray = path.join(outside, "stray.mp4");
|
||||||
|
await writeFile(stray, "not inside the root");
|
||||||
|
await expect(provider.detectScenes({ path: stray })).rejects.toThrow();
|
||||||
|
await expect(
|
||||||
|
provider.detectScenes({
|
||||||
|
path: path.join(root, "..", path.basename(outside), "stray.mp4"),
|
||||||
|
}),
|
||||||
|
).rejects.toThrow();
|
||||||
|
await expect(
|
||||||
|
provider.redact({
|
||||||
|
path: clip,
|
||||||
|
redactions: [],
|
||||||
|
outPath: path.join(outside, "escaped.mp4"),
|
||||||
|
}),
|
||||||
|
).rejects.toThrow();
|
||||||
|
} finally {
|
||||||
|
await rm(outside, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("redacts only inside the box", async () => {
|
||||||
|
const box = { x: 0.25, y: 0.25, w: 0.25, h: 0.25 };
|
||||||
|
const pixels = { x: 80, y: 44, w: 80, h: 46 };
|
||||||
|
for (const mode of ["solid", "blur"] as const) {
|
||||||
|
const outPath = path.join(root, `redacted-${mode}.mp4`);
|
||||||
|
const result = await provider.redact({
|
||||||
|
path: clip,
|
||||||
|
outPath,
|
||||||
|
redactions: [
|
||||||
|
{
|
||||||
|
keyframes: [
|
||||||
|
{ tMs: 0, box },
|
||||||
|
{ tMs: 2000, box },
|
||||||
|
],
|
||||||
|
mode,
|
||||||
|
paddingPct: 0,
|
||||||
|
featherPct: 0,
|
||||||
|
color: "#102030",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
expect(result.framesProcessed).toBeGreaterThan(0);
|
||||||
|
const source = await extractRgbFrame(clip, 1000);
|
||||||
|
const redacted = await extractRgbFrame(result.outPath, 1000);
|
||||||
|
expect(redacted.length).toBe(WIDTH * HEIGHT * 3);
|
||||||
|
const diff = regionDiff(source, redacted, pixels);
|
||||||
|
expect(diff.inside, mode).toBeGreaterThan(8);
|
||||||
|
expect(diff.outside, mode).toBeLessThan(1.5);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("honours AbortSignal", async () => {
|
||||||
|
const aborted = new AbortController();
|
||||||
|
aborted.abort();
|
||||||
|
await expect(
|
||||||
|
provider.detectScenes({ path: clip, signal: aborted.signal }),
|
||||||
|
).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
await expect(provider.capabilities(aborted.signal)).rejects.toMatchObject({
|
||||||
|
name: "AbortError",
|
||||||
|
});
|
||||||
|
|
||||||
|
const long = await renderTestClip(path.join(root, "long.mp4"), {
|
||||||
|
seconds: 20,
|
||||||
|
width: 1280,
|
||||||
|
height: 720,
|
||||||
|
});
|
||||||
|
const controller = new AbortController();
|
||||||
|
const pending = provider.redact({
|
||||||
|
path: long,
|
||||||
|
outPath: path.join(root, "aborted.mp4"),
|
||||||
|
redactions: [
|
||||||
|
{
|
||||||
|
keyframes: [
|
||||||
|
{ tMs: 0, box: { x: 0.1, y: 0.1, w: 0.3, h: 0.3 } },
|
||||||
|
{ tMs: 20_000, box: { x: 0.5, y: 0.5, w: 0.3, h: 0.3 } },
|
||||||
|
],
|
||||||
|
mode: "blur",
|
||||||
|
paddingPct: 10,
|
||||||
|
featherPct: 0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
setTimeout(() => controller.abort(), 150);
|
||||||
|
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
66
packages/adapters/src/media-emulator-clips.ts
Normal file
66
packages/adapters/src/media-emulator-clips.ts
Normal file
@ -0,0 +1,66 @@
|
|||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import { runMediaTool } from "./media-folder-source-probe.js";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Synthetic test clips (ffmpeg testsrc2 + sine) for media tests across the
|
||||||
|
* codebase. Name the output with media-emulator markers to steer emulator
|
||||||
|
* results, e.g. `clip__faces-1__.mp4`.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export function ffmpegAvailable(ffmpegPath = "ffmpeg"): boolean {
|
||||||
|
return spawnSync(ffmpegPath, ["-version"], { stdio: "ignore" }).status === 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TestClipOptions {
|
||||||
|
seconds?: number;
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
fps?: number;
|
||||||
|
audio?: boolean;
|
||||||
|
ffmpegPath?: string;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function renderTestClip(outPath: string, options: TestClipOptions = {}) {
|
||||||
|
const { seconds = 2, width = 320, height = 180, fps = 25, audio = true } = options;
|
||||||
|
const args = [
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-y",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
`testsrc2=s=${width}x${height}:r=${fps}:d=${seconds}`,
|
||||||
|
];
|
||||||
|
if (audio)
|
||||||
|
args.push("-f", "lavfi", "-i", `sine=f=440:sample_rate=48000:d=${seconds}`, "-shortest");
|
||||||
|
args.push("-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p");
|
||||||
|
if (audio) args.push("-c:a", "aac");
|
||||||
|
args.push(outPath);
|
||||||
|
await runMediaTool(options.ffmpegPath ?? "ffmpeg", args, { signal: options.signal });
|
||||||
|
return outPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Decode one frame at `tMs` as packed RGB24 bytes. */
|
||||||
|
export async function extractRgbFrame(
|
||||||
|
file: string,
|
||||||
|
tMs: number,
|
||||||
|
options: { ffmpegPath?: string } = {},
|
||||||
|
): Promise<Buffer> {
|
||||||
|
const { stdout } = await runMediaTool(options.ffmpegPath ?? "ffmpeg", [
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-i",
|
||||||
|
file,
|
||||||
|
"-ss",
|
||||||
|
(tMs / 1000).toFixed(3),
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-f",
|
||||||
|
"rawvideo",
|
||||||
|
"-pix_fmt",
|
||||||
|
"rgb24",
|
||||||
|
"-",
|
||||||
|
]);
|
||||||
|
return stdout;
|
||||||
|
}
|
||||||
208
packages/adapters/src/media-emulator-source.ts
Normal file
208
packages/adapters/src/media-emulator-source.ts
Normal file
@ -0,0 +1,208 @@
|
|||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { mkdir, rename, rm, stat } from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import type {
|
||||||
|
MediaSourceItem,
|
||||||
|
MediaSourceListRequest,
|
||||||
|
MediaSourceListResult,
|
||||||
|
MediaSourceProvider,
|
||||||
|
} from "@rakazo/adapter-kit";
|
||||||
|
import { PathOutsideRootError, resolveInside } from "@rakazo/media/node";
|
||||||
|
import { z } from "zod";
|
||||||
|
import type { MediaToolPaths } from "./media-folder-source-probe.js";
|
||||||
|
import { runMediaTool } from "./media-folder-source-probe.js";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Deterministic synthetic library for tests and the scripted runtime. Config
|
||||||
|
* `{ count, seed }` describes `count` assets split into three capture sessions
|
||||||
|
* on separate days in three cities (more than 2 km apart), alternating two
|
||||||
|
* visual themes (`__theme-a__` testsrc2, `__theme-b__` SMPTE bars). Every fifth
|
||||||
|
* asset is a photo. File names carry media-emulator markers:
|
||||||
|
* index % 10 == 0 __faces-1____person-a__
|
||||||
|
* index % 10 == 1 __faces-1____person-b__
|
||||||
|
* index == 2 __nsfw-explicit_nudity-0.9__
|
||||||
|
* index % 4 == 3 __speech__ (videos only)
|
||||||
|
* index % 10 == 6 __cut-2000__ (videos only)
|
||||||
|
* `resolve` renders the clip on first use (4 s, 320x180, ffmpeg testsrc2 or
|
||||||
|
* smptehdbars plus a sine tone) and caches it under `fixturesDir/<seed>/`.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const ConfigSchema = z.object({
|
||||||
|
count: z.number().int().min(1).max(100_000),
|
||||||
|
seed: z.number().int().min(0).max(2_147_483_647),
|
||||||
|
});
|
||||||
|
|
||||||
|
const CLIP_MS = 4000;
|
||||||
|
const CLIP_SIZE = { width: 320, height: 180, fps: 25 };
|
||||||
|
const SESSIONS = [
|
||||||
|
{ lat: 52.52, lon: 13.405 },
|
||||||
|
{ lat: 53.5511, lon: 9.9937 },
|
||||||
|
{ lat: 48.1374, lon: 11.5755 },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export interface EmulatorAsset {
|
||||||
|
index: number;
|
||||||
|
fileName: string;
|
||||||
|
kind: "video" | "photo";
|
||||||
|
theme: "a" | "b";
|
||||||
|
session: number;
|
||||||
|
capturedAt: Date;
|
||||||
|
gpsLat: number;
|
||||||
|
gpsLon: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The deterministic description of asset `index` in an emulator library. */
|
||||||
|
export function emulatorAsset(seed: number, count: number, index: number): EmulatorAsset {
|
||||||
|
const kind = index % 5 === 4 ? "photo" : "video";
|
||||||
|
const theme = index % 2 === 0 ? "a" : "b";
|
||||||
|
const session = Math.min(SESSIONS.length - 1, Math.floor((index * SESSIONS.length) / count));
|
||||||
|
const firstInSession = Math.ceil((session * count) / SESSIONS.length);
|
||||||
|
const day = Date.UTC(2026, 4, 2 + (seed % 28) + session * 2, 8 + session);
|
||||||
|
const capturedAt = new Date(day + (index - firstInSession) * 3 * 60_000);
|
||||||
|
const place = SESSIONS[session] ?? SESSIONS[0];
|
||||||
|
const jitter = ((index * 7919 + seed) % 100) / 100_000;
|
||||||
|
const markers = [`__theme-${theme}__`];
|
||||||
|
if (index % 10 === 0) markers.push("__faces-1____person-a__");
|
||||||
|
if (index % 10 === 1) markers.push("__faces-1____person-b__");
|
||||||
|
if (index === 2) markers.push("__nsfw-explicit_nudity-0.9__");
|
||||||
|
if (kind === "video" && index % 4 === 3) markers.push("__speech__");
|
||||||
|
if (kind === "video" && index % 10 === 6) markers.push("__cut-2000__");
|
||||||
|
const ext = kind === "photo" ? "jpg" : "mp4";
|
||||||
|
return {
|
||||||
|
index,
|
||||||
|
fileName: `emu-${seed}-${String(index).padStart(5, "0")}${markers.join("")}.${ext}`,
|
||||||
|
kind,
|
||||||
|
theme,
|
||||||
|
session,
|
||||||
|
capturedAt,
|
||||||
|
gpsLat: Math.round((place.lat + jitter) * 1e5) / 1e5,
|
||||||
|
gpsLon: Math.round((place.lon + jitter) * 1e5) / 1e5,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function iso6709(lat: number, lon: number): string {
|
||||||
|
const sign = (value: number, width: number) =>
|
||||||
|
`${value < 0 ? "-" : "+"}${Math.abs(value).toFixed(4).padStart(width, "0")}`;
|
||||||
|
return `${sign(lat, 7)}${sign(lon, 8)}/`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EmulatorMediaSourceOptions extends MediaToolPaths {
|
||||||
|
/** Cache for rendered clips; must lie inside the media root to be analysable. */
|
||||||
|
fixturesDir: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class EmulatorMediaSourceProvider implements MediaSourceProvider {
|
||||||
|
readonly kind = "emulator" as const;
|
||||||
|
|
||||||
|
constructor(private readonly options: EmulatorMediaSourceOptions) {}
|
||||||
|
|
||||||
|
async list(request: MediaSourceListRequest): Promise<MediaSourceListResult> {
|
||||||
|
request.signal?.throwIfAborted();
|
||||||
|
const { count, seed } = ConfigSchema.parse(request.config);
|
||||||
|
const start = request.cursor ? decodeIndex(request.cursor) : 0;
|
||||||
|
const end = Math.min(count, start + Math.max(1, Math.floor(request.limit)));
|
||||||
|
const items: MediaSourceItem[] = [];
|
||||||
|
for (let index = start; index < end; index++) {
|
||||||
|
const asset = emulatorAsset(seed, count, index);
|
||||||
|
items.push({
|
||||||
|
externalId: asset.fileName,
|
||||||
|
kind: asset.kind,
|
||||||
|
capturedAt: asset.capturedAt,
|
||||||
|
durationMs: asset.kind === "video" ? CLIP_MS : null,
|
||||||
|
width: CLIP_SIZE.width,
|
||||||
|
height: CLIP_SIZE.height,
|
||||||
|
fps: asset.kind === "video" ? CLIP_SIZE.fps : null,
|
||||||
|
gpsLat: asset.gpsLat,
|
||||||
|
gpsLon: asset.gpsLon,
|
||||||
|
deviceModel: "Emulator",
|
||||||
|
albumNames: [],
|
||||||
|
isFavorite: index % 9 === 0,
|
||||||
|
fileName: asset.fileName,
|
||||||
|
sizeBytes: null,
|
||||||
|
revision: `emulator-v1-${seed}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
items,
|
||||||
|
nextCursor: end < count ? Buffer.from(String(end)).toString("base64url") : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async resolve(config: Record<string, unknown>, externalId: string): Promise<string> {
|
||||||
|
const { count, seed } = ConfigSchema.parse(config);
|
||||||
|
const match = /^emu-(\d+)-(\d{5})/.exec(externalId);
|
||||||
|
const index = match ? Number(match[2]) : -1;
|
||||||
|
if (!match || Number(match[1]) !== seed || index >= count)
|
||||||
|
throw new PathOutsideRootError(externalId);
|
||||||
|
const asset = emulatorAsset(seed, count, index);
|
||||||
|
if (asset.fileName !== externalId) throw new PathOutsideRootError(externalId);
|
||||||
|
await mkdir(path.join(this.options.fixturesDir, String(seed)), { recursive: true });
|
||||||
|
const target = await resolveInside(this.options.fixturesDir, `${seed}/${asset.fileName}`);
|
||||||
|
if (await exists(target)) return target;
|
||||||
|
const partial = path.join(
|
||||||
|
path.dirname(target),
|
||||||
|
`.partial-${randomUUID()}.${asset.kind === "photo" ? "jpg" : "mp4"}`,
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await runMediaTool(this.options.ffmpegPath ?? "ffmpeg", renderArgs(asset, partial));
|
||||||
|
await rename(partial, target);
|
||||||
|
} finally {
|
||||||
|
await rm(partial, { force: true });
|
||||||
|
}
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderArgs(asset: EmulatorAsset, out: string): string[] {
|
||||||
|
const { width, height, fps } = CLIP_SIZE;
|
||||||
|
const pattern = asset.theme === "a" ? "testsrc2" : "smptehdbars";
|
||||||
|
const video = `${pattern}=s=${width}x${height}:r=${fps}:d=${CLIP_MS / 1000}`;
|
||||||
|
const metadata = [
|
||||||
|
"-metadata",
|
||||||
|
`creation_time=${asset.capturedAt.toISOString()}`,
|
||||||
|
"-metadata",
|
||||||
|
`location=${iso6709(asset.gpsLat, asset.gpsLon)}`,
|
||||||
|
];
|
||||||
|
if (asset.kind === "photo")
|
||||||
|
return ["-v", "error", "-y", "-f", "lavfi", "-i", video, "-frames:v", "1", out];
|
||||||
|
return [
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-y",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
video,
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
`sine=f=${220 + (asset.index % 12) * 40}:sample_rate=48000:d=${CLIP_MS / 1000}`,
|
||||||
|
"-shortest",
|
||||||
|
"-c:v",
|
||||||
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"ultrafast",
|
||||||
|
"-pix_fmt",
|
||||||
|
"yuv420p",
|
||||||
|
"-c:a",
|
||||||
|
"aac",
|
||||||
|
...metadata,
|
||||||
|
"-movflags",
|
||||||
|
"+faststart",
|
||||||
|
out,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
function decodeIndex(cursor: string): number {
|
||||||
|
const value = Number(Buffer.from(cursor, "base64url").toString("utf8"));
|
||||||
|
if (!Number.isInteger(value) || value < 0) throw new Error("invalid emulator cursor");
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function exists(file: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
return (await stat(file)).isFile();
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
335
packages/adapters/src/media-emulator.test.ts
Normal file
335
packages/adapters/src/media-emulator.test.ts
Normal file
@ -0,0 +1,335 @@
|
|||||||
|
import { copyFile, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||||
|
import { cosineSimilarity, haversineKm } from "@rakazo/media";
|
||||||
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
buildRedactionGraph,
|
||||||
|
EmulatorMediaInferenceProvider,
|
||||||
|
parseEmulatorMarkers,
|
||||||
|
redactionRegions,
|
||||||
|
} from "./media-emulator.js";
|
||||||
|
import { ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js";
|
||||||
|
import { EmulatorMediaSourceProvider, emulatorAsset } from "./media-emulator-source.js";
|
||||||
|
import { probeMedia } from "./media-folder-source-probe.js";
|
||||||
|
|
||||||
|
const hasFfmpeg = ffmpegAvailable();
|
||||||
|
let root: string;
|
||||||
|
let provider: EmulatorMediaInferenceProvider;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
root = await mkdtemp(path.join(tmpdir(), "media-emulator-"));
|
||||||
|
provider = new EmulatorMediaInferenceProvider({ root });
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await rm(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
/** A non-media file: the emulator falls back to a 10 s clip, so no ffmpeg is needed. */
|
||||||
|
async function fixture(name: string, content = "synthetic bytes") {
|
||||||
|
const file = path.join(root, name);
|
||||||
|
await writeFile(file, content);
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("parseEmulatorMarkers", () => {
|
||||||
|
it("reads every documented marker", () => {
|
||||||
|
expect(
|
||||||
|
parseEmulatorMarkers(
|
||||||
|
"/x/clip__faces-2____person-a____noface-gap-1000-2500____nsfw-explicit_nudity-0.9____nsfw-violence-0.4____cut-1500____cut-3000____speech____theme-b____inpaintq-0.3__.mp4",
|
||||||
|
),
|
||||||
|
).toEqual({
|
||||||
|
faces: 2,
|
||||||
|
person: "a",
|
||||||
|
noFaceGaps: [{ startMs: 1000, endMs: 2500 }],
|
||||||
|
nsfw: { explicit_nudity: 0.9, violence: 0.4 },
|
||||||
|
cutsMs: [1500, 3000],
|
||||||
|
speech: true,
|
||||||
|
theme: "b",
|
||||||
|
inpaintQuality: 0.3,
|
||||||
|
});
|
||||||
|
expect(parseEmulatorMarkers("__nsfw-unknown-0.5____nsfw-violence-7__.mp4").nsfw).toEqual({});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("redaction regions", () => {
|
||||||
|
it("holds the union of neighbouring keyframes and bounds the graph", () => {
|
||||||
|
const regions = redactionRegions(
|
||||||
|
{
|
||||||
|
keyframes: [
|
||||||
|
{ tMs: 1000, box: { x: 0.5, y: 0.5, w: 0.1, h: 0.1 } },
|
||||||
|
{ tMs: 0, box: { x: 0.1, y: 0.1, w: 0.1, h: 0.1 } },
|
||||||
|
],
|
||||||
|
mode: "blur",
|
||||||
|
paddingPct: 0,
|
||||||
|
featherPct: 0,
|
||||||
|
},
|
||||||
|
1000,
|
||||||
|
1000,
|
||||||
|
);
|
||||||
|
expect(regions).toEqual([
|
||||||
|
{ startMs: 0, endMs: 1000, box: { x: 100, y: 100, w: 500, h: 500 } },
|
||||||
|
{ startMs: 1000, endMs: 1100, box: { x: 500, y: 500, w: 100, h: 100 } },
|
||||||
|
]);
|
||||||
|
const keyframes = Array.from({ length: 5000 }, (_, i) => ({
|
||||||
|
tMs: i * 40,
|
||||||
|
box: { x: (i % 50) / 100, y: 0.2, w: 0.1, h: 0.1 },
|
||||||
|
}));
|
||||||
|
const many = redactionRegions(
|
||||||
|
{ keyframes, mode: "pixelate", paddingPct: 25, featherPct: 0 },
|
||||||
|
1920,
|
||||||
|
1080,
|
||||||
|
);
|
||||||
|
expect(many.length).toBeLessThanOrEqual(400);
|
||||||
|
expect(many[0]?.startMs).toBe(0);
|
||||||
|
expect(many.at(-1)?.endMs).toBe(4999 * 40 + 100);
|
||||||
|
const graph = buildRedactionGraph(
|
||||||
|
[{ keyframes, mode: "solid", paddingPct: 0, featherPct: 0, color: "#ff0000" }],
|
||||||
|
1920,
|
||||||
|
1080,
|
||||||
|
);
|
||||||
|
expect(graph.filter).toContain("color=0xff0000@1:t=fill");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("EmulatorMediaInferenceProvider", () => {
|
||||||
|
it("is quiet without markers", async () => {
|
||||||
|
const file = await fixture("plain.mp4");
|
||||||
|
const scenes = await provider.detectScenes({ path: file });
|
||||||
|
expect(scenes.scenes).toEqual([{ startMs: 0, endMs: 10_000 }]);
|
||||||
|
expect(scenes.shots).toHaveLength(1);
|
||||||
|
expect((await provider.trackFaces({ path: file, sampleFps: 2 })).tracks).toEqual([]);
|
||||||
|
const faces = await provider.detectFaces({ path: file, sampleFps: 1 });
|
||||||
|
expect(faces.frames).toHaveLength(10);
|
||||||
|
expect(faces.frames.every((frame) => frame.faces.length === 0)).toBe(true);
|
||||||
|
const classified = await provider.classifyContent({ path: file, sampleFps: 2 });
|
||||||
|
expect(classified.frames).toHaveLength(20);
|
||||||
|
for (const frame of classified.frames)
|
||||||
|
expect(Object.values(frame.scores).every((score) => score === 0.01)).toBe(true);
|
||||||
|
expect(await provider.transcribe({ path: file })).toEqual({ language: null, words: [] });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forces faces, gaps, cuts, classifier scores and speech from the file name", async () => {
|
||||||
|
const file = await fixture(
|
||||||
|
"rich__faces-2____noface-gap-2000-3000____cut-4000____nsfw-explicit_nudity-0.9____speech__.mp4",
|
||||||
|
);
|
||||||
|
const tracks = await provider.trackFaces({ path: file, sampleFps: 2 });
|
||||||
|
expect(tracks.tracks.map((track) => track.trackId)).toEqual(["track-1", "track-2"]);
|
||||||
|
const times = tracks.tracks[0]?.keyframes.map((keyframe) => keyframe.tMs) ?? [];
|
||||||
|
expect(times).toContain(1500);
|
||||||
|
expect(times).not.toContain(2000);
|
||||||
|
expect(times).not.toContain(2500);
|
||||||
|
expect(times).toContain(3000);
|
||||||
|
const gapFrame = await provider.detectFaces({
|
||||||
|
path: file,
|
||||||
|
sampleFps: 1,
|
||||||
|
timesMs: [2500, 3500],
|
||||||
|
});
|
||||||
|
expect(gapFrame.frames.map((frame) => frame.faces.length)).toEqual([0, 2]);
|
||||||
|
|
||||||
|
expect((await provider.detectScenes({ path: file })).scenes).toEqual([
|
||||||
|
{ startMs: 0, endMs: 4000 },
|
||||||
|
{ startMs: 4000, endMs: 10_000 },
|
||||||
|
]);
|
||||||
|
const classified = await provider.classifyContent({
|
||||||
|
path: file,
|
||||||
|
sampleFps: 0.5,
|
||||||
|
extraTimesMs: [4000, 4001],
|
||||||
|
});
|
||||||
|
expect(classified.frames.map((frame) => frame.tMs)).toEqual([0, 2000, 4000, 4001, 6000, 8000]);
|
||||||
|
for (const frame of classified.frames) {
|
||||||
|
expect(frame.scores.explicit_nudity).toBe(0.9);
|
||||||
|
expect(frame.scores.violence).toBe(0.01);
|
||||||
|
}
|
||||||
|
const english = await provider.transcribe({ path: file });
|
||||||
|
const german = await provider.transcribe({ path: file, language: "de-DE" });
|
||||||
|
expect(english.language).toBe("en");
|
||||||
|
expect(english.words.map((word) => word.text).slice(0, 2)).toEqual(["welcome", "back"]);
|
||||||
|
expect(german.language).toBe("de");
|
||||||
|
expect(german.words[0]).toEqual({
|
||||||
|
startMs: 400,
|
||||||
|
endMs: 720,
|
||||||
|
text: "willkommen",
|
||||||
|
confidence: 0.95,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is deterministic per file content and keeps boxes stable", async () => {
|
||||||
|
const file = await fixture("same__faces-1__.mp4");
|
||||||
|
const copy = path.join(root, "copy__faces-1__.mp4");
|
||||||
|
await copyFile(file, copy);
|
||||||
|
const a = await provider.detectFaces({ path: file, sampleFps: 4 });
|
||||||
|
const b = await provider.detectFaces({ path: file, sampleFps: 4 });
|
||||||
|
const c = await provider.detectFaces({ path: copy, sampleFps: 4 });
|
||||||
|
expect(a).toEqual(b);
|
||||||
|
expect(a).toEqual(c);
|
||||||
|
const boxes = a.frames.map((frame) => frame.faces[0]?.box);
|
||||||
|
expect(Math.abs((boxes.at(-1)?.x ?? 0) - (boxes[0]?.x ?? 0))).toBeLessThan(0.05);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("embeds people and themes consistently", async () => {
|
||||||
|
const personA1 = await fixture("one__faces-1____person-a__.mp4", "first");
|
||||||
|
const personA2 = await fixture("two__faces-1____person-a__.mp4", "second");
|
||||||
|
const personB = await fixture("three__faces-1____person-b__.mp4", "third");
|
||||||
|
const embed = async (file: string) =>
|
||||||
|
(await provider.trackFaces({ path: file, sampleFps: 1 })).tracks[0]?.embedding ?? [];
|
||||||
|
expect(cosineSimilarity(await embed(personA1), await embed(personA2))).toBeCloseTo(1, 5);
|
||||||
|
expect(cosineSimilarity(await embed(personA1), await embed(personB))).toBeLessThan(0.8);
|
||||||
|
|
||||||
|
const themeA1 = await fixture("x__theme-a__.mp4", "x");
|
||||||
|
const themeA2 = await fixture("y__theme-a__.mp4", "y");
|
||||||
|
const themeB = await fixture("z__theme-b__.mp4", "z");
|
||||||
|
const mean = async (file: string) =>
|
||||||
|
(await provider.embedFrames({ path: file, timesMs: [0, 1000] })).vectors[0] ?? [];
|
||||||
|
expect(cosineSimilarity(await mean(themeA1), await mean(themeA2))).toBeGreaterThan(0.95);
|
||||||
|
expect(cosineSimilarity(await mean(themeA1), await mean(themeB))).toBeLessThan(0.85);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never synthesizes identities", async () => {
|
||||||
|
const file = await fixture("synth.mp4");
|
||||||
|
const capabilities = await provider.capabilities();
|
||||||
|
expect(capabilities.operations.synthesizeIdentity).toMatchObject({
|
||||||
|
available: false,
|
||||||
|
license: null,
|
||||||
|
});
|
||||||
|
expect(capabilities.operations.inpaintVideo).toMatchObject({
|
||||||
|
available: true,
|
||||||
|
experimental: true,
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
provider.synthesizeIdentity({
|
||||||
|
path: file,
|
||||||
|
track: [],
|
||||||
|
seed: 1,
|
||||||
|
outPath: path.join(root, "s.mp4"),
|
||||||
|
}),
|
||||||
|
).rejects.toBeInstanceOf(UnsupportedMediaOperationError);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects paths outside its root", async () => {
|
||||||
|
await expect(provider.detectScenes({ path: "/etc/hostname" })).rejects.toThrow(
|
||||||
|
"outside the allowed root",
|
||||||
|
);
|
||||||
|
await expect(provider.detectScenes({ path: "relative.mp4" })).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.skipIf(!hasFfmpeg)("writes masks and inpaints with the reported quality", async () => {
|
||||||
|
const clip = await renderTestClip(path.join(root, "mark__inpaintq-0.4__.mp4"), {
|
||||||
|
seconds: 1,
|
||||||
|
audio: false,
|
||||||
|
});
|
||||||
|
const masks = await provider.segmentPrompted({
|
||||||
|
path: clip,
|
||||||
|
prompt: { tMs: 0, point: { x: 0.5, y: 0.5 } },
|
||||||
|
propagate: true,
|
||||||
|
outDir: path.join(root, "marks/m1"),
|
||||||
|
});
|
||||||
|
expect((await readdir(masks.maskDir)).sort()).toEqual(["0.png", "500.png"]);
|
||||||
|
expect(masks.keyframes[0]?.box).toEqual({ x: 0.44, y: 0.44, w: 0.12, h: 0.12 });
|
||||||
|
const inpainted = await provider.inpaintVideo({
|
||||||
|
path: clip,
|
||||||
|
maskDir: masks.maskDir,
|
||||||
|
outPath: path.join(root, "marks/m1.mp4"),
|
||||||
|
});
|
||||||
|
expect(inpainted.quality.score).toBe(0.4);
|
||||||
|
expect((await probeMedia(inpainted.outPath)).durationMs).toBeGreaterThan(900);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.skipIf(!hasFfmpeg)("renders very large redaction graphs through a script file", async () => {
|
||||||
|
const clip = await renderTestClip(path.join(root, "busy.mp4"), { seconds: 1, audio: false });
|
||||||
|
const instruction = (offset: number) => ({
|
||||||
|
keyframes: Array.from({ length: 2000 }, (_, i) => ({
|
||||||
|
tMs: i,
|
||||||
|
box: { x: offset + (i % 40) / 400, y: 0.1, w: 0.1, h: 0.1 },
|
||||||
|
})),
|
||||||
|
mode: "blur" as const,
|
||||||
|
paddingPct: 0,
|
||||||
|
featherPct: 0,
|
||||||
|
});
|
||||||
|
const instructions = [instruction(0), instruction(0.3), instruction(0.6)];
|
||||||
|
expect(buildRedactionGraph(instructions, 320, 180).filter.length).toBeGreaterThan(100_000);
|
||||||
|
const result = await provider.redact({
|
||||||
|
path: clip,
|
||||||
|
redactions: instructions,
|
||||||
|
outPath: path.join(root, "busy-redacted.mp4"),
|
||||||
|
});
|
||||||
|
expect(result.framesProcessed).toBe(25);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.skipIf(!hasFfmpeg)("measures loudness with ebur128", async () => {
|
||||||
|
const clip = await renderTestClip(path.join(root, "tone.mp4"), { seconds: 2 });
|
||||||
|
const features = await provider.audioFeatures({ path: clip });
|
||||||
|
expect(features.integratedLufs).toBeLessThan(-10);
|
||||||
|
expect(features.integratedLufs).toBeGreaterThan(-40);
|
||||||
|
expect(features.energy.length).toBeGreaterThan(0);
|
||||||
|
expect(features.beatsMs.length).toBeGreaterThan(0);
|
||||||
|
const silent = await renderTestClip(path.join(root, "mute.mp4"), { seconds: 1, audio: false });
|
||||||
|
expect(await provider.audioFeatures({ path: silent })).toMatchObject({
|
||||||
|
integratedLufs: null,
|
||||||
|
silences: [{ startMs: 0, endMs: 1000 }],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("EmulatorMediaSourceProvider", () => {
|
||||||
|
const config = { count: 30, seed: 4 };
|
||||||
|
|
||||||
|
it("describes three sessions, two themes and the marker assets", async () => {
|
||||||
|
const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") });
|
||||||
|
const first = await source.list({ config, cursor: null, limit: 12 });
|
||||||
|
const second = await source.list({ config, cursor: first.nextCursor, limit: 100 });
|
||||||
|
expect(second.nextCursor).toBeNull();
|
||||||
|
const items = [...first.items, ...second.items];
|
||||||
|
expect(items).toHaveLength(30);
|
||||||
|
expect(new Set(items.map((item) => item.externalId)).size).toBe(30);
|
||||||
|
expect(await source.list({ config, cursor: null, limit: 12 })).toEqual(first);
|
||||||
|
|
||||||
|
const assets = items.map((_, index) => emulatorAsset(config.seed, config.count, index));
|
||||||
|
expect(new Set(assets.map((asset) => asset.session))).toEqual(new Set([0, 1, 2]));
|
||||||
|
expect(new Set(assets.map((asset) => asset.theme))).toEqual(new Set(["a", "b"]));
|
||||||
|
const sessionStarts = [0, 1, 2].map(
|
||||||
|
(session) => assets.find((asset) => asset.session === session)!,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
haversineKm(
|
||||||
|
{ lat: sessionStarts[0]!.gpsLat, lon: sessionStarts[0]!.gpsLon },
|
||||||
|
{ lat: sessionStarts[1]!.gpsLat, lon: sessionStarts[1]!.gpsLon },
|
||||||
|
),
|
||||||
|
).toBeGreaterThan(2);
|
||||||
|
const names = items.map((item) => item.externalId).join(" ");
|
||||||
|
expect(names).toContain("__faces-1____person-a__");
|
||||||
|
expect(names).toContain("__faces-1____person-b__");
|
||||||
|
expect(names).toContain("__nsfw-explicit_nudity-0.9__");
|
||||||
|
expect(names).toContain("__speech__");
|
||||||
|
expect(items.some((item) => item.kind === "photo")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects unknown external ids", async () => {
|
||||||
|
const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") });
|
||||||
|
for (const bad of [
|
||||||
|
"../etc/passwd",
|
||||||
|
"emu-4-00099__theme-b__.mp4",
|
||||||
|
"emu-5-00000__theme-a__.mp4",
|
||||||
|
"emu-4-00000.mp4",
|
||||||
|
])
|
||||||
|
await expect(source.resolve(config, bad), bad).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.skipIf(!hasFfmpeg)("renders and caches clips on demand", async () => {
|
||||||
|
const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") });
|
||||||
|
const asset = emulatorAsset(config.seed, config.count, 0);
|
||||||
|
const file = await source.resolve(config, asset.fileName);
|
||||||
|
expect(path.basename(file)).toBe(asset.fileName);
|
||||||
|
const probed = await probeMedia(file);
|
||||||
|
expect(probed).toMatchObject({ durationMs: 4000, width: 320, height: 180, hasAudio: true });
|
||||||
|
expect(probed.capturedAt?.toISOString()).toBe(asset.capturedAt.toISOString());
|
||||||
|
expect(await source.resolve(config, asset.fileName)).toBe(file);
|
||||||
|
const tracks = await provider.trackFaces({ path: file, sampleFps: 1 });
|
||||||
|
expect(tracks.tracks).toHaveLength(1);
|
||||||
|
const photo = emulatorAsset(config.seed, config.count, 4);
|
||||||
|
expect(photo.kind).toBe("photo");
|
||||||
|
expect((await probeMedia(await source.resolve(config, photo.fileName))).width).toBe(320);
|
||||||
|
});
|
||||||
|
});
|
||||||
1003
packages/adapters/src/media-emulator.ts
Normal file
1003
packages/adapters/src/media-emulator.ts
Normal file
File diff suppressed because it is too large
Load Diff
53
packages/adapters/src/media-factory.test.ts
Normal file
53
packages/adapters/src/media-factory.test.ts
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
import { homedir } from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
createMediaInferenceProvider,
|
||||||
|
createMediaSourceProvider,
|
||||||
|
mediaConfigFromEnv,
|
||||||
|
resolveMediaInferenceKind,
|
||||||
|
} from "./media-factory.js";
|
||||||
|
|
||||||
|
const worker = {
|
||||||
|
MEDIA_WORKER_URL: "http://127.0.0.1:8790",
|
||||||
|
MEDIA_WORKER_TOKEN: "placeholder-token-placeholder-token-00",
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("media factory", () => {
|
||||||
|
it("reads one generic worker connection and derives paths", () => {
|
||||||
|
expect(mediaConfigFromEnv({ DATA_DIR: "/srv/evoo", ...worker })).toEqual({
|
||||||
|
enabled: true,
|
||||||
|
workerUrl: worker.MEDIA_WORKER_URL,
|
||||||
|
workerToken: worker.MEDIA_WORKER_TOKEN,
|
||||||
|
profilesDir: path.join(homedir(), ".evoo/media-profiles"),
|
||||||
|
mediaRoot: "/srv/evoo/media",
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
mediaConfigFromEnv({ DATA_DIR: "/srv/evoo", MEDIA_PROFILES_DIR: "/etc/evoo/profiles" }),
|
||||||
|
).toMatchObject({ enabled: false, workerUrl: null, profilesDir: "/etc/evoo/profiles" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("selects the sidecar only with URL and token", () => {
|
||||||
|
expect(resolveMediaInferenceKind(worker)).toBe("sidecar");
|
||||||
|
expect(createMediaInferenceProvider(worker)?.id).toBe("sidecar");
|
||||||
|
expect(resolveMediaInferenceKind({ MEDIA_WORKER_URL: worker.MEDIA_WORKER_URL })).toBeNull();
|
||||||
|
expect(createMediaInferenceProvider({})).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("pins the emulator under the scripted runtime", () => {
|
||||||
|
expect(resolveMediaInferenceKind({ ...worker, AGENT_RUNTIME: "scripted" })).toBe("emulator");
|
||||||
|
expect(createMediaInferenceProvider({ AGENT_RUNTIME: "scripted" })?.id).toBe("emulator");
|
||||||
|
expect(mediaConfigFromEnv({ AGENT_RUNTIME: "scripted" }).enabled).toBe(true);
|
||||||
|
// pin-test-env sets AGENT_RUNTIME=scripted for every unit test.
|
||||||
|
expect(createMediaInferenceProvider()?.id).toBe("emulator");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("creates server-side sources only", () => {
|
||||||
|
expect(createMediaSourceProvider("folder", {}).kind).toBe("folder");
|
||||||
|
expect(createMediaSourceProvider("emulator", { AGENT_RUNTIME: "scripted" }).kind).toBe(
|
||||||
|
"emulator",
|
||||||
|
);
|
||||||
|
expect(() => createMediaSourceProvider("emulator", {})).toThrow();
|
||||||
|
expect(() => createMediaSourceProvider("apple_photos", {})).toThrow();
|
||||||
|
});
|
||||||
|
});
|
||||||
90
packages/adapters/src/media-factory.ts
Normal file
90
packages/adapters/src/media-factory.ts
Normal file
@ -0,0 +1,90 @@
|
|||||||
|
import { homedir } from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { MediaInferenceProvider, MediaSourceProvider } from "@rakazo/adapter-kit";
|
||||||
|
import { expandHome } from "@rakazo/media/node";
|
||||||
|
import { EmulatorMediaInferenceProvider } from "./media-emulator.js";
|
||||||
|
import { EmulatorMediaSourceProvider } from "./media-emulator-source.js";
|
||||||
|
import { FolderMediaSourceProvider } from "./media-folder-source.js";
|
||||||
|
import { SidecarMediaInferenceProvider } from "./media-sidecar.js";
|
||||||
|
|
||||||
|
export interface MediaConfig {
|
||||||
|
/** Media Studio runs: a sidecar is configured or the scripted runtime uses the emulator. */
|
||||||
|
enabled: boolean;
|
||||||
|
workerUrl: string | null;
|
||||||
|
workerToken: string | null;
|
||||||
|
profilesDir: string;
|
||||||
|
/** `${DATA_DIR}/media`, absolute. Every derived file and sidecar path lives here. */
|
||||||
|
mediaRoot: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type MediaInferenceKind = "sidecar" | "emulator";
|
||||||
|
|
||||||
|
/** Emulators replace the sidecar whenever the scripted runtime is pinned (tests, e2e). */
|
||||||
|
export function mediaEmulatorEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||||
|
return env.AGENT_RUNTIME === "scripted";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mediaConfigFromEnv(env: NodeJS.ProcessEnv = process.env): MediaConfig {
|
||||||
|
const workerUrl = env.MEDIA_WORKER_URL?.trim() || null;
|
||||||
|
const workerToken = env.MEDIA_WORKER_TOKEN?.trim() || null;
|
||||||
|
return {
|
||||||
|
enabled: mediaEmulatorEnabled(env) || Boolean(workerUrl && workerToken),
|
||||||
|
workerUrl,
|
||||||
|
workerToken,
|
||||||
|
profilesDir: path.resolve(
|
||||||
|
expandHome(env.MEDIA_PROFILES_DIR?.trim() || "~/.evoo/media-profiles", homedir()),
|
||||||
|
),
|
||||||
|
mediaRoot: path.resolve(env.DATA_DIR ?? "./data", "media"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveMediaInferenceKind(
|
||||||
|
env: NodeJS.ProcessEnv = process.env,
|
||||||
|
): MediaInferenceKind | null {
|
||||||
|
if (mediaEmulatorEnabled(env)) return "emulator";
|
||||||
|
const config = mediaConfigFromEnv(env);
|
||||||
|
return config.workerUrl && config.workerToken ? "sidecar" : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The configured inference provider, or null when Media Studio is not set up. */
|
||||||
|
export function createMediaInferenceProvider(
|
||||||
|
env: NodeJS.ProcessEnv = process.env,
|
||||||
|
): MediaInferenceProvider | null {
|
||||||
|
const config = mediaConfigFromEnv(env);
|
||||||
|
switch (resolveMediaInferenceKind(env)) {
|
||||||
|
case "emulator":
|
||||||
|
return new EmulatorMediaInferenceProvider({ root: config.mediaRoot });
|
||||||
|
case "sidecar":
|
||||||
|
return new SidecarMediaInferenceProvider({
|
||||||
|
baseUrl: config.workerUrl as string,
|
||||||
|
token: config.workerToken as string,
|
||||||
|
});
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Emulator clips are cached inside the media root so inference can read them. */
|
||||||
|
export function emulatorFixturesDir(env: NodeJS.ProcessEnv = process.env): string {
|
||||||
|
return path.join(mediaConfigFromEnv(env).mediaRoot, "emulator-library");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Server-side source adapter for a persisted MediaSource kind. Client-pushed
|
||||||
|
* kinds (apple_photos, ios_library) have no server adapter.
|
||||||
|
*/
|
||||||
|
export function createMediaSourceProvider(
|
||||||
|
kind: string,
|
||||||
|
env: NodeJS.ProcessEnv = process.env,
|
||||||
|
): MediaSourceProvider {
|
||||||
|
switch (kind) {
|
||||||
|
case "folder":
|
||||||
|
return new FolderMediaSourceProvider();
|
||||||
|
case "emulator":
|
||||||
|
if (!mediaEmulatorEnabled(env)) break;
|
||||||
|
return new EmulatorMediaSourceProvider({ fixturesDir: emulatorFixturesDir(env) });
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
throw new Error(`No server-side media source for "${kind}". Use folder.`);
|
||||||
|
}
|
||||||
194
packages/adapters/src/media-folder-source-probe.ts
Normal file
194
packages/adapters/src/media-folder-source-probe.ts
Normal file
@ -0,0 +1,194 @@
|
|||||||
|
import { execFile } from "node:child_process";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Local ffmpeg/ffprobe helpers shared by the folder source and the media
|
||||||
|
* emulator. Every call is an argv array without a shell; nothing here builds a
|
||||||
|
* command string.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export class MediaToolMissingError extends Error {
|
||||||
|
constructor(readonly tool: string) {
|
||||||
|
super(`${tool} is not installed or not on PATH; install ffmpeg to use local media processing`);
|
||||||
|
this.name = "MediaToolMissingError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MediaToolPaths {
|
||||||
|
ffmpegPath?: string;
|
||||||
|
ffprobePath?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Run a media tool and collect stdout (binary) and stderr. Rejects on non-zero exit. */
|
||||||
|
export function runMediaTool(
|
||||||
|
bin: string,
|
||||||
|
args: string[],
|
||||||
|
options: { signal?: AbortSignal; maxBuffer?: number } = {},
|
||||||
|
): Promise<{ stdout: Buffer; stderr: string }> {
|
||||||
|
options.signal?.throwIfAborted();
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
execFile(
|
||||||
|
bin,
|
||||||
|
args,
|
||||||
|
{
|
||||||
|
encoding: "buffer",
|
||||||
|
maxBuffer: options.maxBuffer ?? 64 * 1024 * 1024,
|
||||||
|
signal: options.signal,
|
||||||
|
shell: false,
|
||||||
|
windowsHide: true,
|
||||||
|
},
|
||||||
|
(error, stdout, stderr) => {
|
||||||
|
if (options.signal?.aborted) {
|
||||||
|
reject(options.signal.reason);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||||
|
reject(new MediaToolMissingError(bin));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const tail = stderr.toString("utf8").trim().split("\n").slice(-3).join(" ");
|
||||||
|
reject(new Error(`${bin} failed: ${tail || error.message}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
resolve({ stdout, stderr: stderr.toString("utf8") });
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ProbedMedia {
|
||||||
|
durationMs: number | null;
|
||||||
|
width: number | null;
|
||||||
|
height: number | null;
|
||||||
|
fps: number | null;
|
||||||
|
codec: string | null;
|
||||||
|
rotation: number;
|
||||||
|
hasAudio: boolean;
|
||||||
|
hasVideo: boolean;
|
||||||
|
capturedAt: Date | null;
|
||||||
|
gpsLat: number | null;
|
||||||
|
gpsLon: number | null;
|
||||||
|
deviceModel: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FfprobeStream {
|
||||||
|
codec_type?: string;
|
||||||
|
codec_name?: string;
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
avg_frame_rate?: string;
|
||||||
|
r_frame_rate?: string;
|
||||||
|
duration?: string;
|
||||||
|
tags?: Record<string, string>;
|
||||||
|
side_data_list?: Array<{ rotation?: number }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FfprobeOutput {
|
||||||
|
streams?: FfprobeStream[];
|
||||||
|
format?: { duration?: string; tags?: Record<string, string> };
|
||||||
|
frames?: Array<{ tags?: Record<string, string> }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Probe a local file. Throws when ffprobe is missing or cannot read the file. */
|
||||||
|
export async function probeMedia(
|
||||||
|
file: string,
|
||||||
|
options: { ffprobePath?: string; signal?: AbortSignal; frameTags?: boolean } = {},
|
||||||
|
): Promise<ProbedMedia> {
|
||||||
|
const args = ["-v", "error", "-print_format", "json", "-show_format", "-show_streams"];
|
||||||
|
if (options.frameTags) args.push("-show_frames", "-read_intervals", "%+#1");
|
||||||
|
args.push("--", file);
|
||||||
|
const { stdout } = await runMediaTool(options.ffprobePath ?? "ffprobe", args, {
|
||||||
|
signal: options.signal,
|
||||||
|
});
|
||||||
|
return parseFfprobe(JSON.parse(stdout.toString("utf8")) as FfprobeOutput);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseFfprobe(output: FfprobeOutput): ProbedMedia {
|
||||||
|
const streams = output.streams ?? [];
|
||||||
|
const video = streams.find((stream) => stream.codec_type === "video");
|
||||||
|
const tags = collectTags(output);
|
||||||
|
const rotation = video ? streamRotation(video) : 0;
|
||||||
|
const quarterTurn = Math.abs(rotation) % 180 === 90;
|
||||||
|
const durationSeconds = Number(output.format?.duration ?? video?.duration);
|
||||||
|
const gps = parseIso6709(
|
||||||
|
tags.get("com.apple.quicktime.location.iso6709") ?? tags.get("location"),
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
durationMs:
|
||||||
|
Number.isFinite(durationSeconds) && durationSeconds > 0
|
||||||
|
? Math.round(durationSeconds * 1000)
|
||||||
|
: null,
|
||||||
|
width: (quarterTurn ? video?.height : video?.width) ?? null,
|
||||||
|
height: (quarterTurn ? video?.width : video?.height) ?? null,
|
||||||
|
fps: video ? (parseRate(video.avg_frame_rate) ?? parseRate(video.r_frame_rate)) : null,
|
||||||
|
codec: video?.codec_name ?? null,
|
||||||
|
rotation,
|
||||||
|
hasAudio: streams.some((stream) => stream.codec_type === "audio"),
|
||||||
|
hasVideo: Boolean(video),
|
||||||
|
capturedAt: parseCaptureDate(
|
||||||
|
tags.get("com.apple.quicktime.creationdate") ??
|
||||||
|
tags.get("datetimeoriginal") ??
|
||||||
|
tags.get("creation_time") ??
|
||||||
|
tags.get("datetime"),
|
||||||
|
),
|
||||||
|
gpsLat: gps?.lat ?? null,
|
||||||
|
gpsLon: gps?.lon ?? null,
|
||||||
|
deviceModel: tags.get("com.apple.quicktime.model") ?? tags.get("model") ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Format, stream and first-frame tags merged with lowercase keys; format wins. */
|
||||||
|
function collectTags(output: FfprobeOutput): Map<string, string> {
|
||||||
|
const merged = new Map<string, string>();
|
||||||
|
const sources = [
|
||||||
|
output.format?.tags,
|
||||||
|
...(output.streams ?? []).map((stream) => stream.tags),
|
||||||
|
...(output.frames ?? []).map((frame) => frame.tags),
|
||||||
|
];
|
||||||
|
for (const source of sources) {
|
||||||
|
for (const [key, value] of Object.entries(source ?? {})) {
|
||||||
|
const normalized = key.toLowerCase();
|
||||||
|
if (!merged.has(normalized) && typeof value === "string" && value.trim())
|
||||||
|
merged.set(normalized, value.trim());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return merged;
|
||||||
|
}
|
||||||
|
|
||||||
|
function streamRotation(stream: FfprobeStream): number {
|
||||||
|
const side = stream.side_data_list?.find((entry) => typeof entry.rotation === "number");
|
||||||
|
if (side?.rotation !== undefined) return side.rotation;
|
||||||
|
const tagged = Number(stream.tags?.rotate);
|
||||||
|
return Number.isFinite(tagged) ? tagged : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseRate(rate: string | undefined): number | null {
|
||||||
|
if (!rate) return null;
|
||||||
|
const [num, den] = rate.split("/").map(Number);
|
||||||
|
if (!num || !Number.isFinite(num)) return null;
|
||||||
|
const value = den ? num / den : num;
|
||||||
|
return Number.isFinite(value) && value > 0 ? Math.round(value * 1000) / 1000 : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ISO 8601 or EXIF ("2026:05:01 09:00:00") timestamps; null when unparseable. */
|
||||||
|
export function parseCaptureDate(value: string | undefined): Date | null {
|
||||||
|
if (!value) return null;
|
||||||
|
const exif = /^(\d{4}):(\d{2}):(\d{2})[ T](\d{2}):(\d{2}):(\d{2})/.exec(value);
|
||||||
|
const date = exif
|
||||||
|
? new Date(`${exif[1]}-${exif[2]}-${exif[3]}T${exif[4]}:${exif[5]}:${exif[6]}Z`)
|
||||||
|
: new Date(value);
|
||||||
|
const time = date.getTime();
|
||||||
|
return Number.isFinite(time) && time > 0 ? date : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "+52.5200+013.4050+034.000/" → { lat, lon }; null when absent or out of range. */
|
||||||
|
export function parseIso6709(value: string | undefined): { lat: number; lon: number } | null {
|
||||||
|
if (!value) return null;
|
||||||
|
const match = /^([+-]\d+(?:\.\d+)?)([+-]\d+(?:\.\d+)?)/.exec(value.trim());
|
||||||
|
if (!match) return null;
|
||||||
|
const lat = Number(match[1]);
|
||||||
|
const lon = Number(match[2]);
|
||||||
|
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return null;
|
||||||
|
if (Math.abs(lat) > 90 || Math.abs(lon) > 180) return null;
|
||||||
|
return { lat, lon };
|
||||||
|
}
|
||||||
177
packages/adapters/src/media-folder-source.test.ts
Normal file
177
packages/adapters/src/media-folder-source.test.ts
Normal file
@ -0,0 +1,177 @@
|
|||||||
|
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { MediaSourceItem } from "@rakazo/adapter-kit";
|
||||||
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||||
|
import { ffmpegAvailable } from "./media-emulator-clips.js";
|
||||||
|
import { FolderMediaSourceProvider } from "./media-folder-source.js";
|
||||||
|
import { parseIso6709, runMediaTool } from "./media-folder-source-probe.js";
|
||||||
|
|
||||||
|
let root: string;
|
||||||
|
const provider = new FolderMediaSourceProvider();
|
||||||
|
|
||||||
|
async function touch(relative: string, content = "") {
|
||||||
|
const file = path.join(root, relative);
|
||||||
|
await mkdir(path.dirname(file), { recursive: true });
|
||||||
|
await writeFile(file, content);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function listAll(limit: number, cursor: string | null = null) {
|
||||||
|
const pages: MediaSourceItem[][] = [];
|
||||||
|
let next = cursor;
|
||||||
|
do {
|
||||||
|
const page = await provider.list({ config: { path: root }, cursor: next, limit });
|
||||||
|
expect(page.items.length).toBeLessThanOrEqual(limit);
|
||||||
|
pages.push(page.items);
|
||||||
|
next = page.nextCursor;
|
||||||
|
} while (next);
|
||||||
|
return pages.flat().map((item) => item.externalId);
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
root = await mkdtemp(path.join(tmpdir(), "media-folder-"));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await rm(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("FolderMediaSourceProvider", () => {
|
||||||
|
it("lists media in stable code-unit order across pages", async () => {
|
||||||
|
for (const name of ["b/c.mp4", "b-d.jpg", "a.MOV", "b/a/z.webm", "c.png", "notes.txt"])
|
||||||
|
await touch(name);
|
||||||
|
const ids = await listAll(2);
|
||||||
|
expect(ids).toEqual(["a.MOV", "b-d.jpg", "b/a/z.webm", "b/c.mp4", "c.png"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the cursor stable when files are added mid-sync", async () => {
|
||||||
|
for (const name of ["a.mp4", "c.mp4", "e.mp4", "g.mp4"]) await touch(name);
|
||||||
|
const first = await provider.list({ config: { path: root }, cursor: null, limit: 2 });
|
||||||
|
expect(first.items.map((item) => item.externalId)).toEqual(["a.mp4", "c.mp4"]);
|
||||||
|
await touch("b.mp4"); // before the cursor: picked up by the next full sync
|
||||||
|
await touch("d.mp4"); // after the cursor: part of this sync
|
||||||
|
const rest = await listAll(2, first.nextCursor);
|
||||||
|
expect(rest).toEqual(["d.mp4", "e.mp4", "g.mp4"]);
|
||||||
|
expect(await listAll(10)).toEqual(["a.mp4", "b.mp4", "c.mp4", "d.mp4", "e.mp4", "g.mp4"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips hidden entries and symlinks without following them", async () => {
|
||||||
|
await touch("keep/clip.mp4");
|
||||||
|
await touch(".hidden.mp4");
|
||||||
|
await touch(".cache/inside.mp4");
|
||||||
|
await touch("keep/.DS_Store.jpg");
|
||||||
|
const outside = await mkdtemp(path.join(tmpdir(), "media-folder-outside-"));
|
||||||
|
try {
|
||||||
|
await writeFile(path.join(outside, "secret.mp4"), "");
|
||||||
|
await symlink(path.join(outside, "secret.mp4"), path.join(root, "link.mp4"));
|
||||||
|
await symlink(outside, path.join(root, "linked-dir"));
|
||||||
|
await symlink(path.join(root, "keep"), path.join(root, "keep-alias"));
|
||||||
|
expect(await listAll(10)).toEqual(["keep/clip.mp4"]);
|
||||||
|
await expect(provider.resolve({ path: root }, "link.mp4")).rejects.toThrow();
|
||||||
|
await expect(provider.resolve({ path: root }, "linked-dir/secret.mp4")).rejects.toThrow();
|
||||||
|
await expect(provider.resolve({ path: root }, "keep-alias/clip.mp4")).rejects.toThrow();
|
||||||
|
} finally {
|
||||||
|
await rm(outside, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves inside the root and rejects traversal", async () => {
|
||||||
|
await touch("keep/clip.mp4");
|
||||||
|
await touch(".hidden.mp4");
|
||||||
|
await expect(provider.resolve({ path: root }, "keep/clip.mp4")).resolves.toBe(
|
||||||
|
path.join(root, "keep/clip.mp4"),
|
||||||
|
);
|
||||||
|
for (const bad of [
|
||||||
|
"../x",
|
||||||
|
"../x.mp4",
|
||||||
|
"keep/../../x.mp4",
|
||||||
|
"/etc/passwd",
|
||||||
|
".hidden.mp4",
|
||||||
|
"keep",
|
||||||
|
])
|
||||||
|
await expect(provider.resolve({ path: root }, bad), bad).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects relative roots and forged cursors", async () => {
|
||||||
|
await expect(
|
||||||
|
provider.list({ config: { path: "media" }, cursor: null, limit: 5 }),
|
||||||
|
).rejects.toThrow();
|
||||||
|
await expect(
|
||||||
|
provider.list({ config: { path: root }, cursor: "not base64url!", limit: 5 }),
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("pages a 1,000-file folder without duplicates or gaps", async () => {
|
||||||
|
const expected: string[] = [];
|
||||||
|
for (let dir = 0; dir < 10; dir++) {
|
||||||
|
for (let file = 0; file < 100; file++) {
|
||||||
|
const relative = `d${dir}/f${String(file).padStart(3, "0")}.${file % 3 === 0 ? "jpg" : "mp4"}`;
|
||||||
|
// Unreadable media must not abort a page; most files are empty and skip probing.
|
||||||
|
await touch(relative, file % 50 === 7 ? "garbage, not a video" : "");
|
||||||
|
expected.push(relative);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await touch("d3/ignore.txt");
|
||||||
|
const pages: MediaSourceItem[][] = [];
|
||||||
|
let cursor: string | null = null;
|
||||||
|
do {
|
||||||
|
const page = await provider.list({ config: { path: root }, cursor, limit: 137 });
|
||||||
|
pages.push(page.items);
|
||||||
|
cursor = page.nextCursor;
|
||||||
|
} while (cursor);
|
||||||
|
const items = pages.flat();
|
||||||
|
expect(pages.length).toBe(Math.ceil(1000 / 137));
|
||||||
|
expect(items.map((item) => item.externalId)).toEqual(expected.sort());
|
||||||
|
expect(new Set(items.map((item) => item.externalId)).size).toBe(1000);
|
||||||
|
const garbage = items.find((item) => item.externalId === "d0/f007.mp4");
|
||||||
|
expect(garbage).toMatchObject({ durationMs: null, width: null, sizeBytes: 20, kind: "video" });
|
||||||
|
expect(garbage?.capturedAt).toBeInstanceOf(Date);
|
||||||
|
expect(garbage?.revision).toMatch(/^\d+-20$/);
|
||||||
|
expect(items.find((item) => item.externalId === "d0/f000.jpg")?.kind).toBe("photo");
|
||||||
|
});
|
||||||
|
|
||||||
|
it.skipIf(!ffmpegAvailable())(
|
||||||
|
"reads duration, size, capture time and GPS with ffprobe",
|
||||||
|
async () => {
|
||||||
|
await mkdir(path.join(root, "trip"), { recursive: true });
|
||||||
|
await runMediaTool("ffmpeg", [
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-f",
|
||||||
|
"lavfi",
|
||||||
|
"-i",
|
||||||
|
"testsrc2=s=320x180:r=25:d=1",
|
||||||
|
"-c:v",
|
||||||
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"ultrafast",
|
||||||
|
"-metadata",
|
||||||
|
"creation_time=2026-05-02T08:00:00Z",
|
||||||
|
"-metadata",
|
||||||
|
"location=+52.5200+013.4050/",
|
||||||
|
path.join(root, "trip/clip.mp4"),
|
||||||
|
]);
|
||||||
|
const [item] = (await provider.list({ config: { path: root }, cursor: null, limit: 5 }))
|
||||||
|
.items;
|
||||||
|
expect(item).toMatchObject({
|
||||||
|
externalId: "trip/clip.mp4",
|
||||||
|
kind: "video",
|
||||||
|
durationMs: 1000,
|
||||||
|
width: 320,
|
||||||
|
height: 180,
|
||||||
|
fps: 25,
|
||||||
|
gpsLat: 52.52,
|
||||||
|
gpsLon: 13.405,
|
||||||
|
albumNames: ["trip"],
|
||||||
|
});
|
||||||
|
expect(item?.capturedAt?.toISOString()).toBe("2026-05-02T08:00:00.000Z");
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
it("parses ISO 6709 locations", () => {
|
||||||
|
expect(parseIso6709("+52.5200+013.4050+034.000/")).toEqual({ lat: 52.52, lon: 13.405 });
|
||||||
|
expect(parseIso6709("-33.8688+151.2093/")).toEqual({ lat: -33.8688, lon: 151.2093 });
|
||||||
|
expect(parseIso6709("+95.0+013.0/")).toBeNull();
|
||||||
|
expect(parseIso6709("nowhere")).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
221
packages/adapters/src/media-folder-source.ts
Normal file
221
packages/adapters/src/media-folder-source.ts
Normal file
@ -0,0 +1,221 @@
|
|||||||
|
import type { Dirent } from "node:fs";
|
||||||
|
import { lstat, readdir } from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import type {
|
||||||
|
MediaSourceItem,
|
||||||
|
MediaSourceListRequest,
|
||||||
|
MediaSourceListResult,
|
||||||
|
MediaSourceProvider,
|
||||||
|
} from "@rakazo/adapter-kit";
|
||||||
|
import type { MediaAssetKind } from "@rakazo/contracts";
|
||||||
|
import { PathOutsideRootError, resolveInside } from "@rakazo/media/node";
|
||||||
|
import { z } from "zod";
|
||||||
|
import { probeMedia } from "./media-folder-source-probe.js";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Server-side folder source. The deployment owner picks an absolute root; the
|
||||||
|
* walk lists media files below it in a stable order (relative POSIX paths,
|
||||||
|
* code-unit sorted) so the opaque cursor (the last relative path) stays valid
|
||||||
|
* while files are added. Hidden entries and symlinks are skipped and never
|
||||||
|
* followed. Only the directories needed for one page are read.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const FOLDER_MEDIA_EXTENSIONS: Readonly<Record<string, MediaAssetKind>> = {
|
||||||
|
mp4: "video",
|
||||||
|
mov: "video",
|
||||||
|
m4v: "video",
|
||||||
|
mkv: "video",
|
||||||
|
webm: "video",
|
||||||
|
avi: "video",
|
||||||
|
jpg: "photo",
|
||||||
|
jpeg: "photo",
|
||||||
|
png: "photo",
|
||||||
|
heic: "photo",
|
||||||
|
heif: "photo",
|
||||||
|
webp: "photo",
|
||||||
|
};
|
||||||
|
|
||||||
|
const FolderConfigSchema = z.object({
|
||||||
|
path: z.string().min(1).refine(path.isAbsolute, "folder path must be absolute"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const MAX_PAGE = 1000;
|
||||||
|
const PROBE_CONCURRENCY = 4;
|
||||||
|
|
||||||
|
export interface FolderMediaSourceOptions {
|
||||||
|
ffprobePath?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function encodeFolderCursor(relativePath: string): string {
|
||||||
|
return Buffer.from(relativePath, "utf8").toString("base64url");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decodeFolderCursor(cursor: string | null): string | null {
|
||||||
|
if (!cursor) return null;
|
||||||
|
const decoded = Buffer.from(cursor, "base64url").toString("utf8");
|
||||||
|
if (!decoded || decoded.includes("\0") || encodeFolderCursor(decoded) !== cursor)
|
||||||
|
throw new Error("invalid folder cursor");
|
||||||
|
return decoded;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mediaKindOf(name: string): MediaAssetKind | null {
|
||||||
|
const ext = path.extname(name).slice(1).toLowerCase();
|
||||||
|
return FOLDER_MEDIA_EXTENSIONS[ext] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Sort key that makes a depth-first walk yield relative paths in code-unit order. */
|
||||||
|
function entryKey(entry: Dirent): string {
|
||||||
|
return entry.isDirectory() ? `${entry.name}/` : entry.name;
|
||||||
|
}
|
||||||
|
|
||||||
|
function compareKeys(a: string, b: string): number {
|
||||||
|
return a < b ? -1 : a > b ? 1 : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Yield relative paths of media files strictly after `after`, in order.
|
||||||
|
* Directories whose whole subtree sorts before the cursor are not read.
|
||||||
|
*/
|
||||||
|
async function* walkAfter(
|
||||||
|
root: string,
|
||||||
|
relativeDir: string,
|
||||||
|
after: string | null,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): AsyncGenerator<string> {
|
||||||
|
signal?.throwIfAborted();
|
||||||
|
let entries: Dirent[];
|
||||||
|
try {
|
||||||
|
entries = await readdir(path.join(root, relativeDir), { withFileTypes: true });
|
||||||
|
} catch (error) {
|
||||||
|
const code = (error as NodeJS.ErrnoException).code;
|
||||||
|
// A directory removed or locked mid-walk is skipped; the root itself must exist.
|
||||||
|
if (relativeDir && (code === "ENOENT" || code === "EACCES" || code === "ENOTDIR")) return;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
const visible = entries
|
||||||
|
.filter((entry) => !entry.name.startsWith(".") && !entry.isSymbolicLink())
|
||||||
|
.filter((entry) => entry.isDirectory() || (entry.isFile() && mediaKindOf(entry.name)))
|
||||||
|
.map((entry) => ({ entry, key: entryKey(entry) }))
|
||||||
|
.sort((a, b) => compareKeys(a.key, b.key));
|
||||||
|
for (const { entry, key } of visible) {
|
||||||
|
const relative = relativeDir ? `${relativeDir}/${key}` : key;
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
// Subtree entirely before the cursor: every path in it starts with `relative`.
|
||||||
|
if (after !== null && relative < after && !after.startsWith(relative)) continue;
|
||||||
|
yield* walkAfter(root, relative.slice(0, -1), after, signal);
|
||||||
|
} else if (after === null || relative > after) {
|
||||||
|
yield relative;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function mapLimited<T, R>(
|
||||||
|
items: T[],
|
||||||
|
limit: number,
|
||||||
|
fn: (item: T) => Promise<R>,
|
||||||
|
): Promise<R[]> {
|
||||||
|
const out = new Array<R>(items.length);
|
||||||
|
let next = 0;
|
||||||
|
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
||||||
|
while (next < items.length) {
|
||||||
|
const index = next++;
|
||||||
|
out[index] = await fn(items[index] as T);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await Promise.all(workers);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class FolderMediaSourceProvider implements MediaSourceProvider {
|
||||||
|
readonly kind = "folder" as const;
|
||||||
|
|
||||||
|
constructor(private readonly options: FolderMediaSourceOptions = {}) {}
|
||||||
|
|
||||||
|
async list(request: MediaSourceListRequest): Promise<MediaSourceListResult> {
|
||||||
|
const { path: root } = FolderConfigSchema.parse(request.config);
|
||||||
|
const limit = Math.max(1, Math.min(MAX_PAGE, Math.floor(request.limit)));
|
||||||
|
const after = decodeFolderCursor(request.cursor);
|
||||||
|
const relatives: string[] = [];
|
||||||
|
let more = false;
|
||||||
|
for await (const relative of walkAfter(root, "", after, request.signal)) {
|
||||||
|
if (relatives.length === limit) {
|
||||||
|
more = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
relatives.push(relative);
|
||||||
|
}
|
||||||
|
const items = await mapLimited(relatives, PROBE_CONCURRENCY, (relative) =>
|
||||||
|
this.describe(root, relative, request.signal),
|
||||||
|
);
|
||||||
|
const present = items.filter((item): item is MediaSourceItem => item !== null);
|
||||||
|
const last = relatives.at(-1);
|
||||||
|
return { items: present, nextCursor: more && last ? encodeFolderCursor(last) : null };
|
||||||
|
}
|
||||||
|
|
||||||
|
async resolve(config: Record<string, unknown>, externalId: string): Promise<string> {
|
||||||
|
const { path: root } = FolderConfigSchema.parse(config);
|
||||||
|
const segments = externalId.split("/");
|
||||||
|
if (segments.some((segment) => segment.startsWith(".")) || !mediaKindOf(externalId))
|
||||||
|
throw new PathOutsideRootError(externalId);
|
||||||
|
const absolute = await resolveInside(root, externalId);
|
||||||
|
// Symlinks are never followed, at any depth below the root.
|
||||||
|
let current = await resolveInside(root, ".");
|
||||||
|
for (const segment of segments) {
|
||||||
|
current = path.join(current, segment);
|
||||||
|
if ((await lstat(current)).isSymbolicLink()) throw new PathOutsideRootError(externalId);
|
||||||
|
}
|
||||||
|
if (!(await lstat(absolute)).isFile()) throw new PathOutsideRootError(externalId);
|
||||||
|
return absolute;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stat and probe one file. Probe failures yield null metadata, never a failed
|
||||||
|
* page. The parent folder name doubles as an album for rule-based labels.
|
||||||
|
*/
|
||||||
|
private async describe(
|
||||||
|
root: string,
|
||||||
|
relative: string,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<MediaSourceItem | null> {
|
||||||
|
const absolute = path.join(root, relative);
|
||||||
|
let stats: Awaited<ReturnType<typeof lstat>>;
|
||||||
|
try {
|
||||||
|
stats = await lstat(absolute);
|
||||||
|
} catch {
|
||||||
|
return null; // Removed between walk and stat.
|
||||||
|
}
|
||||||
|
if (!stats.isFile()) return null;
|
||||||
|
const kind = mediaKindOf(relative) ?? "video";
|
||||||
|
let probed: Awaited<ReturnType<typeof probeMedia>> | null = null;
|
||||||
|
if (stats.size > 0) {
|
||||||
|
try {
|
||||||
|
probed = await probeMedia(absolute, {
|
||||||
|
ffprobePath: this.options.ffprobePath,
|
||||||
|
signal,
|
||||||
|
frameTags: kind === "photo",
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Unreadable, unsupported, or no ffprobe on this host: keep the file with null metadata.
|
||||||
|
signal?.throwIfAborted();
|
||||||
|
probed = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
externalId: relative,
|
||||||
|
kind,
|
||||||
|
capturedAt: probed?.capturedAt ?? new Date(Math.trunc(stats.mtimeMs)),
|
||||||
|
durationMs: kind === "video" ? (probed?.durationMs ?? null) : null,
|
||||||
|
width: probed?.width ?? null,
|
||||||
|
height: probed?.height ?? null,
|
||||||
|
fps: kind === "video" ? (probed?.fps ?? null) : null,
|
||||||
|
gpsLat: probed?.gpsLat ?? null,
|
||||||
|
gpsLon: probed?.gpsLon ?? null,
|
||||||
|
deviceModel: probed?.deviceModel ?? null,
|
||||||
|
albumNames: relative.includes("/") ? [path.posix.basename(path.posix.dirname(relative))] : [],
|
||||||
|
isFavorite: false,
|
||||||
|
fileName: path.posix.basename(relative),
|
||||||
|
sizeBytes: stats.size,
|
||||||
|
revision: `${Math.trunc(stats.mtimeMs)}-${stats.size}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
222
packages/adapters/src/media-sidecar.test.ts
Normal file
222
packages/adapters/src/media-sidecar.test.ts
Normal file
@ -0,0 +1,222 @@
|
|||||||
|
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||||
|
import { MediaInferenceOperationSchema } from "@rakazo/contracts";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { MediaSidecarError, SidecarMediaInferenceProvider } from "./media-sidecar.js";
|
||||||
|
|
||||||
|
const TOKEN = "test-token-0123456789abcdef0123456789abcdef";
|
||||||
|
const SCENES = {
|
||||||
|
shots: [{ startMs: 0, endMs: 1000, score: 1 }],
|
||||||
|
scenes: [{ startMs: 0, endMs: 1000 }],
|
||||||
|
};
|
||||||
|
|
||||||
|
interface Call {
|
||||||
|
url: string;
|
||||||
|
init: RequestInit;
|
||||||
|
}
|
||||||
|
|
||||||
|
function json(status: number, body: unknown): Response {
|
||||||
|
return new Response(JSON.stringify(body), {
|
||||||
|
status,
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function fakeFetch(responses: Array<Response | ((init: RequestInit) => Promise<Response>)>) {
|
||||||
|
const calls: Call[] = [];
|
||||||
|
const impl = (async (input: string | URL | Request, init?: RequestInit) => {
|
||||||
|
calls.push({ url: String(input), init: init ?? {} });
|
||||||
|
const next = responses.shift();
|
||||||
|
if (!next) throw new Error("unexpected request");
|
||||||
|
return typeof next === "function" ? next(init ?? {}) : next;
|
||||||
|
}) as typeof fetch;
|
||||||
|
return { impl, calls };
|
||||||
|
}
|
||||||
|
|
||||||
|
function client(fetchImpl: typeof fetch, extra: { retryBaseMs?: number } = {}) {
|
||||||
|
return new SidecarMediaInferenceProvider({
|
||||||
|
baseUrl: "http://127.0.0.1:8800/",
|
||||||
|
token: TOKEN,
|
||||||
|
fetch: fetchImpl,
|
||||||
|
retryBaseMs: 0,
|
||||||
|
...extra,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("SidecarMediaInferenceProvider", () => {
|
||||||
|
it("posts camelCase JSON with bearer auth and without the signal", async () => {
|
||||||
|
const { impl, calls } = fakeFetch([json(200, SCENES)]);
|
||||||
|
const controller = new AbortController();
|
||||||
|
const result = await client(impl).detectScenes({
|
||||||
|
path: "/data/media/a.mp4",
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
expect(result).toEqual(SCENES);
|
||||||
|
expect(calls).toHaveLength(1);
|
||||||
|
const [call] = calls;
|
||||||
|
expect(call?.url).toBe("http://127.0.0.1:8800/v1/detectScenes");
|
||||||
|
expect(call?.init.method).toBe("POST");
|
||||||
|
const headers = call?.init.headers as Record<string, string>;
|
||||||
|
expect(headers.authorization).toBe(`Bearer ${TOKEN}`);
|
||||||
|
expect(headers["content-type"]).toBe("application/json");
|
||||||
|
expect(JSON.parse(String(call?.init.body))).toEqual({ path: "/data/media/a.mp4" });
|
||||||
|
expect(call?.init.signal).toBeInstanceOf(AbortSignal);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads capabilities with GET", async () => {
|
||||||
|
const capabilities = {
|
||||||
|
configured: true,
|
||||||
|
operations: Object.fromEntries(
|
||||||
|
MediaInferenceOperationSchema.options.map((operation) => [
|
||||||
|
operation,
|
||||||
|
{
|
||||||
|
available: operation !== "synthesizeIdentity",
|
||||||
|
experimental: false,
|
||||||
|
model: "m",
|
||||||
|
license: "Apache-2.0",
|
||||||
|
commercialUse: true,
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
accelerator: "cpu",
|
||||||
|
hardwareEncoders: [],
|
||||||
|
};
|
||||||
|
const { impl, calls } = fakeFetch([json(200, capabilities)]);
|
||||||
|
await expect(client(impl).capabilities()).resolves.toEqual(capabilities);
|
||||||
|
expect(calls[0]?.url).toBe("http://127.0.0.1:8800/v1/capabilities");
|
||||||
|
expect(calls[0]?.init.method).toBe("GET");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps 501 to UnsupportedMediaOperationError", async () => {
|
||||||
|
const { impl } = fakeFetch([
|
||||||
|
json(501, {
|
||||||
|
error: { code: "unsupported", message: "no licensed model", operation: "inpaintVideo" },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
const error = await client(impl)
|
||||||
|
.inpaintVideo({ path: "/m/a.mp4", maskDir: "/m/masks", outPath: "/m/out.mp4" })
|
||||||
|
.catch((caught: unknown) => caught);
|
||||||
|
expect(error).toBeInstanceOf(UnsupportedMediaOperationError);
|
||||||
|
expect((error as UnsupportedMediaOperationError).operation).toBe("inpaintVideo");
|
||||||
|
expect((error as Error).message).toContain("no licensed model");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries 503 busy with backoff, at most three times", async () => {
|
||||||
|
const busy = () => json(503, { error: { code: "busy", message: "queue full" } });
|
||||||
|
const recovered = fakeFetch([busy(), busy(), json(200, SCENES)]);
|
||||||
|
await expect(client(recovered.impl).detectScenes({ path: "/m/a.mp4" })).resolves.toEqual(
|
||||||
|
SCENES,
|
||||||
|
);
|
||||||
|
expect(recovered.calls).toHaveLength(3);
|
||||||
|
|
||||||
|
const exhausted = fakeFetch([busy(), busy(), busy(), busy(), busy()]);
|
||||||
|
const error = await client(exhausted.impl)
|
||||||
|
.detectScenes({ path: "/m/a.mp4" })
|
||||||
|
.catch((caught: unknown) => caught);
|
||||||
|
expect(exhausted.calls).toHaveLength(4);
|
||||||
|
expect(error).toBeInstanceOf(MediaSidecarError);
|
||||||
|
expect((error as MediaSidecarError).code).toBe("busy");
|
||||||
|
expect((error as MediaSidecarError).status).toBe(503);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps typed sidecar errors", async () => {
|
||||||
|
const { impl } = fakeFetch([
|
||||||
|
json(400, { error: { code: "invalid_path", message: "path is outside MEDIA_ROOT" } }),
|
||||||
|
json(500, "not json-shaped"),
|
||||||
|
]);
|
||||||
|
const provider = client(impl);
|
||||||
|
await expect(provider.detectScenes({ path: "/etc/passwd" })).rejects.toMatchObject({
|
||||||
|
name: "MediaSidecarError",
|
||||||
|
code: "invalid_path",
|
||||||
|
status: 400,
|
||||||
|
operation: "detectScenes",
|
||||||
|
});
|
||||||
|
await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({
|
||||||
|
code: "http_error",
|
||||||
|
status: 500,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes caller aborts through as AbortError", async () => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const { impl } = fakeFetch([
|
||||||
|
(init) =>
|
||||||
|
new Promise<Response>((_, reject) => {
|
||||||
|
init.signal?.addEventListener("abort", () => reject(init.signal?.reason));
|
||||||
|
setTimeout(() => controller.abort(), 10);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
await expect(
|
||||||
|
client(impl).trackFaces({ path: "/m/a.mp4", sampleFps: 5, signal: controller.signal }),
|
||||||
|
).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
|
||||||
|
const already = new AbortController();
|
||||||
|
already.abort();
|
||||||
|
const idle = fakeFetch([]);
|
||||||
|
await expect(
|
||||||
|
client(idle.impl).detectScenes({ path: "/m/a.mp4", signal: already.signal }),
|
||||||
|
).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
expect(idle.calls).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a timeout as a typed error", async () => {
|
||||||
|
const { impl } = fakeFetch([
|
||||||
|
(init) =>
|
||||||
|
new Promise<Response>((_, reject) => {
|
||||||
|
init.signal?.addEventListener("abort", () => reject(init.signal?.reason));
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
const provider = new SidecarMediaInferenceProvider({
|
||||||
|
baseUrl: "http://127.0.0.1:8800",
|
||||||
|
token: TOKEN,
|
||||||
|
fetch: impl,
|
||||||
|
timeoutsMs: { detectScenes: 20 },
|
||||||
|
});
|
||||||
|
await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({
|
||||||
|
code: "timeout",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects malformed responses", async () => {
|
||||||
|
const { impl } = fakeFetch([
|
||||||
|
json(200, { shots: [{ startMs: -1, endMs: 10, score: 1 }], scenes: [] }),
|
||||||
|
json(200, {
|
||||||
|
frames: [{ tMs: 0, faces: [{ box: { x: 2, y: 0, w: 0.1, h: 0.1 }, confidence: 1 }] }],
|
||||||
|
}),
|
||||||
|
json(200, { model: "m", dim: 3, vectors: [[1, 2]] }),
|
||||||
|
new Response("<html>", { status: 200 }),
|
||||||
|
]);
|
||||||
|
const provider = client(impl);
|
||||||
|
await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({
|
||||||
|
code: "malformed_response",
|
||||||
|
});
|
||||||
|
await expect(provider.detectFaces({ path: "/m/a.mp4", sampleFps: 1 })).rejects.toMatchObject({
|
||||||
|
code: "malformed_response",
|
||||||
|
});
|
||||||
|
await expect(provider.embedFrames({ path: "/m/a.mp4", timesMs: [0] })).rejects.toMatchObject({
|
||||||
|
code: "malformed_response",
|
||||||
|
});
|
||||||
|
await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({
|
||||||
|
code: "malformed_response",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never puts the token into error messages", async () => {
|
||||||
|
const { impl } = fakeFetch([
|
||||||
|
json(401, { error: { code: "unauthorized", message: `bad token ${TOKEN}` } }),
|
||||||
|
json(500, { error: { code: "internal", message: `crashed with Bearer ${TOKEN}` } }),
|
||||||
|
async () => {
|
||||||
|
throw new TypeError(`connect ECONNREFUSED (auth ${TOKEN})`);
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
const provider = client(impl);
|
||||||
|
for (let i = 0; i < 3; i++) {
|
||||||
|
const error = (await provider
|
||||||
|
.detectScenes({ path: "/m/a.mp4" })
|
||||||
|
.catch((caught: unknown) => caught)) as Error;
|
||||||
|
expect(error).toBeInstanceOf(MediaSidecarError);
|
||||||
|
expect(error.message).not.toContain(TOKEN);
|
||||||
|
expect(JSON.stringify(error)).not.toContain(TOKEN);
|
||||||
|
expect(String(error.stack)).not.toContain(TOKEN);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
410
packages/adapters/src/media-sidecar.ts
Normal file
410
packages/adapters/src/media-sidecar.ts
Normal file
@ -0,0 +1,410 @@
|
|||||||
|
import type {
|
||||||
|
AudioFeaturesResult,
|
||||||
|
ClassifyContentResult,
|
||||||
|
DetectFacesResult,
|
||||||
|
DetectScenesResult,
|
||||||
|
EmbedFramesResult,
|
||||||
|
FaceLandmarksResult,
|
||||||
|
InpaintVideoResult,
|
||||||
|
MediaInferenceProvider,
|
||||||
|
SaliencyResult,
|
||||||
|
SegmentPromptedResult,
|
||||||
|
TrackFacesResult,
|
||||||
|
TranscribeResult,
|
||||||
|
} from "@rakazo/adapter-kit";
|
||||||
|
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||||
|
import type { MediaCapabilities, MediaInferenceOperation } from "@rakazo/contracts";
|
||||||
|
import {
|
||||||
|
BoxKeyframeSchema,
|
||||||
|
ContentCategorySchema,
|
||||||
|
MediaCapabilitiesSchema,
|
||||||
|
NormalizedBoxSchema,
|
||||||
|
NormalizedPointSchema,
|
||||||
|
} from "@rakazo/contracts";
|
||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* HTTP client for the local media sidecar (infra/media-worker, API v1).
|
||||||
|
* Bearer auth, JSON bodies, per-operation timeouts, bounded retry on 503 busy,
|
||||||
|
* 501 → UnsupportedMediaOperationError, every other failure → MediaSidecarError.
|
||||||
|
* Responses are validated before they reach the caller. The token never
|
||||||
|
* appears in an error message.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const DEFAULT_TIMEOUT_MS = 30 * 60_000;
|
||||||
|
const LONG_TIMEOUT_MS = 4 * 60 * 60_000;
|
||||||
|
const MAX_BUSY_RETRIES = 3;
|
||||||
|
|
||||||
|
export type MediaSidecarErrorCode =
|
||||||
|
| "bad_request"
|
||||||
|
| "invalid_path"
|
||||||
|
| "unauthorized"
|
||||||
|
| "too_large"
|
||||||
|
| "busy"
|
||||||
|
| "internal"
|
||||||
|
| "timeout"
|
||||||
|
| "unreachable"
|
||||||
|
| "malformed_response"
|
||||||
|
| "http_error";
|
||||||
|
|
||||||
|
export class MediaSidecarError extends Error {
|
||||||
|
constructor(
|
||||||
|
readonly code: MediaSidecarErrorCode,
|
||||||
|
message: string,
|
||||||
|
readonly status: number | null,
|
||||||
|
readonly operation: MediaInferenceOperation | "capabilities",
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "MediaSidecarError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Response schemas (shapes mirror the adapter-kit result types)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const Ms = z.number().int().min(0);
|
||||||
|
const Unit = z.number().min(0).max(1);
|
||||||
|
const Span = z.object({ startMs: Ms, endMs: Ms });
|
||||||
|
const ScoredKeyframe = BoxKeyframeSchema.extend({ confidence: Unit });
|
||||||
|
|
||||||
|
const schemas = {
|
||||||
|
detectScenes: z.object({
|
||||||
|
shots: z.array(Span.extend({ score: z.number() })),
|
||||||
|
scenes: z.array(Span),
|
||||||
|
}) satisfies z.ZodType<DetectScenesResult>,
|
||||||
|
transcribe: z.object({
|
||||||
|
language: z.string().nullable(),
|
||||||
|
words: z.array(Span.extend({ text: z.string(), confidence: Unit })),
|
||||||
|
}) satisfies z.ZodType<TranscribeResult>,
|
||||||
|
detectFaces: z.object({
|
||||||
|
frames: z.array(
|
||||||
|
z.object({
|
||||||
|
tMs: Ms,
|
||||||
|
faces: z.array(
|
||||||
|
z.object({
|
||||||
|
box: NormalizedBoxSchema,
|
||||||
|
confidence: Unit,
|
||||||
|
landmarks: z.array(NormalizedPointSchema).optional(),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
}) satisfies z.ZodType<DetectFacesResult>,
|
||||||
|
trackFaces: z.object({
|
||||||
|
sampleFps: z.number().positive(),
|
||||||
|
tracks: z.array(
|
||||||
|
z.object({
|
||||||
|
trackId: z.string().min(1),
|
||||||
|
keyframes: z.array(ScoredKeyframe),
|
||||||
|
embedding: z.array(z.number()).nullable(),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
}) satisfies z.ZodType<TrackFacesResult>,
|
||||||
|
faceLandmarks: z.object({
|
||||||
|
frames: z.array(
|
||||||
|
z.object({
|
||||||
|
tMs: Ms,
|
||||||
|
faces: z.array(
|
||||||
|
z.object({ box: NormalizedBoxSchema, landmarks: z.array(NormalizedPointSchema) }),
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
}) satisfies z.ZodType<FaceLandmarksResult>,
|
||||||
|
embedFrames: z
|
||||||
|
.object({
|
||||||
|
model: z.string(),
|
||||||
|
dim: z.number().int().positive(),
|
||||||
|
vectors: z.array(z.array(z.number())),
|
||||||
|
})
|
||||||
|
.refine((value) => value.vectors.every((vector) => vector.length === value.dim), {
|
||||||
|
message: "vector length does not match dim",
|
||||||
|
}) satisfies z.ZodType<EmbedFramesResult>,
|
||||||
|
classifyContent: z.object({
|
||||||
|
model: z.string(),
|
||||||
|
frames: z.array(z.object({ tMs: Ms, scores: z.partialRecord(ContentCategorySchema, Unit) })),
|
||||||
|
}) satisfies z.ZodType<ClassifyContentResult>,
|
||||||
|
segmentPrompted: z.object({
|
||||||
|
maskDir: z.string().min(1),
|
||||||
|
keyframes: z.array(ScoredKeyframe),
|
||||||
|
score: Unit,
|
||||||
|
}) satisfies z.ZodType<SegmentPromptedResult>,
|
||||||
|
inpaintVideo: z.object({
|
||||||
|
outPath: z.string().min(1),
|
||||||
|
quality: z.object({ temporalStability: Unit, edgeQuality: Unit, score: Unit }),
|
||||||
|
}) satisfies z.ZodType<InpaintVideoResult>,
|
||||||
|
redact: z.object({
|
||||||
|
outPath: z.string().min(1),
|
||||||
|
framesProcessed: z.number().int().min(0),
|
||||||
|
}) satisfies z.ZodType<{ outPath: string; framesProcessed: number }>,
|
||||||
|
synthesizeIdentity: z.object({ outPath: z.string().min(1) }) satisfies z.ZodType<{
|
||||||
|
outPath: string;
|
||||||
|
}>,
|
||||||
|
audioFeatures: z.object({
|
||||||
|
integratedLufs: z.number().nullable(),
|
||||||
|
truePeakDb: z.number().nullable(),
|
||||||
|
tempoBpm: z.number().nullable(),
|
||||||
|
beatsMs: z.array(Ms),
|
||||||
|
energy: z.array(z.object({ tMs: Ms, value: z.number() })),
|
||||||
|
silences: z.array(Span),
|
||||||
|
}) satisfies z.ZodType<AudioFeaturesResult>,
|
||||||
|
saliency: z.object({
|
||||||
|
points: z.array(z.object({ tMs: Ms, cx: Unit, cy: Unit, strength: z.number() })),
|
||||||
|
}) satisfies z.ZodType<SaliencyResult>,
|
||||||
|
} as const satisfies Record<MediaInferenceOperation, z.ZodType>;
|
||||||
|
|
||||||
|
export const MEDIA_SIDECAR_RESPONSE_SCHEMAS = schemas;
|
||||||
|
|
||||||
|
const ErrorBodySchema = z.object({
|
||||||
|
error: z.object({ code: z.string(), message: z.string().optional() }),
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Client
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export interface SidecarMediaInferenceOptions {
|
||||||
|
baseUrl: string;
|
||||||
|
token: string;
|
||||||
|
fetch?: typeof fetch;
|
||||||
|
/** Per-operation timeout overrides in ms. */
|
||||||
|
timeoutsMs?: Partial<Record<MediaInferenceOperation | "capabilities", number>>;
|
||||||
|
/** First busy-retry delay; doubles per attempt. */
|
||||||
|
retryBaseMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
type Request<K extends MediaInferenceOperation> = Parameters<MediaInferenceProvider[K]>[0];
|
||||||
|
type Result<K extends MediaInferenceOperation> = Awaited<ReturnType<MediaInferenceProvider[K]>>;
|
||||||
|
|
||||||
|
const KNOWN_CODES = new Set<string>([
|
||||||
|
"bad_request",
|
||||||
|
"invalid_path",
|
||||||
|
"unauthorized",
|
||||||
|
"too_large",
|
||||||
|
"busy",
|
||||||
|
"internal",
|
||||||
|
]);
|
||||||
|
|
||||||
|
export class SidecarMediaInferenceProvider implements MediaInferenceProvider {
|
||||||
|
readonly id = "sidecar";
|
||||||
|
private readonly baseUrl: string;
|
||||||
|
private readonly fetchImpl: typeof fetch;
|
||||||
|
|
||||||
|
constructor(private readonly options: SidecarMediaInferenceOptions) {
|
||||||
|
this.baseUrl = options.baseUrl.replace(/\/+$/, "");
|
||||||
|
this.fetchImpl = options.fetch ?? fetch;
|
||||||
|
}
|
||||||
|
|
||||||
|
async capabilities(signal?: AbortSignal): Promise<MediaCapabilities> {
|
||||||
|
return this.call("capabilities", "GET", "/v1/capabilities", undefined, signal, (body) =>
|
||||||
|
MediaCapabilitiesSchema.parse(body),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
detectScenes(request: Request<"detectScenes">) {
|
||||||
|
return this.operation("detectScenes", request);
|
||||||
|
}
|
||||||
|
transcribe(request: Request<"transcribe">) {
|
||||||
|
return this.operation("transcribe", request);
|
||||||
|
}
|
||||||
|
detectFaces(request: Request<"detectFaces">) {
|
||||||
|
return this.operation("detectFaces", request);
|
||||||
|
}
|
||||||
|
trackFaces(request: Request<"trackFaces">) {
|
||||||
|
return this.operation("trackFaces", request);
|
||||||
|
}
|
||||||
|
faceLandmarks(request: Request<"faceLandmarks">) {
|
||||||
|
return this.operation("faceLandmarks", request);
|
||||||
|
}
|
||||||
|
embedFrames(request: Request<"embedFrames">) {
|
||||||
|
return this.operation("embedFrames", request);
|
||||||
|
}
|
||||||
|
classifyContent(request: Request<"classifyContent">) {
|
||||||
|
return this.operation("classifyContent", request);
|
||||||
|
}
|
||||||
|
segmentPrompted(request: Request<"segmentPrompted">) {
|
||||||
|
return this.operation("segmentPrompted", request);
|
||||||
|
}
|
||||||
|
inpaintVideo(request: Request<"inpaintVideo">) {
|
||||||
|
return this.operation("inpaintVideo", request);
|
||||||
|
}
|
||||||
|
redact(request: Request<"redact">) {
|
||||||
|
return this.operation("redact", request);
|
||||||
|
}
|
||||||
|
synthesizeIdentity(request: Request<"synthesizeIdentity">) {
|
||||||
|
return this.operation("synthesizeIdentity", request);
|
||||||
|
}
|
||||||
|
audioFeatures(request: Request<"audioFeatures">) {
|
||||||
|
return this.operation("audioFeatures", request);
|
||||||
|
}
|
||||||
|
saliency(request: Request<"saliency">) {
|
||||||
|
return this.operation("saliency", request);
|
||||||
|
}
|
||||||
|
|
||||||
|
private operation<K extends MediaInferenceOperation>(
|
||||||
|
operation: K,
|
||||||
|
request: Request<K>,
|
||||||
|
): Promise<Result<K>> {
|
||||||
|
const { signal, ...body } = request;
|
||||||
|
const schema = schemas[operation] as z.ZodType;
|
||||||
|
return this.call(operation, "POST", `/v1/${operation}`, body, signal, (json) => {
|
||||||
|
return schema.parse(json) as Result<K>;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private timeoutFor(operation: MediaInferenceOperation | "capabilities"): number {
|
||||||
|
const override = this.options.timeoutsMs?.[operation];
|
||||||
|
if (override !== undefined) return override;
|
||||||
|
return operation === "inpaintVideo" || operation === "synthesizeIdentity"
|
||||||
|
? LONG_TIMEOUT_MS
|
||||||
|
: DEFAULT_TIMEOUT_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async call<T>(
|
||||||
|
operation: MediaInferenceOperation | "capabilities",
|
||||||
|
method: "GET" | "POST",
|
||||||
|
route: string,
|
||||||
|
body: unknown,
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
parse: (json: unknown) => T,
|
||||||
|
): Promise<T> {
|
||||||
|
signal?.throwIfAborted();
|
||||||
|
const timeout = AbortSignal.timeout(this.timeoutFor(operation));
|
||||||
|
const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
||||||
|
const retryBase = this.options.retryBaseMs ?? 500;
|
||||||
|
for (let attempt = 0; ; attempt++) {
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await this.fetchImpl(`${this.baseUrl}${route}`, {
|
||||||
|
method,
|
||||||
|
headers: {
|
||||||
|
authorization: `Bearer ${this.options.token}`,
|
||||||
|
accept: "application/json",
|
||||||
|
...(body === undefined ? {} : { "content-type": "application/json" }),
|
||||||
|
},
|
||||||
|
body: body === undefined ? undefined : JSON.stringify(body),
|
||||||
|
signal: combined,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
throw this.transportError(error, operation, signal, timeout);
|
||||||
|
}
|
||||||
|
if (response.status === 503 && attempt < MAX_BUSY_RETRIES) {
|
||||||
|
await response.body?.cancel().catch(() => undefined);
|
||||||
|
try {
|
||||||
|
await delay(retryBase * 2 ** attempt, combined);
|
||||||
|
} catch (error) {
|
||||||
|
throw this.transportError(error, operation, signal, timeout);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let json: unknown;
|
||||||
|
let text: string;
|
||||||
|
try {
|
||||||
|
text = await response.text();
|
||||||
|
} catch (error) {
|
||||||
|
throw this.transportError(error, operation, signal, timeout);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
json = text ? JSON.parse(text) : null;
|
||||||
|
} catch {
|
||||||
|
json = undefined;
|
||||||
|
}
|
||||||
|
if (!response.ok) throw this.httpError(response.status, json, operation);
|
||||||
|
if (json === undefined)
|
||||||
|
throw new MediaSidecarError(
|
||||||
|
"malformed_response",
|
||||||
|
`media sidecar returned invalid JSON for ${operation}`,
|
||||||
|
response.status,
|
||||||
|
operation,
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
return parse(json);
|
||||||
|
} catch (error) {
|
||||||
|
const detail =
|
||||||
|
error instanceof z.ZodError
|
||||||
|
? error.issues
|
||||||
|
.slice(0, 3)
|
||||||
|
.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`)
|
||||||
|
.join("; ")
|
||||||
|
: "unexpected shape";
|
||||||
|
throw new MediaSidecarError(
|
||||||
|
"malformed_response",
|
||||||
|
this.scrub(`media sidecar returned a malformed ${operation} response: ${detail}`),
|
||||||
|
response.status,
|
||||||
|
operation,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private httpError(
|
||||||
|
status: number,
|
||||||
|
json: unknown,
|
||||||
|
operation: MediaInferenceOperation | "capabilities",
|
||||||
|
): Error {
|
||||||
|
const parsed = ErrorBodySchema.safeParse(json);
|
||||||
|
const message = this.scrub(
|
||||||
|
parsed.success && parsed.data.error.message
|
||||||
|
? parsed.data.error.message
|
||||||
|
: `media sidecar responded ${status}`,
|
||||||
|
);
|
||||||
|
if (status === 501 && operation !== "capabilities")
|
||||||
|
return new UnsupportedMediaOperationError(operation, message);
|
||||||
|
const serverCode = parsed.success ? parsed.data.error.code : "";
|
||||||
|
const code: MediaSidecarErrorCode = KNOWN_CODES.has(serverCode)
|
||||||
|
? (serverCode as MediaSidecarErrorCode)
|
||||||
|
: status === 401
|
||||||
|
? "unauthorized"
|
||||||
|
: status === 503
|
||||||
|
? "busy"
|
||||||
|
: "http_error";
|
||||||
|
return new MediaSidecarError(code, `${operation}: ${message}`, status, operation);
|
||||||
|
}
|
||||||
|
|
||||||
|
private transportError(
|
||||||
|
error: unknown,
|
||||||
|
operation: MediaInferenceOperation | "capabilities",
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
timeout: AbortSignal,
|
||||||
|
): unknown {
|
||||||
|
if (signal?.aborted) return signal.reason;
|
||||||
|
if (timeout.aborted)
|
||||||
|
return new MediaSidecarError(
|
||||||
|
"timeout",
|
||||||
|
`media sidecar ${operation} timed out`,
|
||||||
|
null,
|
||||||
|
operation,
|
||||||
|
);
|
||||||
|
const detail = error instanceof Error ? error.message : String(error);
|
||||||
|
return new MediaSidecarError(
|
||||||
|
"unreachable",
|
||||||
|
this.scrub(`media sidecar is unreachable: ${detail}`),
|
||||||
|
null,
|
||||||
|
operation,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Strip the bearer token from any text that may end up in logs or errors. */
|
||||||
|
private scrub(text: string): string {
|
||||||
|
const token = this.options.token;
|
||||||
|
return token ? text.split(token).join("[redacted]") : text;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function delay(ms: number, signal: AbortSignal): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
if (signal.aborted) {
|
||||||
|
reject(signal.reason);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
signal.removeEventListener("abort", onAbort);
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
reject(signal.reason);
|
||||||
|
};
|
||||||
|
signal.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user