feat(media): inference emulator, sidecar client, folder source and factory
Add the Media Studio adapters behind the frozen provider contracts: a deterministic offline inference emulator driven by file-name markers that renders real redactions with ffmpeg, a synthetic emulator library, the bearer-auth HTTP client for the local media sidecar with validated responses, typed errors, busy retries and timeouts, a lazily paginated folder source that never follows symlinks, and the configuration factory. A shared conformance suite runs against the emulator and, in the Docker integration job, against the sidecar. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj
This commit is contained in:
Claude
parent
7ccc96ac11
commit
270ea8068a
281
packages/adapters/src/media-conformance.test.ts
Normal file
281
packages/adapters/src/media-conformance.test.ts
Normal file
@ -0,0 +1,281 @@
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { MediaInferenceProvider } from "@rakazo/adapter-kit";
|
||||
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||
import type { MediaInferenceOperation } from "@rakazo/contracts";
|
||||
import { MediaCapabilitiesSchema, MediaInferenceOperationSchema } from "@rakazo/contracts";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { EmulatorMediaInferenceProvider } from "./media-emulator.js";
|
||||
import { extractRgbFrame, ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js";
|
||||
import { MEDIA_SIDECAR_RESPONSE_SCHEMAS, SidecarMediaInferenceProvider } from "./media-sidecar.js";
|
||||
|
||||
/*
|
||||
* One contract for every MediaInferenceProvider. The emulator always runs; the
|
||||
* sidecar runs only in the Docker integration job, which sets
|
||||
* MEDIA_CONFORMANCE_URL, MEDIA_CONFORMANCE_TOKEN and MEDIA_CONFORMANCE_ROOT (a
|
||||
* directory inside the sidecar's MEDIA_ROOT, mounted at the same path here).
|
||||
*/
|
||||
|
||||
const WIDTH = 320;
|
||||
const HEIGHT = 180;
|
||||
const hasFfmpeg = ffmpegAvailable();
|
||||
|
||||
interface Target {
|
||||
name: string;
|
||||
/** Directory inside the provider's root where the suite writes fixtures. */
|
||||
root: () => Promise<string>;
|
||||
provider: (root: string) => MediaInferenceProvider;
|
||||
}
|
||||
|
||||
const targets: Target[] = [
|
||||
{
|
||||
name: "emulator",
|
||||
root: () => mkdtemp(path.join(tmpdir(), "media-conformance-")),
|
||||
provider: (root) => new EmulatorMediaInferenceProvider({ root }),
|
||||
},
|
||||
];
|
||||
|
||||
const sidecarUrl = process.env.MEDIA_CONFORMANCE_URL;
|
||||
const sidecarToken = process.env.MEDIA_CONFORMANCE_TOKEN;
|
||||
const sidecarRoot = process.env.MEDIA_CONFORMANCE_ROOT;
|
||||
if (sidecarUrl && sidecarToken && sidecarRoot) {
|
||||
targets.push({
|
||||
name: "sidecar",
|
||||
root: async () => {
|
||||
await mkdir(sidecarRoot, { recursive: true });
|
||||
return mkdtemp(path.join(sidecarRoot, "conformance-"));
|
||||
},
|
||||
provider: () => new SidecarMediaInferenceProvider({ baseUrl: sidecarUrl, token: sidecarToken }),
|
||||
});
|
||||
}
|
||||
|
||||
function requestFor(operation: MediaInferenceOperation, clip: string, root: string) {
|
||||
const box = { x: 0.25, y: 0.25, w: 0.25, h: 0.25 };
|
||||
switch (operation) {
|
||||
case "detectScenes":
|
||||
case "audioFeatures":
|
||||
return { path: clip };
|
||||
case "transcribe":
|
||||
return { path: clip, language: "en" };
|
||||
case "detectFaces":
|
||||
case "trackFaces":
|
||||
return { path: clip, sampleFps: 2 };
|
||||
case "faceLandmarks":
|
||||
case "embedFrames":
|
||||
case "saliency":
|
||||
return { path: clip, timesMs: [0, 500, 1000] };
|
||||
case "classifyContent":
|
||||
return { path: clip, sampleFps: 2, extraTimesMs: [750] };
|
||||
case "segmentPrompted":
|
||||
return {
|
||||
path: clip,
|
||||
prompt: { tMs: 500, box },
|
||||
propagate: true,
|
||||
outDir: path.join(root, "masks"),
|
||||
};
|
||||
case "inpaintVideo":
|
||||
return {
|
||||
path: clip,
|
||||
maskDir: path.join(root, "masks"),
|
||||
outPath: path.join(root, "inpaint.mp4"),
|
||||
};
|
||||
case "redact":
|
||||
return {
|
||||
path: clip,
|
||||
redactions: [
|
||||
{
|
||||
keyframes: [
|
||||
{ tMs: 0, box },
|
||||
{ tMs: 2000, box },
|
||||
],
|
||||
mode: "blur" as const,
|
||||
paddingPct: 0,
|
||||
featherPct: 0,
|
||||
},
|
||||
],
|
||||
outPath: path.join(root, "redacted-any.mp4"),
|
||||
};
|
||||
case "synthesizeIdentity":
|
||||
return {
|
||||
path: clip,
|
||||
track: [{ tMs: 0, box }],
|
||||
seed: 7,
|
||||
outPath: path.join(root, "synthetic.mp4"),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Mean absolute RGB difference inside and outside a pixel box (with a guard margin). */
|
||||
function regionDiff(
|
||||
a: Buffer,
|
||||
b: Buffer,
|
||||
box: { x: number; y: number; w: number; h: number },
|
||||
margin = 4,
|
||||
) {
|
||||
let inside = 0;
|
||||
let insideCount = 0;
|
||||
let outside = 0;
|
||||
let outsideCount = 0;
|
||||
for (let y = 0; y < HEIGHT; y++) {
|
||||
for (let x = 0; x < WIDTH; x++) {
|
||||
const within = x >= box.x && x < box.x + box.w && y >= box.y && y < box.y + box.h;
|
||||
const near =
|
||||
x >= box.x - margin &&
|
||||
x < box.x + box.w + margin &&
|
||||
y >= box.y - margin &&
|
||||
y < box.y + box.h + margin;
|
||||
let diff = 0;
|
||||
for (let c = 0; c < 3; c++) {
|
||||
const i = (y * WIDTH + x) * 3 + c;
|
||||
diff += Math.abs((a[i] ?? 0) - (b[i] ?? 0));
|
||||
}
|
||||
if (within) {
|
||||
inside += diff / 3;
|
||||
insideCount++;
|
||||
} else if (!near) {
|
||||
outside += diff / 3;
|
||||
outsideCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { inside: inside / insideCount, outside: outside / outsideCount };
|
||||
}
|
||||
|
||||
for (const target of targets) {
|
||||
describe.skipIf(!hasFfmpeg)(`MediaInferenceProvider conformance: ${target.name}`, () => {
|
||||
let root: string;
|
||||
let clip: string;
|
||||
let provider: MediaInferenceProvider;
|
||||
|
||||
beforeAll(async () => {
|
||||
root = await target.root();
|
||||
clip = await renderTestClip(path.join(root, "clip.mp4"), {
|
||||
seconds: 2,
|
||||
width: WIDTH,
|
||||
height: HEIGHT,
|
||||
});
|
||||
provider = target.provider(root);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (root) await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("reports a capabilities document for every operation", async () => {
|
||||
const capabilities = MediaCapabilitiesSchema.parse(await provider.capabilities());
|
||||
expect(Object.keys(capabilities.operations).sort()).toEqual(
|
||||
[...MediaInferenceOperationSchema.options].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns schema-valid output or Unsupported for every operation", async () => {
|
||||
const capabilities = await provider.capabilities();
|
||||
// segmentPrompted runs first so inpaintVideo finds its mask directory.
|
||||
for (const operation of MediaInferenceOperationSchema.options) {
|
||||
const request = requestFor(operation, clip, root);
|
||||
const call = provider[operation] as (input: unknown) => Promise<unknown>;
|
||||
try {
|
||||
const result = await call.call(provider, request);
|
||||
const parsed = MEDIA_SIDECAR_RESPONSE_SCHEMAS[operation].safeParse(result);
|
||||
expect(parsed.success, `${operation}: ${parsed.error?.message}`).toBe(true);
|
||||
expect(capabilities.operations[operation]?.available, operation).toBe(true);
|
||||
} catch (error) {
|
||||
if (!(error instanceof UnsupportedMediaOperationError)) throw error;
|
||||
expect(error.operation).toBe(operation);
|
||||
expect(capabilities.operations[operation]?.available, operation).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects paths outside the media root", async () => {
|
||||
const outside = await mkdtemp(path.join(tmpdir(), "media-outside-"));
|
||||
try {
|
||||
const stray = path.join(outside, "stray.mp4");
|
||||
await writeFile(stray, "not inside the root");
|
||||
await expect(provider.detectScenes({ path: stray })).rejects.toThrow();
|
||||
await expect(
|
||||
provider.detectScenes({
|
||||
path: path.join(root, "..", path.basename(outside), "stray.mp4"),
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
provider.redact({
|
||||
path: clip,
|
||||
redactions: [],
|
||||
outPath: path.join(outside, "escaped.mp4"),
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
} finally {
|
||||
await rm(outside, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("redacts only inside the box", async () => {
|
||||
const box = { x: 0.25, y: 0.25, w: 0.25, h: 0.25 };
|
||||
const pixels = { x: 80, y: 44, w: 80, h: 46 };
|
||||
for (const mode of ["solid", "blur"] as const) {
|
||||
const outPath = path.join(root, `redacted-${mode}.mp4`);
|
||||
const result = await provider.redact({
|
||||
path: clip,
|
||||
outPath,
|
||||
redactions: [
|
||||
{
|
||||
keyframes: [
|
||||
{ tMs: 0, box },
|
||||
{ tMs: 2000, box },
|
||||
],
|
||||
mode,
|
||||
paddingPct: 0,
|
||||
featherPct: 0,
|
||||
color: "#102030",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.framesProcessed).toBeGreaterThan(0);
|
||||
const source = await extractRgbFrame(clip, 1000);
|
||||
const redacted = await extractRgbFrame(result.outPath, 1000);
|
||||
expect(redacted.length).toBe(WIDTH * HEIGHT * 3);
|
||||
const diff = regionDiff(source, redacted, pixels);
|
||||
expect(diff.inside, mode).toBeGreaterThan(8);
|
||||
expect(diff.outside, mode).toBeLessThan(1.5);
|
||||
}
|
||||
});
|
||||
|
||||
it("honours AbortSignal", async () => {
|
||||
const aborted = new AbortController();
|
||||
aborted.abort();
|
||||
await expect(
|
||||
provider.detectScenes({ path: clip, signal: aborted.signal }),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
await expect(provider.capabilities(aborted.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
|
||||
const long = await renderTestClip(path.join(root, "long.mp4"), {
|
||||
seconds: 20,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const pending = provider.redact({
|
||||
path: long,
|
||||
outPath: path.join(root, "aborted.mp4"),
|
||||
redactions: [
|
||||
{
|
||||
keyframes: [
|
||||
{ tMs: 0, box: { x: 0.1, y: 0.1, w: 0.3, h: 0.3 } },
|
||||
{ tMs: 20_000, box: { x: 0.5, y: 0.5, w: 0.3, h: 0.3 } },
|
||||
],
|
||||
mode: "blur",
|
||||
paddingPct: 10,
|
||||
featherPct: 0,
|
||||
},
|
||||
],
|
||||
signal: controller.signal,
|
||||
});
|
||||
setTimeout(() => controller.abort(), 150);
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
});
|
||||
});
|
||||
}
|
||||
66
packages/adapters/src/media-emulator-clips.ts
Normal file
66
packages/adapters/src/media-emulator-clips.ts
Normal file
@ -0,0 +1,66 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { runMediaTool } from "./media-folder-source-probe.js";
|
||||
|
||||
/*
|
||||
* Synthetic test clips (ffmpeg testsrc2 + sine) for media tests across the
|
||||
* codebase. Name the output with media-emulator markers to steer emulator
|
||||
* results, e.g. `clip__faces-1__.mp4`.
|
||||
*/
|
||||
|
||||
export function ffmpegAvailable(ffmpegPath = "ffmpeg"): boolean {
|
||||
return spawnSync(ffmpegPath, ["-version"], { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
export interface TestClipOptions {
|
||||
seconds?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
fps?: number;
|
||||
audio?: boolean;
|
||||
ffmpegPath?: string;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export async function renderTestClip(outPath: string, options: TestClipOptions = {}) {
|
||||
const { seconds = 2, width = 320, height = 180, fps = 25, audio = true } = options;
|
||||
const args = [
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
`testsrc2=s=${width}x${height}:r=${fps}:d=${seconds}`,
|
||||
];
|
||||
if (audio)
|
||||
args.push("-f", "lavfi", "-i", `sine=f=440:sample_rate=48000:d=${seconds}`, "-shortest");
|
||||
args.push("-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p");
|
||||
if (audio) args.push("-c:a", "aac");
|
||||
args.push(outPath);
|
||||
await runMediaTool(options.ffmpegPath ?? "ffmpeg", args, { signal: options.signal });
|
||||
return outPath;
|
||||
}
|
||||
|
||||
/** Decode one frame at `tMs` as packed RGB24 bytes. */
|
||||
export async function extractRgbFrame(
|
||||
file: string,
|
||||
tMs: number,
|
||||
options: { ffmpegPath?: string } = {},
|
||||
): Promise<Buffer> {
|
||||
const { stdout } = await runMediaTool(options.ffmpegPath ?? "ffmpeg", [
|
||||
"-v",
|
||||
"error",
|
||||
"-i",
|
||||
file,
|
||||
"-ss",
|
||||
(tMs / 1000).toFixed(3),
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"rgb24",
|
||||
"-",
|
||||
]);
|
||||
return stdout;
|
||||
}
|
||||
208
packages/adapters/src/media-emulator-source.ts
Normal file
208
packages/adapters/src/media-emulator-source.ts
Normal file
@ -0,0 +1,208 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdir, rename, rm, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type {
|
||||
MediaSourceItem,
|
||||
MediaSourceListRequest,
|
||||
MediaSourceListResult,
|
||||
MediaSourceProvider,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import { PathOutsideRootError, resolveInside } from "@rakazo/media/node";
|
||||
import { z } from "zod";
|
||||
import type { MediaToolPaths } from "./media-folder-source-probe.js";
|
||||
import { runMediaTool } from "./media-folder-source-probe.js";
|
||||
|
||||
/*
|
||||
* Deterministic synthetic library for tests and the scripted runtime. Config
|
||||
* `{ count, seed }` describes `count` assets split into three capture sessions
|
||||
* on separate days in three cities (more than 2 km apart), alternating two
|
||||
* visual themes (`__theme-a__` testsrc2, `__theme-b__` SMPTE bars). Every fifth
|
||||
* asset is a photo. File names carry media-emulator markers:
|
||||
* index % 10 == 0 __faces-1____person-a__
|
||||
* index % 10 == 1 __faces-1____person-b__
|
||||
* index == 2 __nsfw-explicit_nudity-0.9__
|
||||
* index % 4 == 3 __speech__ (videos only)
|
||||
* index % 10 == 6 __cut-2000__ (videos only)
|
||||
* `resolve` renders the clip on first use (4 s, 320x180, ffmpeg testsrc2 or
|
||||
* smptehdbars plus a sine tone) and caches it under `fixturesDir/<seed>/`.
|
||||
*/
|
||||
|
||||
const ConfigSchema = z.object({
|
||||
count: z.number().int().min(1).max(100_000),
|
||||
seed: z.number().int().min(0).max(2_147_483_647),
|
||||
});
|
||||
|
||||
const CLIP_MS = 4000;
|
||||
const CLIP_SIZE = { width: 320, height: 180, fps: 25 };
|
||||
const SESSIONS = [
|
||||
{ lat: 52.52, lon: 13.405 },
|
||||
{ lat: 53.5511, lon: 9.9937 },
|
||||
{ lat: 48.1374, lon: 11.5755 },
|
||||
] as const;
|
||||
|
||||
export interface EmulatorAsset {
|
||||
index: number;
|
||||
fileName: string;
|
||||
kind: "video" | "photo";
|
||||
theme: "a" | "b";
|
||||
session: number;
|
||||
capturedAt: Date;
|
||||
gpsLat: number;
|
||||
gpsLon: number;
|
||||
}
|
||||
|
||||
/** The deterministic description of asset `index` in an emulator library. */
|
||||
export function emulatorAsset(seed: number, count: number, index: number): EmulatorAsset {
|
||||
const kind = index % 5 === 4 ? "photo" : "video";
|
||||
const theme = index % 2 === 0 ? "a" : "b";
|
||||
const session = Math.min(SESSIONS.length - 1, Math.floor((index * SESSIONS.length) / count));
|
||||
const firstInSession = Math.ceil((session * count) / SESSIONS.length);
|
||||
const day = Date.UTC(2026, 4, 2 + (seed % 28) + session * 2, 8 + session);
|
||||
const capturedAt = new Date(day + (index - firstInSession) * 3 * 60_000);
|
||||
const place = SESSIONS[session] ?? SESSIONS[0];
|
||||
const jitter = ((index * 7919 + seed) % 100) / 100_000;
|
||||
const markers = [`__theme-${theme}__`];
|
||||
if (index % 10 === 0) markers.push("__faces-1____person-a__");
|
||||
if (index % 10 === 1) markers.push("__faces-1____person-b__");
|
||||
if (index === 2) markers.push("__nsfw-explicit_nudity-0.9__");
|
||||
if (kind === "video" && index % 4 === 3) markers.push("__speech__");
|
||||
if (kind === "video" && index % 10 === 6) markers.push("__cut-2000__");
|
||||
const ext = kind === "photo" ? "jpg" : "mp4";
|
||||
return {
|
||||
index,
|
||||
fileName: `emu-${seed}-${String(index).padStart(5, "0")}${markers.join("")}.${ext}`,
|
||||
kind,
|
||||
theme,
|
||||
session,
|
||||
capturedAt,
|
||||
gpsLat: Math.round((place.lat + jitter) * 1e5) / 1e5,
|
||||
gpsLon: Math.round((place.lon + jitter) * 1e5) / 1e5,
|
||||
};
|
||||
}
|
||||
|
||||
function iso6709(lat: number, lon: number): string {
|
||||
const sign = (value: number, width: number) =>
|
||||
`${value < 0 ? "-" : "+"}${Math.abs(value).toFixed(4).padStart(width, "0")}`;
|
||||
return `${sign(lat, 7)}${sign(lon, 8)}/`;
|
||||
}
|
||||
|
||||
export interface EmulatorMediaSourceOptions extends MediaToolPaths {
|
||||
/** Cache for rendered clips; must lie inside the media root to be analysable. */
|
||||
fixturesDir: string;
|
||||
}
|
||||
|
||||
export class EmulatorMediaSourceProvider implements MediaSourceProvider {
|
||||
readonly kind = "emulator" as const;
|
||||
|
||||
constructor(private readonly options: EmulatorMediaSourceOptions) {}
|
||||
|
||||
async list(request: MediaSourceListRequest): Promise<MediaSourceListResult> {
|
||||
request.signal?.throwIfAborted();
|
||||
const { count, seed } = ConfigSchema.parse(request.config);
|
||||
const start = request.cursor ? decodeIndex(request.cursor) : 0;
|
||||
const end = Math.min(count, start + Math.max(1, Math.floor(request.limit)));
|
||||
const items: MediaSourceItem[] = [];
|
||||
for (let index = start; index < end; index++) {
|
||||
const asset = emulatorAsset(seed, count, index);
|
||||
items.push({
|
||||
externalId: asset.fileName,
|
||||
kind: asset.kind,
|
||||
capturedAt: asset.capturedAt,
|
||||
durationMs: asset.kind === "video" ? CLIP_MS : null,
|
||||
width: CLIP_SIZE.width,
|
||||
height: CLIP_SIZE.height,
|
||||
fps: asset.kind === "video" ? CLIP_SIZE.fps : null,
|
||||
gpsLat: asset.gpsLat,
|
||||
gpsLon: asset.gpsLon,
|
||||
deviceModel: "Emulator",
|
||||
albumNames: [],
|
||||
isFavorite: index % 9 === 0,
|
||||
fileName: asset.fileName,
|
||||
sizeBytes: null,
|
||||
revision: `emulator-v1-${seed}`,
|
||||
});
|
||||
}
|
||||
return {
|
||||
items,
|
||||
nextCursor: end < count ? Buffer.from(String(end)).toString("base64url") : null,
|
||||
};
|
||||
}
|
||||
|
||||
async resolve(config: Record<string, unknown>, externalId: string): Promise<string> {
|
||||
const { count, seed } = ConfigSchema.parse(config);
|
||||
const match = /^emu-(\d+)-(\d{5})/.exec(externalId);
|
||||
const index = match ? Number(match[2]) : -1;
|
||||
if (!match || Number(match[1]) !== seed || index >= count)
|
||||
throw new PathOutsideRootError(externalId);
|
||||
const asset = emulatorAsset(seed, count, index);
|
||||
if (asset.fileName !== externalId) throw new PathOutsideRootError(externalId);
|
||||
await mkdir(path.join(this.options.fixturesDir, String(seed)), { recursive: true });
|
||||
const target = await resolveInside(this.options.fixturesDir, `${seed}/${asset.fileName}`);
|
||||
if (await exists(target)) return target;
|
||||
const partial = path.join(
|
||||
path.dirname(target),
|
||||
`.partial-${randomUUID()}.${asset.kind === "photo" ? "jpg" : "mp4"}`,
|
||||
);
|
||||
try {
|
||||
await runMediaTool(this.options.ffmpegPath ?? "ffmpeg", renderArgs(asset, partial));
|
||||
await rename(partial, target);
|
||||
} finally {
|
||||
await rm(partial, { force: true });
|
||||
}
|
||||
return target;
|
||||
}
|
||||
}
|
||||
|
||||
function renderArgs(asset: EmulatorAsset, out: string): string[] {
|
||||
const { width, height, fps } = CLIP_SIZE;
|
||||
const pattern = asset.theme === "a" ? "testsrc2" : "smptehdbars";
|
||||
const video = `${pattern}=s=${width}x${height}:r=${fps}:d=${CLIP_MS / 1000}`;
|
||||
const metadata = [
|
||||
"-metadata",
|
||||
`creation_time=${asset.capturedAt.toISOString()}`,
|
||||
"-metadata",
|
||||
`location=${iso6709(asset.gpsLat, asset.gpsLon)}`,
|
||||
];
|
||||
if (asset.kind === "photo")
|
||||
return ["-v", "error", "-y", "-f", "lavfi", "-i", video, "-frames:v", "1", out];
|
||||
return [
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
video,
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
`sine=f=${220 + (asset.index % 12) * 40}:sample_rate=48000:d=${CLIP_MS / 1000}`,
|
||||
"-shortest",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"aac",
|
||||
...metadata,
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
out,
|
||||
];
|
||||
}
|
||||
|
||||
function decodeIndex(cursor: string): number {
|
||||
const value = Number(Buffer.from(cursor, "base64url").toString("utf8"));
|
||||
if (!Number.isInteger(value) || value < 0) throw new Error("invalid emulator cursor");
|
||||
return value;
|
||||
}
|
||||
|
||||
async function exists(file: string): Promise<boolean> {
|
||||
try {
|
||||
return (await stat(file)).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
335
packages/adapters/src/media-emulator.test.ts
Normal file
335
packages/adapters/src/media-emulator.test.ts
Normal file
@ -0,0 +1,335 @@
|
||||
import { copyFile, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||
import { cosineSimilarity, haversineKm } from "@rakazo/media";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildRedactionGraph,
|
||||
EmulatorMediaInferenceProvider,
|
||||
parseEmulatorMarkers,
|
||||
redactionRegions,
|
||||
} from "./media-emulator.js";
|
||||
import { ffmpegAvailable, renderTestClip } from "./media-emulator-clips.js";
|
||||
import { EmulatorMediaSourceProvider, emulatorAsset } from "./media-emulator-source.js";
|
||||
import { probeMedia } from "./media-folder-source-probe.js";
|
||||
|
||||
const hasFfmpeg = ffmpegAvailable();
|
||||
let root: string;
|
||||
let provider: EmulatorMediaInferenceProvider;
|
||||
|
||||
beforeAll(async () => {
|
||||
root = await mkdtemp(path.join(tmpdir(), "media-emulator-"));
|
||||
provider = new EmulatorMediaInferenceProvider({ root });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** A non-media file: the emulator falls back to a 10 s clip, so no ffmpeg is needed. */
|
||||
async function fixture(name: string, content = "synthetic bytes") {
|
||||
const file = path.join(root, name);
|
||||
await writeFile(file, content);
|
||||
return file;
|
||||
}
|
||||
|
||||
describe("parseEmulatorMarkers", () => {
|
||||
it("reads every documented marker", () => {
|
||||
expect(
|
||||
parseEmulatorMarkers(
|
||||
"/x/clip__faces-2____person-a____noface-gap-1000-2500____nsfw-explicit_nudity-0.9____nsfw-violence-0.4____cut-1500____cut-3000____speech____theme-b____inpaintq-0.3__.mp4",
|
||||
),
|
||||
).toEqual({
|
||||
faces: 2,
|
||||
person: "a",
|
||||
noFaceGaps: [{ startMs: 1000, endMs: 2500 }],
|
||||
nsfw: { explicit_nudity: 0.9, violence: 0.4 },
|
||||
cutsMs: [1500, 3000],
|
||||
speech: true,
|
||||
theme: "b",
|
||||
inpaintQuality: 0.3,
|
||||
});
|
||||
expect(parseEmulatorMarkers("__nsfw-unknown-0.5____nsfw-violence-7__.mp4").nsfw).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe("redaction regions", () => {
|
||||
it("holds the union of neighbouring keyframes and bounds the graph", () => {
|
||||
const regions = redactionRegions(
|
||||
{
|
||||
keyframes: [
|
||||
{ tMs: 1000, box: { x: 0.5, y: 0.5, w: 0.1, h: 0.1 } },
|
||||
{ tMs: 0, box: { x: 0.1, y: 0.1, w: 0.1, h: 0.1 } },
|
||||
],
|
||||
mode: "blur",
|
||||
paddingPct: 0,
|
||||
featherPct: 0,
|
||||
},
|
||||
1000,
|
||||
1000,
|
||||
);
|
||||
expect(regions).toEqual([
|
||||
{ startMs: 0, endMs: 1000, box: { x: 100, y: 100, w: 500, h: 500 } },
|
||||
{ startMs: 1000, endMs: 1100, box: { x: 500, y: 500, w: 100, h: 100 } },
|
||||
]);
|
||||
const keyframes = Array.from({ length: 5000 }, (_, i) => ({
|
||||
tMs: i * 40,
|
||||
box: { x: (i % 50) / 100, y: 0.2, w: 0.1, h: 0.1 },
|
||||
}));
|
||||
const many = redactionRegions(
|
||||
{ keyframes, mode: "pixelate", paddingPct: 25, featherPct: 0 },
|
||||
1920,
|
||||
1080,
|
||||
);
|
||||
expect(many.length).toBeLessThanOrEqual(400);
|
||||
expect(many[0]?.startMs).toBe(0);
|
||||
expect(many.at(-1)?.endMs).toBe(4999 * 40 + 100);
|
||||
const graph = buildRedactionGraph(
|
||||
[{ keyframes, mode: "solid", paddingPct: 0, featherPct: 0, color: "#ff0000" }],
|
||||
1920,
|
||||
1080,
|
||||
);
|
||||
expect(graph.filter).toContain("color=0xff0000@1:t=fill");
|
||||
});
|
||||
});
|
||||
|
||||
describe("EmulatorMediaInferenceProvider", () => {
|
||||
it("is quiet without markers", async () => {
|
||||
const file = await fixture("plain.mp4");
|
||||
const scenes = await provider.detectScenes({ path: file });
|
||||
expect(scenes.scenes).toEqual([{ startMs: 0, endMs: 10_000 }]);
|
||||
expect(scenes.shots).toHaveLength(1);
|
||||
expect((await provider.trackFaces({ path: file, sampleFps: 2 })).tracks).toEqual([]);
|
||||
const faces = await provider.detectFaces({ path: file, sampleFps: 1 });
|
||||
expect(faces.frames).toHaveLength(10);
|
||||
expect(faces.frames.every((frame) => frame.faces.length === 0)).toBe(true);
|
||||
const classified = await provider.classifyContent({ path: file, sampleFps: 2 });
|
||||
expect(classified.frames).toHaveLength(20);
|
||||
for (const frame of classified.frames)
|
||||
expect(Object.values(frame.scores).every((score) => score === 0.01)).toBe(true);
|
||||
expect(await provider.transcribe({ path: file })).toEqual({ language: null, words: [] });
|
||||
});
|
||||
|
||||
it("forces faces, gaps, cuts, classifier scores and speech from the file name", async () => {
|
||||
const file = await fixture(
|
||||
"rich__faces-2____noface-gap-2000-3000____cut-4000____nsfw-explicit_nudity-0.9____speech__.mp4",
|
||||
);
|
||||
const tracks = await provider.trackFaces({ path: file, sampleFps: 2 });
|
||||
expect(tracks.tracks.map((track) => track.trackId)).toEqual(["track-1", "track-2"]);
|
||||
const times = tracks.tracks[0]?.keyframes.map((keyframe) => keyframe.tMs) ?? [];
|
||||
expect(times).toContain(1500);
|
||||
expect(times).not.toContain(2000);
|
||||
expect(times).not.toContain(2500);
|
||||
expect(times).toContain(3000);
|
||||
const gapFrame = await provider.detectFaces({
|
||||
path: file,
|
||||
sampleFps: 1,
|
||||
timesMs: [2500, 3500],
|
||||
});
|
||||
expect(gapFrame.frames.map((frame) => frame.faces.length)).toEqual([0, 2]);
|
||||
|
||||
expect((await provider.detectScenes({ path: file })).scenes).toEqual([
|
||||
{ startMs: 0, endMs: 4000 },
|
||||
{ startMs: 4000, endMs: 10_000 },
|
||||
]);
|
||||
const classified = await provider.classifyContent({
|
||||
path: file,
|
||||
sampleFps: 0.5,
|
||||
extraTimesMs: [4000, 4001],
|
||||
});
|
||||
expect(classified.frames.map((frame) => frame.tMs)).toEqual([0, 2000, 4000, 4001, 6000, 8000]);
|
||||
for (const frame of classified.frames) {
|
||||
expect(frame.scores.explicit_nudity).toBe(0.9);
|
||||
expect(frame.scores.violence).toBe(0.01);
|
||||
}
|
||||
const english = await provider.transcribe({ path: file });
|
||||
const german = await provider.transcribe({ path: file, language: "de-DE" });
|
||||
expect(english.language).toBe("en");
|
||||
expect(english.words.map((word) => word.text).slice(0, 2)).toEqual(["welcome", "back"]);
|
||||
expect(german.language).toBe("de");
|
||||
expect(german.words[0]).toEqual({
|
||||
startMs: 400,
|
||||
endMs: 720,
|
||||
text: "willkommen",
|
||||
confidence: 0.95,
|
||||
});
|
||||
});
|
||||
|
||||
it("is deterministic per file content and keeps boxes stable", async () => {
|
||||
const file = await fixture("same__faces-1__.mp4");
|
||||
const copy = path.join(root, "copy__faces-1__.mp4");
|
||||
await copyFile(file, copy);
|
||||
const a = await provider.detectFaces({ path: file, sampleFps: 4 });
|
||||
const b = await provider.detectFaces({ path: file, sampleFps: 4 });
|
||||
const c = await provider.detectFaces({ path: copy, sampleFps: 4 });
|
||||
expect(a).toEqual(b);
|
||||
expect(a).toEqual(c);
|
||||
const boxes = a.frames.map((frame) => frame.faces[0]?.box);
|
||||
expect(Math.abs((boxes.at(-1)?.x ?? 0) - (boxes[0]?.x ?? 0))).toBeLessThan(0.05);
|
||||
});
|
||||
|
||||
it("embeds people and themes consistently", async () => {
|
||||
const personA1 = await fixture("one__faces-1____person-a__.mp4", "first");
|
||||
const personA2 = await fixture("two__faces-1____person-a__.mp4", "second");
|
||||
const personB = await fixture("three__faces-1____person-b__.mp4", "third");
|
||||
const embed = async (file: string) =>
|
||||
(await provider.trackFaces({ path: file, sampleFps: 1 })).tracks[0]?.embedding ?? [];
|
||||
expect(cosineSimilarity(await embed(personA1), await embed(personA2))).toBeCloseTo(1, 5);
|
||||
expect(cosineSimilarity(await embed(personA1), await embed(personB))).toBeLessThan(0.8);
|
||||
|
||||
const themeA1 = await fixture("x__theme-a__.mp4", "x");
|
||||
const themeA2 = await fixture("y__theme-a__.mp4", "y");
|
||||
const themeB = await fixture("z__theme-b__.mp4", "z");
|
||||
const mean = async (file: string) =>
|
||||
(await provider.embedFrames({ path: file, timesMs: [0, 1000] })).vectors[0] ?? [];
|
||||
expect(cosineSimilarity(await mean(themeA1), await mean(themeA2))).toBeGreaterThan(0.95);
|
||||
expect(cosineSimilarity(await mean(themeA1), await mean(themeB))).toBeLessThan(0.85);
|
||||
});
|
||||
|
||||
it("never synthesizes identities", async () => {
|
||||
const file = await fixture("synth.mp4");
|
||||
const capabilities = await provider.capabilities();
|
||||
expect(capabilities.operations.synthesizeIdentity).toMatchObject({
|
||||
available: false,
|
||||
license: null,
|
||||
});
|
||||
expect(capabilities.operations.inpaintVideo).toMatchObject({
|
||||
available: true,
|
||||
experimental: true,
|
||||
});
|
||||
await expect(
|
||||
provider.synthesizeIdentity({
|
||||
path: file,
|
||||
track: [],
|
||||
seed: 1,
|
||||
outPath: path.join(root, "s.mp4"),
|
||||
}),
|
||||
).rejects.toBeInstanceOf(UnsupportedMediaOperationError);
|
||||
});
|
||||
|
||||
it("rejects paths outside its root", async () => {
|
||||
await expect(provider.detectScenes({ path: "/etc/hostname" })).rejects.toThrow(
|
||||
"outside the allowed root",
|
||||
);
|
||||
await expect(provider.detectScenes({ path: "relative.mp4" })).rejects.toThrow();
|
||||
});
|
||||
|
||||
it.skipIf(!hasFfmpeg)("writes masks and inpaints with the reported quality", async () => {
|
||||
const clip = await renderTestClip(path.join(root, "mark__inpaintq-0.4__.mp4"), {
|
||||
seconds: 1,
|
||||
audio: false,
|
||||
});
|
||||
const masks = await provider.segmentPrompted({
|
||||
path: clip,
|
||||
prompt: { tMs: 0, point: { x: 0.5, y: 0.5 } },
|
||||
propagate: true,
|
||||
outDir: path.join(root, "marks/m1"),
|
||||
});
|
||||
expect((await readdir(masks.maskDir)).sort()).toEqual(["0.png", "500.png"]);
|
||||
expect(masks.keyframes[0]?.box).toEqual({ x: 0.44, y: 0.44, w: 0.12, h: 0.12 });
|
||||
const inpainted = await provider.inpaintVideo({
|
||||
path: clip,
|
||||
maskDir: masks.maskDir,
|
||||
outPath: path.join(root, "marks/m1.mp4"),
|
||||
});
|
||||
expect(inpainted.quality.score).toBe(0.4);
|
||||
expect((await probeMedia(inpainted.outPath)).durationMs).toBeGreaterThan(900);
|
||||
});
|
||||
|
||||
it.skipIf(!hasFfmpeg)("renders very large redaction graphs through a script file", async () => {
|
||||
const clip = await renderTestClip(path.join(root, "busy.mp4"), { seconds: 1, audio: false });
|
||||
const instruction = (offset: number) => ({
|
||||
keyframes: Array.from({ length: 2000 }, (_, i) => ({
|
||||
tMs: i,
|
||||
box: { x: offset + (i % 40) / 400, y: 0.1, w: 0.1, h: 0.1 },
|
||||
})),
|
||||
mode: "blur" as const,
|
||||
paddingPct: 0,
|
||||
featherPct: 0,
|
||||
});
|
||||
const instructions = [instruction(0), instruction(0.3), instruction(0.6)];
|
||||
expect(buildRedactionGraph(instructions, 320, 180).filter.length).toBeGreaterThan(100_000);
|
||||
const result = await provider.redact({
|
||||
path: clip,
|
||||
redactions: instructions,
|
||||
outPath: path.join(root, "busy-redacted.mp4"),
|
||||
});
|
||||
expect(result.framesProcessed).toBe(25);
|
||||
});
|
||||
|
||||
it.skipIf(!hasFfmpeg)("measures loudness with ebur128", async () => {
|
||||
const clip = await renderTestClip(path.join(root, "tone.mp4"), { seconds: 2 });
|
||||
const features = await provider.audioFeatures({ path: clip });
|
||||
expect(features.integratedLufs).toBeLessThan(-10);
|
||||
expect(features.integratedLufs).toBeGreaterThan(-40);
|
||||
expect(features.energy.length).toBeGreaterThan(0);
|
||||
expect(features.beatsMs.length).toBeGreaterThan(0);
|
||||
const silent = await renderTestClip(path.join(root, "mute.mp4"), { seconds: 1, audio: false });
|
||||
expect(await provider.audioFeatures({ path: silent })).toMatchObject({
|
||||
integratedLufs: null,
|
||||
silences: [{ startMs: 0, endMs: 1000 }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("EmulatorMediaSourceProvider", () => {
|
||||
const config = { count: 30, seed: 4 };
|
||||
|
||||
it("describes three sessions, two themes and the marker assets", async () => {
|
||||
const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") });
|
||||
const first = await source.list({ config, cursor: null, limit: 12 });
|
||||
const second = await source.list({ config, cursor: first.nextCursor, limit: 100 });
|
||||
expect(second.nextCursor).toBeNull();
|
||||
const items = [...first.items, ...second.items];
|
||||
expect(items).toHaveLength(30);
|
||||
expect(new Set(items.map((item) => item.externalId)).size).toBe(30);
|
||||
expect(await source.list({ config, cursor: null, limit: 12 })).toEqual(first);
|
||||
|
||||
const assets = items.map((_, index) => emulatorAsset(config.seed, config.count, index));
|
||||
expect(new Set(assets.map((asset) => asset.session))).toEqual(new Set([0, 1, 2]));
|
||||
expect(new Set(assets.map((asset) => asset.theme))).toEqual(new Set(["a", "b"]));
|
||||
const sessionStarts = [0, 1, 2].map(
|
||||
(session) => assets.find((asset) => asset.session === session)!,
|
||||
);
|
||||
expect(
|
||||
haversineKm(
|
||||
{ lat: sessionStarts[0]!.gpsLat, lon: sessionStarts[0]!.gpsLon },
|
||||
{ lat: sessionStarts[1]!.gpsLat, lon: sessionStarts[1]!.gpsLon },
|
||||
),
|
||||
).toBeGreaterThan(2);
|
||||
const names = items.map((item) => item.externalId).join(" ");
|
||||
expect(names).toContain("__faces-1____person-a__");
|
||||
expect(names).toContain("__faces-1____person-b__");
|
||||
expect(names).toContain("__nsfw-explicit_nudity-0.9__");
|
||||
expect(names).toContain("__speech__");
|
||||
expect(items.some((item) => item.kind === "photo")).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects unknown external ids", async () => {
|
||||
const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") });
|
||||
for (const bad of [
|
||||
"../etc/passwd",
|
||||
"emu-4-00099__theme-b__.mp4",
|
||||
"emu-5-00000__theme-a__.mp4",
|
||||
"emu-4-00000.mp4",
|
||||
])
|
||||
await expect(source.resolve(config, bad), bad).rejects.toThrow();
|
||||
});
|
||||
|
||||
it.skipIf(!hasFfmpeg)("renders and caches clips on demand", async () => {
|
||||
const source = new EmulatorMediaSourceProvider({ fixturesDir: path.join(root, "library") });
|
||||
const asset = emulatorAsset(config.seed, config.count, 0);
|
||||
const file = await source.resolve(config, asset.fileName);
|
||||
expect(path.basename(file)).toBe(asset.fileName);
|
||||
const probed = await probeMedia(file);
|
||||
expect(probed).toMatchObject({ durationMs: 4000, width: 320, height: 180, hasAudio: true });
|
||||
expect(probed.capturedAt?.toISOString()).toBe(asset.capturedAt.toISOString());
|
||||
expect(await source.resolve(config, asset.fileName)).toBe(file);
|
||||
const tracks = await provider.trackFaces({ path: file, sampleFps: 1 });
|
||||
expect(tracks.tracks).toHaveLength(1);
|
||||
const photo = emulatorAsset(config.seed, config.count, 4);
|
||||
expect(photo.kind).toBe("photo");
|
||||
expect((await probeMedia(await source.resolve(config, photo.fileName))).width).toBe(320);
|
||||
});
|
||||
});
|
||||
1003
packages/adapters/src/media-emulator.ts
Normal file
1003
packages/adapters/src/media-emulator.ts
Normal file
File diff suppressed because it is too large
Load Diff
53
packages/adapters/src/media-factory.test.ts
Normal file
53
packages/adapters/src/media-factory.test.ts
Normal file
@ -0,0 +1,53 @@
|
||||
import { homedir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
createMediaInferenceProvider,
|
||||
createMediaSourceProvider,
|
||||
mediaConfigFromEnv,
|
||||
resolveMediaInferenceKind,
|
||||
} from "./media-factory.js";
|
||||
|
||||
const worker = {
|
||||
MEDIA_WORKER_URL: "http://127.0.0.1:8790",
|
||||
MEDIA_WORKER_TOKEN: "placeholder-token-placeholder-token-00",
|
||||
};
|
||||
|
||||
describe("media factory", () => {
|
||||
it("reads one generic worker connection and derives paths", () => {
|
||||
expect(mediaConfigFromEnv({ DATA_DIR: "/srv/evoo", ...worker })).toEqual({
|
||||
enabled: true,
|
||||
workerUrl: worker.MEDIA_WORKER_URL,
|
||||
workerToken: worker.MEDIA_WORKER_TOKEN,
|
||||
profilesDir: path.join(homedir(), ".evoo/media-profiles"),
|
||||
mediaRoot: "/srv/evoo/media",
|
||||
});
|
||||
expect(
|
||||
mediaConfigFromEnv({ DATA_DIR: "/srv/evoo", MEDIA_PROFILES_DIR: "/etc/evoo/profiles" }),
|
||||
).toMatchObject({ enabled: false, workerUrl: null, profilesDir: "/etc/evoo/profiles" });
|
||||
});
|
||||
|
||||
it("selects the sidecar only with URL and token", () => {
|
||||
expect(resolveMediaInferenceKind(worker)).toBe("sidecar");
|
||||
expect(createMediaInferenceProvider(worker)?.id).toBe("sidecar");
|
||||
expect(resolveMediaInferenceKind({ MEDIA_WORKER_URL: worker.MEDIA_WORKER_URL })).toBeNull();
|
||||
expect(createMediaInferenceProvider({})).toBeNull();
|
||||
});
|
||||
|
||||
it("pins the emulator under the scripted runtime", () => {
|
||||
expect(resolveMediaInferenceKind({ ...worker, AGENT_RUNTIME: "scripted" })).toBe("emulator");
|
||||
expect(createMediaInferenceProvider({ AGENT_RUNTIME: "scripted" })?.id).toBe("emulator");
|
||||
expect(mediaConfigFromEnv({ AGENT_RUNTIME: "scripted" }).enabled).toBe(true);
|
||||
// pin-test-env sets AGENT_RUNTIME=scripted for every unit test.
|
||||
expect(createMediaInferenceProvider()?.id).toBe("emulator");
|
||||
});
|
||||
|
||||
it("creates server-side sources only", () => {
|
||||
expect(createMediaSourceProvider("folder", {}).kind).toBe("folder");
|
||||
expect(createMediaSourceProvider("emulator", { AGENT_RUNTIME: "scripted" }).kind).toBe(
|
||||
"emulator",
|
||||
);
|
||||
expect(() => createMediaSourceProvider("emulator", {})).toThrow();
|
||||
expect(() => createMediaSourceProvider("apple_photos", {})).toThrow();
|
||||
});
|
||||
});
|
||||
90
packages/adapters/src/media-factory.ts
Normal file
90
packages/adapters/src/media-factory.ts
Normal file
@ -0,0 +1,90 @@
|
||||
import { homedir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { MediaInferenceProvider, MediaSourceProvider } from "@rakazo/adapter-kit";
|
||||
import { expandHome } from "@rakazo/media/node";
|
||||
import { EmulatorMediaInferenceProvider } from "./media-emulator.js";
|
||||
import { EmulatorMediaSourceProvider } from "./media-emulator-source.js";
|
||||
import { FolderMediaSourceProvider } from "./media-folder-source.js";
|
||||
import { SidecarMediaInferenceProvider } from "./media-sidecar.js";
|
||||
|
||||
export interface MediaConfig {
|
||||
/** Media Studio runs: a sidecar is configured or the scripted runtime uses the emulator. */
|
||||
enabled: boolean;
|
||||
workerUrl: string | null;
|
||||
workerToken: string | null;
|
||||
profilesDir: string;
|
||||
/** `${DATA_DIR}/media`, absolute. Every derived file and sidecar path lives here. */
|
||||
mediaRoot: string;
|
||||
}
|
||||
|
||||
export type MediaInferenceKind = "sidecar" | "emulator";
|
||||
|
||||
/** Emulators replace the sidecar whenever the scripted runtime is pinned (tests, e2e). */
|
||||
export function mediaEmulatorEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
return env.AGENT_RUNTIME === "scripted";
|
||||
}
|
||||
|
||||
export function mediaConfigFromEnv(env: NodeJS.ProcessEnv = process.env): MediaConfig {
|
||||
const workerUrl = env.MEDIA_WORKER_URL?.trim() || null;
|
||||
const workerToken = env.MEDIA_WORKER_TOKEN?.trim() || null;
|
||||
return {
|
||||
enabled: mediaEmulatorEnabled(env) || Boolean(workerUrl && workerToken),
|
||||
workerUrl,
|
||||
workerToken,
|
||||
profilesDir: path.resolve(
|
||||
expandHome(env.MEDIA_PROFILES_DIR?.trim() || "~/.evoo/media-profiles", homedir()),
|
||||
),
|
||||
mediaRoot: path.resolve(env.DATA_DIR ?? "./data", "media"),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveMediaInferenceKind(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): MediaInferenceKind | null {
|
||||
if (mediaEmulatorEnabled(env)) return "emulator";
|
||||
const config = mediaConfigFromEnv(env);
|
||||
return config.workerUrl && config.workerToken ? "sidecar" : null;
|
||||
}
|
||||
|
||||
/** The configured inference provider, or null when Media Studio is not set up. */
|
||||
export function createMediaInferenceProvider(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): MediaInferenceProvider | null {
|
||||
const config = mediaConfigFromEnv(env);
|
||||
switch (resolveMediaInferenceKind(env)) {
|
||||
case "emulator":
|
||||
return new EmulatorMediaInferenceProvider({ root: config.mediaRoot });
|
||||
case "sidecar":
|
||||
return new SidecarMediaInferenceProvider({
|
||||
baseUrl: config.workerUrl as string,
|
||||
token: config.workerToken as string,
|
||||
});
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Emulator clips are cached inside the media root so inference can read them. */
|
||||
export function emulatorFixturesDir(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return path.join(mediaConfigFromEnv(env).mediaRoot, "emulator-library");
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-side source adapter for a persisted MediaSource kind. Client-pushed
|
||||
* kinds (apple_photos, ios_library) have no server adapter.
|
||||
*/
|
||||
export function createMediaSourceProvider(
|
||||
kind: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): MediaSourceProvider {
|
||||
switch (kind) {
|
||||
case "folder":
|
||||
return new FolderMediaSourceProvider();
|
||||
case "emulator":
|
||||
if (!mediaEmulatorEnabled(env)) break;
|
||||
return new EmulatorMediaSourceProvider({ fixturesDir: emulatorFixturesDir(env) });
|
||||
default:
|
||||
break;
|
||||
}
|
||||
throw new Error(`No server-side media source for "${kind}". Use folder.`);
|
||||
}
|
||||
194
packages/adapters/src/media-folder-source-probe.ts
Normal file
194
packages/adapters/src/media-folder-source-probe.ts
Normal file
@ -0,0 +1,194 @@
|
||||
import { execFile } from "node:child_process";
|
||||
|
||||
/*
|
||||
* Local ffmpeg/ffprobe helpers shared by the folder source and the media
|
||||
* emulator. Every call is an argv array without a shell; nothing here builds a
|
||||
* command string.
|
||||
*/
|
||||
|
||||
export class MediaToolMissingError extends Error {
|
||||
constructor(readonly tool: string) {
|
||||
super(`${tool} is not installed or not on PATH; install ffmpeg to use local media processing`);
|
||||
this.name = "MediaToolMissingError";
|
||||
}
|
||||
}
|
||||
|
||||
export interface MediaToolPaths {
|
||||
ffmpegPath?: string;
|
||||
ffprobePath?: string;
|
||||
}
|
||||
|
||||
/** Run a media tool and collect stdout (binary) and stderr. Rejects on non-zero exit. */
|
||||
export function runMediaTool(
|
||||
bin: string,
|
||||
args: string[],
|
||||
options: { signal?: AbortSignal; maxBuffer?: number } = {},
|
||||
): Promise<{ stdout: Buffer; stderr: string }> {
|
||||
options.signal?.throwIfAborted();
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
bin,
|
||||
args,
|
||||
{
|
||||
encoding: "buffer",
|
||||
maxBuffer: options.maxBuffer ?? 64 * 1024 * 1024,
|
||||
signal: options.signal,
|
||||
shell: false,
|
||||
windowsHide: true,
|
||||
},
|
||||
(error, stdout, stderr) => {
|
||||
if (options.signal?.aborted) {
|
||||
reject(options.signal.reason);
|
||||
return;
|
||||
}
|
||||
if (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
reject(new MediaToolMissingError(bin));
|
||||
return;
|
||||
}
|
||||
const tail = stderr.toString("utf8").trim().split("\n").slice(-3).join(" ");
|
||||
reject(new Error(`${bin} failed: ${tail || error.message}`));
|
||||
return;
|
||||
}
|
||||
resolve({ stdout, stderr: stderr.toString("utf8") });
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export interface ProbedMedia {
|
||||
durationMs: number | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
fps: number | null;
|
||||
codec: string | null;
|
||||
rotation: number;
|
||||
hasAudio: boolean;
|
||||
hasVideo: boolean;
|
||||
capturedAt: Date | null;
|
||||
gpsLat: number | null;
|
||||
gpsLon: number | null;
|
||||
deviceModel: string | null;
|
||||
}
|
||||
|
||||
interface FfprobeStream {
|
||||
codec_type?: string;
|
||||
codec_name?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
avg_frame_rate?: string;
|
||||
r_frame_rate?: string;
|
||||
duration?: string;
|
||||
tags?: Record<string, string>;
|
||||
side_data_list?: Array<{ rotation?: number }>;
|
||||
}
|
||||
|
||||
interface FfprobeOutput {
|
||||
streams?: FfprobeStream[];
|
||||
format?: { duration?: string; tags?: Record<string, string> };
|
||||
frames?: Array<{ tags?: Record<string, string> }>;
|
||||
}
|
||||
|
||||
/** Probe a local file. Throws when ffprobe is missing or cannot read the file. */
|
||||
export async function probeMedia(
|
||||
file: string,
|
||||
options: { ffprobePath?: string; signal?: AbortSignal; frameTags?: boolean } = {},
|
||||
): Promise<ProbedMedia> {
|
||||
const args = ["-v", "error", "-print_format", "json", "-show_format", "-show_streams"];
|
||||
if (options.frameTags) args.push("-show_frames", "-read_intervals", "%+#1");
|
||||
args.push("--", file);
|
||||
const { stdout } = await runMediaTool(options.ffprobePath ?? "ffprobe", args, {
|
||||
signal: options.signal,
|
||||
});
|
||||
return parseFfprobe(JSON.parse(stdout.toString("utf8")) as FfprobeOutput);
|
||||
}
|
||||
|
||||
export function parseFfprobe(output: FfprobeOutput): ProbedMedia {
|
||||
const streams = output.streams ?? [];
|
||||
const video = streams.find((stream) => stream.codec_type === "video");
|
||||
const tags = collectTags(output);
|
||||
const rotation = video ? streamRotation(video) : 0;
|
||||
const quarterTurn = Math.abs(rotation) % 180 === 90;
|
||||
const durationSeconds = Number(output.format?.duration ?? video?.duration);
|
||||
const gps = parseIso6709(
|
||||
tags.get("com.apple.quicktime.location.iso6709") ?? tags.get("location"),
|
||||
);
|
||||
return {
|
||||
durationMs:
|
||||
Number.isFinite(durationSeconds) && durationSeconds > 0
|
||||
? Math.round(durationSeconds * 1000)
|
||||
: null,
|
||||
width: (quarterTurn ? video?.height : video?.width) ?? null,
|
||||
height: (quarterTurn ? video?.width : video?.height) ?? null,
|
||||
fps: video ? (parseRate(video.avg_frame_rate) ?? parseRate(video.r_frame_rate)) : null,
|
||||
codec: video?.codec_name ?? null,
|
||||
rotation,
|
||||
hasAudio: streams.some((stream) => stream.codec_type === "audio"),
|
||||
hasVideo: Boolean(video),
|
||||
capturedAt: parseCaptureDate(
|
||||
tags.get("com.apple.quicktime.creationdate") ??
|
||||
tags.get("datetimeoriginal") ??
|
||||
tags.get("creation_time") ??
|
||||
tags.get("datetime"),
|
||||
),
|
||||
gpsLat: gps?.lat ?? null,
|
||||
gpsLon: gps?.lon ?? null,
|
||||
deviceModel: tags.get("com.apple.quicktime.model") ?? tags.get("model") ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Format, stream and first-frame tags merged with lowercase keys; format wins. */
|
||||
function collectTags(output: FfprobeOutput): Map<string, string> {
|
||||
const merged = new Map<string, string>();
|
||||
const sources = [
|
||||
output.format?.tags,
|
||||
...(output.streams ?? []).map((stream) => stream.tags),
|
||||
...(output.frames ?? []).map((frame) => frame.tags),
|
||||
];
|
||||
for (const source of sources) {
|
||||
for (const [key, value] of Object.entries(source ?? {})) {
|
||||
const normalized = key.toLowerCase();
|
||||
if (!merged.has(normalized) && typeof value === "string" && value.trim())
|
||||
merged.set(normalized, value.trim());
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
function streamRotation(stream: FfprobeStream): number {
|
||||
const side = stream.side_data_list?.find((entry) => typeof entry.rotation === "number");
|
||||
if (side?.rotation !== undefined) return side.rotation;
|
||||
const tagged = Number(stream.tags?.rotate);
|
||||
return Number.isFinite(tagged) ? tagged : 0;
|
||||
}
|
||||
|
||||
function parseRate(rate: string | undefined): number | null {
|
||||
if (!rate) return null;
|
||||
const [num, den] = rate.split("/").map(Number);
|
||||
if (!num || !Number.isFinite(num)) return null;
|
||||
const value = den ? num / den : num;
|
||||
return Number.isFinite(value) && value > 0 ? Math.round(value * 1000) / 1000 : null;
|
||||
}
|
||||
|
||||
/** ISO 8601 or EXIF ("2026:05:01 09:00:00") timestamps; null when unparseable. */
|
||||
export function parseCaptureDate(value: string | undefined): Date | null {
|
||||
if (!value) return null;
|
||||
const exif = /^(\d{4}):(\d{2}):(\d{2})[ T](\d{2}):(\d{2}):(\d{2})/.exec(value);
|
||||
const date = exif
|
||||
? new Date(`${exif[1]}-${exif[2]}-${exif[3]}T${exif[4]}:${exif[5]}:${exif[6]}Z`)
|
||||
: new Date(value);
|
||||
const time = date.getTime();
|
||||
return Number.isFinite(time) && time > 0 ? date : null;
|
||||
}
|
||||
|
||||
/** "+52.5200+013.4050+034.000/" → { lat, lon }; null when absent or out of range. */
|
||||
export function parseIso6709(value: string | undefined): { lat: number; lon: number } | null {
|
||||
if (!value) return null;
|
||||
const match = /^([+-]\d+(?:\.\d+)?)([+-]\d+(?:\.\d+)?)/.exec(value.trim());
|
||||
if (!match) return null;
|
||||
const lat = Number(match[1]);
|
||||
const lon = Number(match[2]);
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return null;
|
||||
if (Math.abs(lat) > 90 || Math.abs(lon) > 180) return null;
|
||||
return { lat, lon };
|
||||
}
|
||||
177
packages/adapters/src/media-folder-source.test.ts
Normal file
177
packages/adapters/src/media-folder-source.test.ts
Normal file
@ -0,0 +1,177 @@
|
||||
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { MediaSourceItem } from "@rakazo/adapter-kit";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { ffmpegAvailable } from "./media-emulator-clips.js";
|
||||
import { FolderMediaSourceProvider } from "./media-folder-source.js";
|
||||
import { parseIso6709, runMediaTool } from "./media-folder-source-probe.js";
|
||||
|
||||
let root: string;
|
||||
const provider = new FolderMediaSourceProvider();
|
||||
|
||||
async function touch(relative: string, content = "") {
|
||||
const file = path.join(root, relative);
|
||||
await mkdir(path.dirname(file), { recursive: true });
|
||||
await writeFile(file, content);
|
||||
}
|
||||
|
||||
async function listAll(limit: number, cursor: string | null = null) {
|
||||
const pages: MediaSourceItem[][] = [];
|
||||
let next = cursor;
|
||||
do {
|
||||
const page = await provider.list({ config: { path: root }, cursor: next, limit });
|
||||
expect(page.items.length).toBeLessThanOrEqual(limit);
|
||||
pages.push(page.items);
|
||||
next = page.nextCursor;
|
||||
} while (next);
|
||||
return pages.flat().map((item) => item.externalId);
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(path.join(tmpdir(), "media-folder-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("FolderMediaSourceProvider", () => {
|
||||
it("lists media in stable code-unit order across pages", async () => {
|
||||
for (const name of ["b/c.mp4", "b-d.jpg", "a.MOV", "b/a/z.webm", "c.png", "notes.txt"])
|
||||
await touch(name);
|
||||
const ids = await listAll(2);
|
||||
expect(ids).toEqual(["a.MOV", "b-d.jpg", "b/a/z.webm", "b/c.mp4", "c.png"]);
|
||||
});
|
||||
|
||||
it("keeps the cursor stable when files are added mid-sync", async () => {
|
||||
for (const name of ["a.mp4", "c.mp4", "e.mp4", "g.mp4"]) await touch(name);
|
||||
const first = await provider.list({ config: { path: root }, cursor: null, limit: 2 });
|
||||
expect(first.items.map((item) => item.externalId)).toEqual(["a.mp4", "c.mp4"]);
|
||||
await touch("b.mp4"); // before the cursor: picked up by the next full sync
|
||||
await touch("d.mp4"); // after the cursor: part of this sync
|
||||
const rest = await listAll(2, first.nextCursor);
|
||||
expect(rest).toEqual(["d.mp4", "e.mp4", "g.mp4"]);
|
||||
expect(await listAll(10)).toEqual(["a.mp4", "b.mp4", "c.mp4", "d.mp4", "e.mp4", "g.mp4"]);
|
||||
});
|
||||
|
||||
it("skips hidden entries and symlinks without following them", async () => {
|
||||
await touch("keep/clip.mp4");
|
||||
await touch(".hidden.mp4");
|
||||
await touch(".cache/inside.mp4");
|
||||
await touch("keep/.DS_Store.jpg");
|
||||
const outside = await mkdtemp(path.join(tmpdir(), "media-folder-outside-"));
|
||||
try {
|
||||
await writeFile(path.join(outside, "secret.mp4"), "");
|
||||
await symlink(path.join(outside, "secret.mp4"), path.join(root, "link.mp4"));
|
||||
await symlink(outside, path.join(root, "linked-dir"));
|
||||
await symlink(path.join(root, "keep"), path.join(root, "keep-alias"));
|
||||
expect(await listAll(10)).toEqual(["keep/clip.mp4"]);
|
||||
await expect(provider.resolve({ path: root }, "link.mp4")).rejects.toThrow();
|
||||
await expect(provider.resolve({ path: root }, "linked-dir/secret.mp4")).rejects.toThrow();
|
||||
await expect(provider.resolve({ path: root }, "keep-alias/clip.mp4")).rejects.toThrow();
|
||||
} finally {
|
||||
await rm(outside, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves inside the root and rejects traversal", async () => {
|
||||
await touch("keep/clip.mp4");
|
||||
await touch(".hidden.mp4");
|
||||
await expect(provider.resolve({ path: root }, "keep/clip.mp4")).resolves.toBe(
|
||||
path.join(root, "keep/clip.mp4"),
|
||||
);
|
||||
for (const bad of [
|
||||
"../x",
|
||||
"../x.mp4",
|
||||
"keep/../../x.mp4",
|
||||
"/etc/passwd",
|
||||
".hidden.mp4",
|
||||
"keep",
|
||||
])
|
||||
await expect(provider.resolve({ path: root }, bad), bad).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("rejects relative roots and forged cursors", async () => {
|
||||
await expect(
|
||||
provider.list({ config: { path: "media" }, cursor: null, limit: 5 }),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
provider.list({ config: { path: root }, cursor: "not base64url!", limit: 5 }),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("pages a 1,000-file folder without duplicates or gaps", async () => {
|
||||
const expected: string[] = [];
|
||||
for (let dir = 0; dir < 10; dir++) {
|
||||
for (let file = 0; file < 100; file++) {
|
||||
const relative = `d${dir}/f${String(file).padStart(3, "0")}.${file % 3 === 0 ? "jpg" : "mp4"}`;
|
||||
// Unreadable media must not abort a page; most files are empty and skip probing.
|
||||
await touch(relative, file % 50 === 7 ? "garbage, not a video" : "");
|
||||
expected.push(relative);
|
||||
}
|
||||
}
|
||||
await touch("d3/ignore.txt");
|
||||
const pages: MediaSourceItem[][] = [];
|
||||
let cursor: string | null = null;
|
||||
do {
|
||||
const page = await provider.list({ config: { path: root }, cursor, limit: 137 });
|
||||
pages.push(page.items);
|
||||
cursor = page.nextCursor;
|
||||
} while (cursor);
|
||||
const items = pages.flat();
|
||||
expect(pages.length).toBe(Math.ceil(1000 / 137));
|
||||
expect(items.map((item) => item.externalId)).toEqual(expected.sort());
|
||||
expect(new Set(items.map((item) => item.externalId)).size).toBe(1000);
|
||||
const garbage = items.find((item) => item.externalId === "d0/f007.mp4");
|
||||
expect(garbage).toMatchObject({ durationMs: null, width: null, sizeBytes: 20, kind: "video" });
|
||||
expect(garbage?.capturedAt).toBeInstanceOf(Date);
|
||||
expect(garbage?.revision).toMatch(/^\d+-20$/);
|
||||
expect(items.find((item) => item.externalId === "d0/f000.jpg")?.kind).toBe("photo");
|
||||
});
|
||||
|
||||
it.skipIf(!ffmpegAvailable())(
|
||||
"reads duration, size, capture time and GPS with ffprobe",
|
||||
async () => {
|
||||
await mkdir(path.join(root, "trip"), { recursive: true });
|
||||
await runMediaTool("ffmpeg", [
|
||||
"-v",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=s=320x180:r=25:d=1",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-metadata",
|
||||
"creation_time=2026-05-02T08:00:00Z",
|
||||
"-metadata",
|
||||
"location=+52.5200+013.4050/",
|
||||
path.join(root, "trip/clip.mp4"),
|
||||
]);
|
||||
const [item] = (await provider.list({ config: { path: root }, cursor: null, limit: 5 }))
|
||||
.items;
|
||||
expect(item).toMatchObject({
|
||||
externalId: "trip/clip.mp4",
|
||||
kind: "video",
|
||||
durationMs: 1000,
|
||||
width: 320,
|
||||
height: 180,
|
||||
fps: 25,
|
||||
gpsLat: 52.52,
|
||||
gpsLon: 13.405,
|
||||
albumNames: ["trip"],
|
||||
});
|
||||
expect(item?.capturedAt?.toISOString()).toBe("2026-05-02T08:00:00.000Z");
|
||||
},
|
||||
);
|
||||
|
||||
it("parses ISO 6709 locations", () => {
|
||||
expect(parseIso6709("+52.5200+013.4050+034.000/")).toEqual({ lat: 52.52, lon: 13.405 });
|
||||
expect(parseIso6709("-33.8688+151.2093/")).toEqual({ lat: -33.8688, lon: 151.2093 });
|
||||
expect(parseIso6709("+95.0+013.0/")).toBeNull();
|
||||
expect(parseIso6709("nowhere")).toBeNull();
|
||||
});
|
||||
});
|
||||
221
packages/adapters/src/media-folder-source.ts
Normal file
221
packages/adapters/src/media-folder-source.ts
Normal file
@ -0,0 +1,221 @@
|
||||
import type { Dirent } from "node:fs";
|
||||
import { lstat, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type {
|
||||
MediaSourceItem,
|
||||
MediaSourceListRequest,
|
||||
MediaSourceListResult,
|
||||
MediaSourceProvider,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { MediaAssetKind } from "@rakazo/contracts";
|
||||
import { PathOutsideRootError, resolveInside } from "@rakazo/media/node";
|
||||
import { z } from "zod";
|
||||
import { probeMedia } from "./media-folder-source-probe.js";
|
||||
|
||||
/*
|
||||
* Server-side folder source. The deployment owner picks an absolute root; the
|
||||
* walk lists media files below it in a stable order (relative POSIX paths,
|
||||
* code-unit sorted) so the opaque cursor (the last relative path) stays valid
|
||||
* while files are added. Hidden entries and symlinks are skipped and never
|
||||
* followed. Only the directories needed for one page are read.
|
||||
*/
|
||||
|
||||
export const FOLDER_MEDIA_EXTENSIONS: Readonly<Record<string, MediaAssetKind>> = {
|
||||
mp4: "video",
|
||||
mov: "video",
|
||||
m4v: "video",
|
||||
mkv: "video",
|
||||
webm: "video",
|
||||
avi: "video",
|
||||
jpg: "photo",
|
||||
jpeg: "photo",
|
||||
png: "photo",
|
||||
heic: "photo",
|
||||
heif: "photo",
|
||||
webp: "photo",
|
||||
};
|
||||
|
||||
const FolderConfigSchema = z.object({
|
||||
path: z.string().min(1).refine(path.isAbsolute, "folder path must be absolute"),
|
||||
});
|
||||
|
||||
const MAX_PAGE = 1000;
|
||||
const PROBE_CONCURRENCY = 4;
|
||||
|
||||
export interface FolderMediaSourceOptions {
|
||||
ffprobePath?: string;
|
||||
}
|
||||
|
||||
export function encodeFolderCursor(relativePath: string): string {
|
||||
return Buffer.from(relativePath, "utf8").toString("base64url");
|
||||
}
|
||||
|
||||
export function decodeFolderCursor(cursor: string | null): string | null {
|
||||
if (!cursor) return null;
|
||||
const decoded = Buffer.from(cursor, "base64url").toString("utf8");
|
||||
if (!decoded || decoded.includes("\0") || encodeFolderCursor(decoded) !== cursor)
|
||||
throw new Error("invalid folder cursor");
|
||||
return decoded;
|
||||
}
|
||||
|
||||
function mediaKindOf(name: string): MediaAssetKind | null {
|
||||
const ext = path.extname(name).slice(1).toLowerCase();
|
||||
return FOLDER_MEDIA_EXTENSIONS[ext] ?? null;
|
||||
}
|
||||
|
||||
/** Sort key that makes a depth-first walk yield relative paths in code-unit order. */
|
||||
function entryKey(entry: Dirent): string {
|
||||
return entry.isDirectory() ? `${entry.name}/` : entry.name;
|
||||
}
|
||||
|
||||
function compareKeys(a: string, b: string): number {
|
||||
return a < b ? -1 : a > b ? 1 : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Yield relative paths of media files strictly after `after`, in order.
|
||||
* Directories whose whole subtree sorts before the cursor are not read.
|
||||
*/
|
||||
async function* walkAfter(
|
||||
root: string,
|
||||
relativeDir: string,
|
||||
after: string | null,
|
||||
signal?: AbortSignal,
|
||||
): AsyncGenerator<string> {
|
||||
signal?.throwIfAborted();
|
||||
let entries: Dirent[];
|
||||
try {
|
||||
entries = await readdir(path.join(root, relativeDir), { withFileTypes: true });
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
// A directory removed or locked mid-walk is skipped; the root itself must exist.
|
||||
if (relativeDir && (code === "ENOENT" || code === "EACCES" || code === "ENOTDIR")) return;
|
||||
throw error;
|
||||
}
|
||||
const visible = entries
|
||||
.filter((entry) => !entry.name.startsWith(".") && !entry.isSymbolicLink())
|
||||
.filter((entry) => entry.isDirectory() || (entry.isFile() && mediaKindOf(entry.name)))
|
||||
.map((entry) => ({ entry, key: entryKey(entry) }))
|
||||
.sort((a, b) => compareKeys(a.key, b.key));
|
||||
for (const { entry, key } of visible) {
|
||||
const relative = relativeDir ? `${relativeDir}/${key}` : key;
|
||||
if (entry.isDirectory()) {
|
||||
// Subtree entirely before the cursor: every path in it starts with `relative`.
|
||||
if (after !== null && relative < after && !after.startsWith(relative)) continue;
|
||||
yield* walkAfter(root, relative.slice(0, -1), after, signal);
|
||||
} else if (after === null || relative > after) {
|
||||
yield relative;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function mapLimited<T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const out = new Array<R>(items.length);
|
||||
let next = 0;
|
||||
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
||||
while (next < items.length) {
|
||||
const index = next++;
|
||||
out[index] = await fn(items[index] as T);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
return out;
|
||||
}
|
||||
|
||||
export class FolderMediaSourceProvider implements MediaSourceProvider {
|
||||
readonly kind = "folder" as const;
|
||||
|
||||
constructor(private readonly options: FolderMediaSourceOptions = {}) {}
|
||||
|
||||
async list(request: MediaSourceListRequest): Promise<MediaSourceListResult> {
|
||||
const { path: root } = FolderConfigSchema.parse(request.config);
|
||||
const limit = Math.max(1, Math.min(MAX_PAGE, Math.floor(request.limit)));
|
||||
const after = decodeFolderCursor(request.cursor);
|
||||
const relatives: string[] = [];
|
||||
let more = false;
|
||||
for await (const relative of walkAfter(root, "", after, request.signal)) {
|
||||
if (relatives.length === limit) {
|
||||
more = true;
|
||||
break;
|
||||
}
|
||||
relatives.push(relative);
|
||||
}
|
||||
const items = await mapLimited(relatives, PROBE_CONCURRENCY, (relative) =>
|
||||
this.describe(root, relative, request.signal),
|
||||
);
|
||||
const present = items.filter((item): item is MediaSourceItem => item !== null);
|
||||
const last = relatives.at(-1);
|
||||
return { items: present, nextCursor: more && last ? encodeFolderCursor(last) : null };
|
||||
}
|
||||
|
||||
async resolve(config: Record<string, unknown>, externalId: string): Promise<string> {
|
||||
const { path: root } = FolderConfigSchema.parse(config);
|
||||
const segments = externalId.split("/");
|
||||
if (segments.some((segment) => segment.startsWith(".")) || !mediaKindOf(externalId))
|
||||
throw new PathOutsideRootError(externalId);
|
||||
const absolute = await resolveInside(root, externalId);
|
||||
// Symlinks are never followed, at any depth below the root.
|
||||
let current = await resolveInside(root, ".");
|
||||
for (const segment of segments) {
|
||||
current = path.join(current, segment);
|
||||
if ((await lstat(current)).isSymbolicLink()) throw new PathOutsideRootError(externalId);
|
||||
}
|
||||
if (!(await lstat(absolute)).isFile()) throw new PathOutsideRootError(externalId);
|
||||
return absolute;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stat and probe one file. Probe failures yield null metadata, never a failed
|
||||
* page. The parent folder name doubles as an album for rule-based labels.
|
||||
*/
|
||||
private async describe(
|
||||
root: string,
|
||||
relative: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<MediaSourceItem | null> {
|
||||
const absolute = path.join(root, relative);
|
||||
let stats: Awaited<ReturnType<typeof lstat>>;
|
||||
try {
|
||||
stats = await lstat(absolute);
|
||||
} catch {
|
||||
return null; // Removed between walk and stat.
|
||||
}
|
||||
if (!stats.isFile()) return null;
|
||||
const kind = mediaKindOf(relative) ?? "video";
|
||||
let probed: Awaited<ReturnType<typeof probeMedia>> | null = null;
|
||||
if (stats.size > 0) {
|
||||
try {
|
||||
probed = await probeMedia(absolute, {
|
||||
ffprobePath: this.options.ffprobePath,
|
||||
signal,
|
||||
frameTags: kind === "photo",
|
||||
});
|
||||
} catch {
|
||||
// Unreadable, unsupported, or no ffprobe on this host: keep the file with null metadata.
|
||||
signal?.throwIfAborted();
|
||||
probed = null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
externalId: relative,
|
||||
kind,
|
||||
capturedAt: probed?.capturedAt ?? new Date(Math.trunc(stats.mtimeMs)),
|
||||
durationMs: kind === "video" ? (probed?.durationMs ?? null) : null,
|
||||
width: probed?.width ?? null,
|
||||
height: probed?.height ?? null,
|
||||
fps: kind === "video" ? (probed?.fps ?? null) : null,
|
||||
gpsLat: probed?.gpsLat ?? null,
|
||||
gpsLon: probed?.gpsLon ?? null,
|
||||
deviceModel: probed?.deviceModel ?? null,
|
||||
albumNames: relative.includes("/") ? [path.posix.basename(path.posix.dirname(relative))] : [],
|
||||
isFavorite: false,
|
||||
fileName: path.posix.basename(relative),
|
||||
sizeBytes: stats.size,
|
||||
revision: `${Math.trunc(stats.mtimeMs)}-${stats.size}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
222
packages/adapters/src/media-sidecar.test.ts
Normal file
222
packages/adapters/src/media-sidecar.test.ts
Normal file
@ -0,0 +1,222 @@
|
||||
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||
import { MediaInferenceOperationSchema } from "@rakazo/contracts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { MediaSidecarError, SidecarMediaInferenceProvider } from "./media-sidecar.js";
|
||||
|
||||
const TOKEN = "test-token-0123456789abcdef0123456789abcdef";
|
||||
const SCENES = {
|
||||
shots: [{ startMs: 0, endMs: 1000, score: 1 }],
|
||||
scenes: [{ startMs: 0, endMs: 1000 }],
|
||||
};
|
||||
|
||||
interface Call {
|
||||
url: string;
|
||||
init: RequestInit;
|
||||
}
|
||||
|
||||
function json(status: number, body: unknown): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function fakeFetch(responses: Array<Response | ((init: RequestInit) => Promise<Response>)>) {
|
||||
const calls: Call[] = [];
|
||||
const impl = (async (input: string | URL | Request, init?: RequestInit) => {
|
||||
calls.push({ url: String(input), init: init ?? {} });
|
||||
const next = responses.shift();
|
||||
if (!next) throw new Error("unexpected request");
|
||||
return typeof next === "function" ? next(init ?? {}) : next;
|
||||
}) as typeof fetch;
|
||||
return { impl, calls };
|
||||
}
|
||||
|
||||
function client(fetchImpl: typeof fetch, extra: { retryBaseMs?: number } = {}) {
|
||||
return new SidecarMediaInferenceProvider({
|
||||
baseUrl: "http://127.0.0.1:8800/",
|
||||
token: TOKEN,
|
||||
fetch: fetchImpl,
|
||||
retryBaseMs: 0,
|
||||
...extra,
|
||||
});
|
||||
}
|
||||
|
||||
describe("SidecarMediaInferenceProvider", () => {
|
||||
it("posts camelCase JSON with bearer auth and without the signal", async () => {
|
||||
const { impl, calls } = fakeFetch([json(200, SCENES)]);
|
||||
const controller = new AbortController();
|
||||
const result = await client(impl).detectScenes({
|
||||
path: "/data/media/a.mp4",
|
||||
signal: controller.signal,
|
||||
});
|
||||
expect(result).toEqual(SCENES);
|
||||
expect(calls).toHaveLength(1);
|
||||
const [call] = calls;
|
||||
expect(call?.url).toBe("http://127.0.0.1:8800/v1/detectScenes");
|
||||
expect(call?.init.method).toBe("POST");
|
||||
const headers = call?.init.headers as Record<string, string>;
|
||||
expect(headers.authorization).toBe(`Bearer ${TOKEN}`);
|
||||
expect(headers["content-type"]).toBe("application/json");
|
||||
expect(JSON.parse(String(call?.init.body))).toEqual({ path: "/data/media/a.mp4" });
|
||||
expect(call?.init.signal).toBeInstanceOf(AbortSignal);
|
||||
});
|
||||
|
||||
it("reads capabilities with GET", async () => {
|
||||
const capabilities = {
|
||||
configured: true,
|
||||
operations: Object.fromEntries(
|
||||
MediaInferenceOperationSchema.options.map((operation) => [
|
||||
operation,
|
||||
{
|
||||
available: operation !== "synthesizeIdentity",
|
||||
experimental: false,
|
||||
model: "m",
|
||||
license: "Apache-2.0",
|
||||
commercialUse: true,
|
||||
},
|
||||
]),
|
||||
),
|
||||
accelerator: "cpu",
|
||||
hardwareEncoders: [],
|
||||
};
|
||||
const { impl, calls } = fakeFetch([json(200, capabilities)]);
|
||||
await expect(client(impl).capabilities()).resolves.toEqual(capabilities);
|
||||
expect(calls[0]?.url).toBe("http://127.0.0.1:8800/v1/capabilities");
|
||||
expect(calls[0]?.init.method).toBe("GET");
|
||||
});
|
||||
|
||||
it("maps 501 to UnsupportedMediaOperationError", async () => {
|
||||
const { impl } = fakeFetch([
|
||||
json(501, {
|
||||
error: { code: "unsupported", message: "no licensed model", operation: "inpaintVideo" },
|
||||
}),
|
||||
]);
|
||||
const error = await client(impl)
|
||||
.inpaintVideo({ path: "/m/a.mp4", maskDir: "/m/masks", outPath: "/m/out.mp4" })
|
||||
.catch((caught: unknown) => caught);
|
||||
expect(error).toBeInstanceOf(UnsupportedMediaOperationError);
|
||||
expect((error as UnsupportedMediaOperationError).operation).toBe("inpaintVideo");
|
||||
expect((error as Error).message).toContain("no licensed model");
|
||||
});
|
||||
|
||||
it("retries 503 busy with backoff, at most three times", async () => {
|
||||
const busy = () => json(503, { error: { code: "busy", message: "queue full" } });
|
||||
const recovered = fakeFetch([busy(), busy(), json(200, SCENES)]);
|
||||
await expect(client(recovered.impl).detectScenes({ path: "/m/a.mp4" })).resolves.toEqual(
|
||||
SCENES,
|
||||
);
|
||||
expect(recovered.calls).toHaveLength(3);
|
||||
|
||||
const exhausted = fakeFetch([busy(), busy(), busy(), busy(), busy()]);
|
||||
const error = await client(exhausted.impl)
|
||||
.detectScenes({ path: "/m/a.mp4" })
|
||||
.catch((caught: unknown) => caught);
|
||||
expect(exhausted.calls).toHaveLength(4);
|
||||
expect(error).toBeInstanceOf(MediaSidecarError);
|
||||
expect((error as MediaSidecarError).code).toBe("busy");
|
||||
expect((error as MediaSidecarError).status).toBe(503);
|
||||
});
|
||||
|
||||
it("maps typed sidecar errors", async () => {
|
||||
const { impl } = fakeFetch([
|
||||
json(400, { error: { code: "invalid_path", message: "path is outside MEDIA_ROOT" } }),
|
||||
json(500, "not json-shaped"),
|
||||
]);
|
||||
const provider = client(impl);
|
||||
await expect(provider.detectScenes({ path: "/etc/passwd" })).rejects.toMatchObject({
|
||||
name: "MediaSidecarError",
|
||||
code: "invalid_path",
|
||||
status: 400,
|
||||
operation: "detectScenes",
|
||||
});
|
||||
await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({
|
||||
code: "http_error",
|
||||
status: 500,
|
||||
});
|
||||
});
|
||||
|
||||
it("passes caller aborts through as AbortError", async () => {
|
||||
const controller = new AbortController();
|
||||
const { impl } = fakeFetch([
|
||||
(init) =>
|
||||
new Promise<Response>((_, reject) => {
|
||||
init.signal?.addEventListener("abort", () => reject(init.signal?.reason));
|
||||
setTimeout(() => controller.abort(), 10);
|
||||
}),
|
||||
]);
|
||||
await expect(
|
||||
client(impl).trackFaces({ path: "/m/a.mp4", sampleFps: 5, signal: controller.signal }),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
|
||||
const already = new AbortController();
|
||||
already.abort();
|
||||
const idle = fakeFetch([]);
|
||||
await expect(
|
||||
client(idle.impl).detectScenes({ path: "/m/a.mp4", signal: already.signal }),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(idle.calls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("reports a timeout as a typed error", async () => {
|
||||
const { impl } = fakeFetch([
|
||||
(init) =>
|
||||
new Promise<Response>((_, reject) => {
|
||||
init.signal?.addEventListener("abort", () => reject(init.signal?.reason));
|
||||
}),
|
||||
]);
|
||||
const provider = new SidecarMediaInferenceProvider({
|
||||
baseUrl: "http://127.0.0.1:8800",
|
||||
token: TOKEN,
|
||||
fetch: impl,
|
||||
timeoutsMs: { detectScenes: 20 },
|
||||
});
|
||||
await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({
|
||||
code: "timeout",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects malformed responses", async () => {
|
||||
const { impl } = fakeFetch([
|
||||
json(200, { shots: [{ startMs: -1, endMs: 10, score: 1 }], scenes: [] }),
|
||||
json(200, {
|
||||
frames: [{ tMs: 0, faces: [{ box: { x: 2, y: 0, w: 0.1, h: 0.1 }, confidence: 1 }] }],
|
||||
}),
|
||||
json(200, { model: "m", dim: 3, vectors: [[1, 2]] }),
|
||||
new Response("<html>", { status: 200 }),
|
||||
]);
|
||||
const provider = client(impl);
|
||||
await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({
|
||||
code: "malformed_response",
|
||||
});
|
||||
await expect(provider.detectFaces({ path: "/m/a.mp4", sampleFps: 1 })).rejects.toMatchObject({
|
||||
code: "malformed_response",
|
||||
});
|
||||
await expect(provider.embedFrames({ path: "/m/a.mp4", timesMs: [0] })).rejects.toMatchObject({
|
||||
code: "malformed_response",
|
||||
});
|
||||
await expect(provider.detectScenes({ path: "/m/a.mp4" })).rejects.toMatchObject({
|
||||
code: "malformed_response",
|
||||
});
|
||||
});
|
||||
|
||||
it("never puts the token into error messages", async () => {
|
||||
const { impl } = fakeFetch([
|
||||
json(401, { error: { code: "unauthorized", message: `bad token ${TOKEN}` } }),
|
||||
json(500, { error: { code: "internal", message: `crashed with Bearer ${TOKEN}` } }),
|
||||
async () => {
|
||||
throw new TypeError(`connect ECONNREFUSED (auth ${TOKEN})`);
|
||||
},
|
||||
]);
|
||||
const provider = client(impl);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const error = (await provider
|
||||
.detectScenes({ path: "/m/a.mp4" })
|
||||
.catch((caught: unknown) => caught)) as Error;
|
||||
expect(error).toBeInstanceOf(MediaSidecarError);
|
||||
expect(error.message).not.toContain(TOKEN);
|
||||
expect(JSON.stringify(error)).not.toContain(TOKEN);
|
||||
expect(String(error.stack)).not.toContain(TOKEN);
|
||||
}
|
||||
});
|
||||
});
|
||||
410
packages/adapters/src/media-sidecar.ts
Normal file
410
packages/adapters/src/media-sidecar.ts
Normal file
@ -0,0 +1,410 @@
|
||||
import type {
|
||||
AudioFeaturesResult,
|
||||
ClassifyContentResult,
|
||||
DetectFacesResult,
|
||||
DetectScenesResult,
|
||||
EmbedFramesResult,
|
||||
FaceLandmarksResult,
|
||||
InpaintVideoResult,
|
||||
MediaInferenceProvider,
|
||||
SaliencyResult,
|
||||
SegmentPromptedResult,
|
||||
TrackFacesResult,
|
||||
TranscribeResult,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
||||
import type { MediaCapabilities, MediaInferenceOperation } from "@rakazo/contracts";
|
||||
import {
|
||||
BoxKeyframeSchema,
|
||||
ContentCategorySchema,
|
||||
MediaCapabilitiesSchema,
|
||||
NormalizedBoxSchema,
|
||||
NormalizedPointSchema,
|
||||
} from "@rakazo/contracts";
|
||||
import { z } from "zod";
|
||||
|
||||
/*
|
||||
* HTTP client for the local media sidecar (infra/media-worker, API v1).
|
||||
* Bearer auth, JSON bodies, per-operation timeouts, bounded retry on 503 busy,
|
||||
* 501 → UnsupportedMediaOperationError, every other failure → MediaSidecarError.
|
||||
* Responses are validated before they reach the caller. The token never
|
||||
* appears in an error message.
|
||||
*/
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30 * 60_000;
|
||||
const LONG_TIMEOUT_MS = 4 * 60 * 60_000;
|
||||
const MAX_BUSY_RETRIES = 3;
|
||||
|
||||
export type MediaSidecarErrorCode =
|
||||
| "bad_request"
|
||||
| "invalid_path"
|
||||
| "unauthorized"
|
||||
| "too_large"
|
||||
| "busy"
|
||||
| "internal"
|
||||
| "timeout"
|
||||
| "unreachable"
|
||||
| "malformed_response"
|
||||
| "http_error";
|
||||
|
||||
export class MediaSidecarError extends Error {
|
||||
constructor(
|
||||
readonly code: MediaSidecarErrorCode,
|
||||
message: string,
|
||||
readonly status: number | null,
|
||||
readonly operation: MediaInferenceOperation | "capabilities",
|
||||
) {
|
||||
super(message);
|
||||
this.name = "MediaSidecarError";
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response schemas (shapes mirror the adapter-kit result types)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Ms = z.number().int().min(0);
|
||||
const Unit = z.number().min(0).max(1);
|
||||
const Span = z.object({ startMs: Ms, endMs: Ms });
|
||||
const ScoredKeyframe = BoxKeyframeSchema.extend({ confidence: Unit });
|
||||
|
||||
const schemas = {
|
||||
detectScenes: z.object({
|
||||
shots: z.array(Span.extend({ score: z.number() })),
|
||||
scenes: z.array(Span),
|
||||
}) satisfies z.ZodType<DetectScenesResult>,
|
||||
transcribe: z.object({
|
||||
language: z.string().nullable(),
|
||||
words: z.array(Span.extend({ text: z.string(), confidence: Unit })),
|
||||
}) satisfies z.ZodType<TranscribeResult>,
|
||||
detectFaces: z.object({
|
||||
frames: z.array(
|
||||
z.object({
|
||||
tMs: Ms,
|
||||
faces: z.array(
|
||||
z.object({
|
||||
box: NormalizedBoxSchema,
|
||||
confidence: Unit,
|
||||
landmarks: z.array(NormalizedPointSchema).optional(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
),
|
||||
}) satisfies z.ZodType<DetectFacesResult>,
|
||||
trackFaces: z.object({
|
||||
sampleFps: z.number().positive(),
|
||||
tracks: z.array(
|
||||
z.object({
|
||||
trackId: z.string().min(1),
|
||||
keyframes: z.array(ScoredKeyframe),
|
||||
embedding: z.array(z.number()).nullable(),
|
||||
}),
|
||||
),
|
||||
}) satisfies z.ZodType<TrackFacesResult>,
|
||||
faceLandmarks: z.object({
|
||||
frames: z.array(
|
||||
z.object({
|
||||
tMs: Ms,
|
||||
faces: z.array(
|
||||
z.object({ box: NormalizedBoxSchema, landmarks: z.array(NormalizedPointSchema) }),
|
||||
),
|
||||
}),
|
||||
),
|
||||
}) satisfies z.ZodType<FaceLandmarksResult>,
|
||||
embedFrames: z
|
||||
.object({
|
||||
model: z.string(),
|
||||
dim: z.number().int().positive(),
|
||||
vectors: z.array(z.array(z.number())),
|
||||
})
|
||||
.refine((value) => value.vectors.every((vector) => vector.length === value.dim), {
|
||||
message: "vector length does not match dim",
|
||||
}) satisfies z.ZodType<EmbedFramesResult>,
|
||||
classifyContent: z.object({
|
||||
model: z.string(),
|
||||
frames: z.array(z.object({ tMs: Ms, scores: z.partialRecord(ContentCategorySchema, Unit) })),
|
||||
}) satisfies z.ZodType<ClassifyContentResult>,
|
||||
segmentPrompted: z.object({
|
||||
maskDir: z.string().min(1),
|
||||
keyframes: z.array(ScoredKeyframe),
|
||||
score: Unit,
|
||||
}) satisfies z.ZodType<SegmentPromptedResult>,
|
||||
inpaintVideo: z.object({
|
||||
outPath: z.string().min(1),
|
||||
quality: z.object({ temporalStability: Unit, edgeQuality: Unit, score: Unit }),
|
||||
}) satisfies z.ZodType<InpaintVideoResult>,
|
||||
redact: z.object({
|
||||
outPath: z.string().min(1),
|
||||
framesProcessed: z.number().int().min(0),
|
||||
}) satisfies z.ZodType<{ outPath: string; framesProcessed: number }>,
|
||||
synthesizeIdentity: z.object({ outPath: z.string().min(1) }) satisfies z.ZodType<{
|
||||
outPath: string;
|
||||
}>,
|
||||
audioFeatures: z.object({
|
||||
integratedLufs: z.number().nullable(),
|
||||
truePeakDb: z.number().nullable(),
|
||||
tempoBpm: z.number().nullable(),
|
||||
beatsMs: z.array(Ms),
|
||||
energy: z.array(z.object({ tMs: Ms, value: z.number() })),
|
||||
silences: z.array(Span),
|
||||
}) satisfies z.ZodType<AudioFeaturesResult>,
|
||||
saliency: z.object({
|
||||
points: z.array(z.object({ tMs: Ms, cx: Unit, cy: Unit, strength: z.number() })),
|
||||
}) satisfies z.ZodType<SaliencyResult>,
|
||||
} as const satisfies Record<MediaInferenceOperation, z.ZodType>;
|
||||
|
||||
export const MEDIA_SIDECAR_RESPONSE_SCHEMAS = schemas;
|
||||
|
||||
const ErrorBodySchema = z.object({
|
||||
error: z.object({ code: z.string(), message: z.string().optional() }),
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Client
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface SidecarMediaInferenceOptions {
|
||||
baseUrl: string;
|
||||
token: string;
|
||||
fetch?: typeof fetch;
|
||||
/** Per-operation timeout overrides in ms. */
|
||||
timeoutsMs?: Partial<Record<MediaInferenceOperation | "capabilities", number>>;
|
||||
/** First busy-retry delay; doubles per attempt. */
|
||||
retryBaseMs?: number;
|
||||
}
|
||||
|
||||
type Request<K extends MediaInferenceOperation> = Parameters<MediaInferenceProvider[K]>[0];
|
||||
type Result<K extends MediaInferenceOperation> = Awaited<ReturnType<MediaInferenceProvider[K]>>;
|
||||
|
||||
const KNOWN_CODES = new Set<string>([
|
||||
"bad_request",
|
||||
"invalid_path",
|
||||
"unauthorized",
|
||||
"too_large",
|
||||
"busy",
|
||||
"internal",
|
||||
]);
|
||||
|
||||
export class SidecarMediaInferenceProvider implements MediaInferenceProvider {
|
||||
readonly id = "sidecar";
|
||||
private readonly baseUrl: string;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
|
||||
constructor(private readonly options: SidecarMediaInferenceOptions) {
|
||||
this.baseUrl = options.baseUrl.replace(/\/+$/, "");
|
||||
this.fetchImpl = options.fetch ?? fetch;
|
||||
}
|
||||
|
||||
async capabilities(signal?: AbortSignal): Promise<MediaCapabilities> {
|
||||
return this.call("capabilities", "GET", "/v1/capabilities", undefined, signal, (body) =>
|
||||
MediaCapabilitiesSchema.parse(body),
|
||||
);
|
||||
}
|
||||
|
||||
detectScenes(request: Request<"detectScenes">) {
|
||||
return this.operation("detectScenes", request);
|
||||
}
|
||||
transcribe(request: Request<"transcribe">) {
|
||||
return this.operation("transcribe", request);
|
||||
}
|
||||
detectFaces(request: Request<"detectFaces">) {
|
||||
return this.operation("detectFaces", request);
|
||||
}
|
||||
trackFaces(request: Request<"trackFaces">) {
|
||||
return this.operation("trackFaces", request);
|
||||
}
|
||||
faceLandmarks(request: Request<"faceLandmarks">) {
|
||||
return this.operation("faceLandmarks", request);
|
||||
}
|
||||
embedFrames(request: Request<"embedFrames">) {
|
||||
return this.operation("embedFrames", request);
|
||||
}
|
||||
classifyContent(request: Request<"classifyContent">) {
|
||||
return this.operation("classifyContent", request);
|
||||
}
|
||||
segmentPrompted(request: Request<"segmentPrompted">) {
|
||||
return this.operation("segmentPrompted", request);
|
||||
}
|
||||
inpaintVideo(request: Request<"inpaintVideo">) {
|
||||
return this.operation("inpaintVideo", request);
|
||||
}
|
||||
redact(request: Request<"redact">) {
|
||||
return this.operation("redact", request);
|
||||
}
|
||||
synthesizeIdentity(request: Request<"synthesizeIdentity">) {
|
||||
return this.operation("synthesizeIdentity", request);
|
||||
}
|
||||
audioFeatures(request: Request<"audioFeatures">) {
|
||||
return this.operation("audioFeatures", request);
|
||||
}
|
||||
saliency(request: Request<"saliency">) {
|
||||
return this.operation("saliency", request);
|
||||
}
|
||||
|
||||
private operation<K extends MediaInferenceOperation>(
|
||||
operation: K,
|
||||
request: Request<K>,
|
||||
): Promise<Result<K>> {
|
||||
const { signal, ...body } = request;
|
||||
const schema = schemas[operation] as z.ZodType;
|
||||
return this.call(operation, "POST", `/v1/${operation}`, body, signal, (json) => {
|
||||
return schema.parse(json) as Result<K>;
|
||||
});
|
||||
}
|
||||
|
||||
private timeoutFor(operation: MediaInferenceOperation | "capabilities"): number {
|
||||
const override = this.options.timeoutsMs?.[operation];
|
||||
if (override !== undefined) return override;
|
||||
return operation === "inpaintVideo" || operation === "synthesizeIdentity"
|
||||
? LONG_TIMEOUT_MS
|
||||
: DEFAULT_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
private async call<T>(
|
||||
operation: MediaInferenceOperation | "capabilities",
|
||||
method: "GET" | "POST",
|
||||
route: string,
|
||||
body: unknown,
|
||||
signal: AbortSignal | undefined,
|
||||
parse: (json: unknown) => T,
|
||||
): Promise<T> {
|
||||
signal?.throwIfAborted();
|
||||
const timeout = AbortSignal.timeout(this.timeoutFor(operation));
|
||||
const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
||||
const retryBase = this.options.retryBaseMs ?? 500;
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await this.fetchImpl(`${this.baseUrl}${route}`, {
|
||||
method,
|
||||
headers: {
|
||||
authorization: `Bearer ${this.options.token}`,
|
||||
accept: "application/json",
|
||||
...(body === undefined ? {} : { "content-type": "application/json" }),
|
||||
},
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal: combined,
|
||||
});
|
||||
} catch (error) {
|
||||
throw this.transportError(error, operation, signal, timeout);
|
||||
}
|
||||
if (response.status === 503 && attempt < MAX_BUSY_RETRIES) {
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
try {
|
||||
await delay(retryBase * 2 ** attempt, combined);
|
||||
} catch (error) {
|
||||
throw this.transportError(error, operation, signal, timeout);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let json: unknown;
|
||||
let text: string;
|
||||
try {
|
||||
text = await response.text();
|
||||
} catch (error) {
|
||||
throw this.transportError(error, operation, signal, timeout);
|
||||
}
|
||||
try {
|
||||
json = text ? JSON.parse(text) : null;
|
||||
} catch {
|
||||
json = undefined;
|
||||
}
|
||||
if (!response.ok) throw this.httpError(response.status, json, operation);
|
||||
if (json === undefined)
|
||||
throw new MediaSidecarError(
|
||||
"malformed_response",
|
||||
`media sidecar returned invalid JSON for ${operation}`,
|
||||
response.status,
|
||||
operation,
|
||||
);
|
||||
try {
|
||||
return parse(json);
|
||||
} catch (error) {
|
||||
const detail =
|
||||
error instanceof z.ZodError
|
||||
? error.issues
|
||||
.slice(0, 3)
|
||||
.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`)
|
||||
.join("; ")
|
||||
: "unexpected shape";
|
||||
throw new MediaSidecarError(
|
||||
"malformed_response",
|
||||
this.scrub(`media sidecar returned a malformed ${operation} response: ${detail}`),
|
||||
response.status,
|
||||
operation,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private httpError(
|
||||
status: number,
|
||||
json: unknown,
|
||||
operation: MediaInferenceOperation | "capabilities",
|
||||
): Error {
|
||||
const parsed = ErrorBodySchema.safeParse(json);
|
||||
const message = this.scrub(
|
||||
parsed.success && parsed.data.error.message
|
||||
? parsed.data.error.message
|
||||
: `media sidecar responded ${status}`,
|
||||
);
|
||||
if (status === 501 && operation !== "capabilities")
|
||||
return new UnsupportedMediaOperationError(operation, message);
|
||||
const serverCode = parsed.success ? parsed.data.error.code : "";
|
||||
const code: MediaSidecarErrorCode = KNOWN_CODES.has(serverCode)
|
||||
? (serverCode as MediaSidecarErrorCode)
|
||||
: status === 401
|
||||
? "unauthorized"
|
||||
: status === 503
|
||||
? "busy"
|
||||
: "http_error";
|
||||
return new MediaSidecarError(code, `${operation}: ${message}`, status, operation);
|
||||
}
|
||||
|
||||
private transportError(
|
||||
error: unknown,
|
||||
operation: MediaInferenceOperation | "capabilities",
|
||||
signal: AbortSignal | undefined,
|
||||
timeout: AbortSignal,
|
||||
): unknown {
|
||||
if (signal?.aborted) return signal.reason;
|
||||
if (timeout.aborted)
|
||||
return new MediaSidecarError(
|
||||
"timeout",
|
||||
`media sidecar ${operation} timed out`,
|
||||
null,
|
||||
operation,
|
||||
);
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
return new MediaSidecarError(
|
||||
"unreachable",
|
||||
this.scrub(`media sidecar is unreachable: ${detail}`),
|
||||
null,
|
||||
operation,
|
||||
);
|
||||
}
|
||||
|
||||
/** Strip the bearer token from any text that may end up in logs or errors. */
|
||||
private scrub(text: string): string {
|
||||
const token = this.options.token;
|
||||
return token ? text.split(token).join("[redacted]") : text;
|
||||
}
|
||||
}
|
||||
|
||||
function delay(ms: number, signal: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (signal.aborted) {
|
||||
reject(signal.reason);
|
||||
return;
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
reject(signal.reason);
|
||||
};
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user