feat(media): edit and export video with ffmpeg
Bots can now cut, reframe and export videos for every common format without a hosted service: ffmpeg and ffprobe from PATH do the work on the server. - ffmpeg wrapper: PATH detection, ffprobe JSON probing (rotation, pixel aspect, frame rate, audio, packet scan when the container has no duration), runs with -progress parsing, abort and stall kills, an output size cap, partial-output cleanup and short errors without temp paths. Inputs go through the file protocol with the demuxer forced from the MIME type, so a crafted upload cannot be probed as a playlist that reads other files or URLs. extractAudio (16 kHz mono WAV) and renderPoster for transcription and media cards. - A pure planner turns edit steps (trim, keep, concat, reframe, speed, volume, mute, subtitles, extract_audio, poster) into one filter_complex pass: leading cuts seek the input, appended clips are normalized to the frame, fps and 48 kHz stereo (silence where a clip has none), reframes crop, pad or blur to the presets, speed chains atempo, subtitles burn in from generated file names with a readable style that stays clear of the controls on 9:16 frames. H.264/AAC MP4 with yuv420p and faststart; location metadata is stripped. - edit_video and export_video operations and tool definitions; the inputs are shared contracts so the media card can start exports. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
parent
7ff93f08bb
commit
485090c391
@ -2,6 +2,8 @@ import type { ConnectorTool } from "@rakazo/adapter-kit";
|
||||
import {
|
||||
BotSecretDestination,
|
||||
BotSecretName,
|
||||
EditVideoInputSchema,
|
||||
ExportVideoInputSchema,
|
||||
MODEL_TIERS,
|
||||
SecretAskPurpose,
|
||||
SecretHttpRequest,
|
||||
@ -12,6 +14,25 @@ import { z } from "zod";
|
||||
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
|
||||
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
|
||||
|
||||
/** A media tool's arguments: its operation's input plus whether to wake the bot when it ends. */
|
||||
function mediaToolSchema(input: z.ZodType): Record<string, unknown> {
|
||||
const schema = z.toJSONSchema(input, { io: "input" });
|
||||
return {
|
||||
...schema,
|
||||
properties: {
|
||||
...schema.properties,
|
||||
continue_after: {
|
||||
type: "boolean",
|
||||
description:
|
||||
"true to be woken with the result when the job ends, to continue a multi-step task. Default false: the card alone shows the result.",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const VIDEO_PRESETS_AND_FITS =
|
||||
"Presets: vertical 9:16, landscape 16:9, square 1:1, portrait 4:5, classic 4:3, cinema 21:9. fit: crop (default) fills the frame and cuts the edges, pad adds bars, blur fills the bars with a blurred copy.";
|
||||
|
||||
export const DELEGATION_TOOL_NAMES = new Set([
|
||||
"run_subagent",
|
||||
"spawn_bot",
|
||||
@ -326,6 +347,16 @@ export const builtinAgentTools: ConnectorTool[] = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "edit_video",
|
||||
description: `Edit a video artifact in a background job; a media card shows progress and the result, a new version of the source. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4 with a poster. title: a few words in the user's language for the card and file name.`,
|
||||
inputSchema: mediaToolSchema(EditVideoInputSchema),
|
||||
},
|
||||
{
|
||||
name: "export_video",
|
||||
description: `Export a video artifact to one or more presets in a background job: one H.264/AAC MP4 with a poster per preset, each a new version of the source, shown on a media card. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`,
|
||||
inputSchema: mediaToolSchema(ExportVideoInputSchema),
|
||||
},
|
||||
{
|
||||
name: "add_mcp_server",
|
||||
description:
|
||||
|
||||
@ -76,6 +76,8 @@ export * from "./mcp-emulator.js";
|
||||
export * from "./mcp-oauth.js";
|
||||
export * from "./mcp-server-tool.js";
|
||||
export * from "./mcp-transport.js";
|
||||
export * from "./media/ffmpeg.js";
|
||||
export * from "./media/video-operations.js";
|
||||
export * from "./memory-provider-factory.js";
|
||||
export * from "./messaging-context.js";
|
||||
export * from "./messaging-delivery.js";
|
||||
|
||||
256
packages/adapters/src/media/ffmpeg.test.ts
Normal file
256
packages/adapters/src/media/ffmpeg.test.ts
Normal file
@ -0,0 +1,256 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { FfmpegProgress } from "./ffmpeg.js";
|
||||
import {
|
||||
createFfmpegProgressParser,
|
||||
detectFfmpeg,
|
||||
extractAudio,
|
||||
FfmpegError,
|
||||
ffmpegInputArgs,
|
||||
ffmpegSeconds,
|
||||
packetDurationMs,
|
||||
parseProbeOutput,
|
||||
probeMedia,
|
||||
renderPoster,
|
||||
runFfmpeg,
|
||||
} from "./ffmpeg.js";
|
||||
import type { TestClips } from "./media-test-support.js";
|
||||
import { createTestClips } from "./media-test-support.js";
|
||||
|
||||
const ffmpeg = await detectFfmpeg();
|
||||
|
||||
describe("ffmpeg progress", () => {
|
||||
it("reports each block, tolerates split chunks and N/A times, and marks the end", () => {
|
||||
const updates: FfmpegProgress[] = [];
|
||||
const feed = createFfmpegProgressParser((update) => updates.push(update));
|
||||
feed("frame=0\nout_time_us=N/A\ntotal_size=48\nprogress=cont");
|
||||
feed("inue\nframe=75\nout_time_us=2995");
|
||||
feed("374\nout_time_ms=2995374\ntotal_size=42300\nprogress=end\n");
|
||||
expect(updates).toEqual([
|
||||
{ totalSize: 48, done: false },
|
||||
{ outTimeUs: 2_995_374, totalSize: 42_300, done: true },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("probe parsing", () => {
|
||||
const stream = (fields: Record<string, unknown>) => ({
|
||||
codec_type: "video",
|
||||
codec_name: "h264",
|
||||
...fields,
|
||||
});
|
||||
|
||||
it("swaps the size of rotated video and reads frame rate and audio", () => {
|
||||
const probe = parseProbeOutput(
|
||||
JSON.stringify({
|
||||
format: { duration: "12.345", format_name: "mov,mp4,m4a,3gp,3g2,mj2" },
|
||||
streams: [
|
||||
stream({
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
avg_frame_rate: "30000/1001",
|
||||
sample_aspect_ratio: "1:1",
|
||||
side_data_list: [{ side_data_type: "Display Matrix", rotation: -90 }],
|
||||
}),
|
||||
{ codec_type: "audio", codec_name: "aac", sample_rate: "48000", channels: 2 },
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(probe).toEqual({
|
||||
durationMs: 12_345,
|
||||
formatName: "mov,mp4,m4a,3gp,3g2,mj2",
|
||||
video: {
|
||||
codec: "h264",
|
||||
width: 1080,
|
||||
height: 1920,
|
||||
rotation: 270,
|
||||
fps: 30000 / 1001,
|
||||
sampleAspectRatio: 1,
|
||||
},
|
||||
audio: { codec: "aac", sampleRate: 48_000, channels: 2 },
|
||||
hasAudio: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("reads legacy rotate tags, skips cover art and falls back to stream durations", () => {
|
||||
const probe = parseProbeOutput(
|
||||
JSON.stringify({
|
||||
format: { format_name: "mov" },
|
||||
streams: [
|
||||
stream({ width: 600, height: 600, disposition: { attached_pic: 1 } }),
|
||||
stream({
|
||||
width: 640,
|
||||
height: 480,
|
||||
avg_frame_rate: "0/0",
|
||||
r_frame_rate: "90000/1",
|
||||
sample_aspect_ratio: "4:3",
|
||||
tags: { rotate: "90" },
|
||||
duration: "4.5",
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(probe.durationMs).toBe(4500);
|
||||
expect(probe.hasAudio).toBe(false);
|
||||
expect(probe.video).toMatchObject({
|
||||
width: 480,
|
||||
height: 640,
|
||||
rotation: 90,
|
||||
fps: 0,
|
||||
sampleAspectRatio: 3 / 4,
|
||||
});
|
||||
});
|
||||
|
||||
it("reads the end of the last packet when the container has no duration", () => {
|
||||
expect(packetDurationMs("0.000000,0.040000\n2.960000,0.040000\nN/A,N/A\n")).toBe(3000);
|
||||
expect(packetDurationMs("")).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ffmpeg arguments", () => {
|
||||
it("opens inputs through the file protocol with the demuxer of their MIME type", () => {
|
||||
expect(
|
||||
ffmpegInputArgs("/a/b.webm", {
|
||||
mimeType: "video/webm",
|
||||
startSeconds: 2,
|
||||
durationSeconds: 1.5,
|
||||
}),
|
||||
).toEqual([
|
||||
...["-ss", "2", "-t", "1.5", "-protocol_whitelist", "file", "-f", "matroska"],
|
||||
...["-i", "file:/a/b.webm"],
|
||||
]);
|
||||
expect(ffmpegInputArgs("/a/b", { mimeType: "application/octet-stream" })).toEqual([
|
||||
...["-protocol_whitelist", "file", "-i", "file:/a/b"],
|
||||
]);
|
||||
expect(ffmpegSeconds(1e-7)).toBe("0");
|
||||
expect(ffmpegSeconds(12.34567)).toBe("12.346");
|
||||
});
|
||||
});
|
||||
|
||||
// CI runners may not have ffmpeg; these run wherever it is installed.
|
||||
describe.skipIf(!ffmpeg)("ffmpeg integration", () => {
|
||||
let clips: TestClips;
|
||||
|
||||
beforeAll(async () => {
|
||||
clips = await createTestClips();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (clips) await rm(clips.dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("probes duration, size, frame rate and audio", async () => {
|
||||
const probe = await probeMedia(clips.landscape, { mimeType: "video/mp4" });
|
||||
expect(probe.durationMs).toBeGreaterThanOrEqual(2900);
|
||||
expect(probe.durationMs).toBeLessThanOrEqual(3100);
|
||||
expect(probe.video).toMatchObject({ codec: "h264", width: 320, height: 180, fps: 25 });
|
||||
expect(probe.hasAudio).toBe(true);
|
||||
const silent = await probeMedia(clips.silentPortrait, { mimeType: "video/mp4" });
|
||||
expect(silent.hasAudio).toBe(false);
|
||||
expect(silent.video).toMatchObject({ width: 180, height: 320 });
|
||||
});
|
||||
|
||||
it("refuses a playlist disguised as an MP4 instead of following its URLs", async () => {
|
||||
const playlist = join(clips.dir, "playlist.mp4");
|
||||
await writeFile(
|
||||
playlist,
|
||||
`#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:1.0,\nfile:${clips.srt}\n#EXT-X-ENDLIST\n`,
|
||||
);
|
||||
await expect(probeMedia(playlist, { mimeType: "video/mp4" })).rejects.toThrow(FfmpegError);
|
||||
});
|
||||
|
||||
it("reports progress up to 1 for a run", async () => {
|
||||
const output = join(clips.dir, "progress.mp4");
|
||||
const fractions: number[] = [];
|
||||
await runFfmpeg(
|
||||
[
|
||||
...ffmpegInputArgs(clips.landscape, { mimeType: "video/mp4" }),
|
||||
...[
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-f",
|
||||
"mp4",
|
||||
`file:${output}`,
|
||||
],
|
||||
],
|
||||
{ durationMs: 3000, onProgress: (fraction) => fractions.push(fraction) },
|
||||
);
|
||||
expect(fractions.at(-1)).toBe(1);
|
||||
expect(fractions).toEqual([...fractions].sort((a, b) => a - b));
|
||||
expect(existsSync(output)).toBe(true);
|
||||
});
|
||||
|
||||
it("fails with the last stderr line, without directory paths, and removes outputs", async () => {
|
||||
const output = join(clips.dir, "broken.mp4");
|
||||
await writeFile(output, "partial");
|
||||
const failure = runFfmpeg(
|
||||
[
|
||||
...ffmpegInputArgs(join(clips.dir, "missing.mp4"), { mimeType: "video/mp4" }),
|
||||
`file:${output}`,
|
||||
],
|
||||
{ outputs: [output] },
|
||||
);
|
||||
const error = await failure.catch((caught: unknown) => caught);
|
||||
expect(error).toBeInstanceOf(FfmpegError);
|
||||
expect((error as Error).message).toMatch(/^ffmpeg failed: .*missing\.mp4.*No such file/);
|
||||
expect((error as Error).message).not.toContain(clips.dir);
|
||||
expect(existsSync(output)).toBe(false);
|
||||
});
|
||||
|
||||
it("kills ffmpeg on abort and removes the partial output", async () => {
|
||||
const output = join(clips.dir, "aborted.mp4");
|
||||
const controller = new AbortController();
|
||||
const run = runFfmpeg(
|
||||
[
|
||||
...["-re", "-f", "lavfi", "-i", "testsrc=size=320x180:rate=25"],
|
||||
...["-t", "60", "-c:v", "libx264", "-preset", "ultrafast", "-f", "mp4", `file:${output}`],
|
||||
],
|
||||
{
|
||||
signal: controller.signal,
|
||||
durationMs: 60_000,
|
||||
onProgress: () => controller.abort(new Error("cancelled")),
|
||||
outputs: [output],
|
||||
},
|
||||
);
|
||||
await expect(run).rejects.toThrow("cancelled");
|
||||
expect(existsSync(output)).toBe(false);
|
||||
});
|
||||
|
||||
it("stops when the output outgrows its cap", async () => {
|
||||
const output = join(clips.dir, "too-large.mp4");
|
||||
const run = runFfmpeg(
|
||||
[
|
||||
...["-f", "lavfi", "-i", "testsrc=size=640x360:rate=25:duration=20"],
|
||||
...["-c:v", "libx264", "-preset", "ultrafast", "-crf", "0", "-f", "mp4", `file:${output}`],
|
||||
],
|
||||
{ maxOutputBytes: 64 * 1024, outputs: [output] },
|
||||
);
|
||||
await expect(run).rejects.toThrow("The result would exceed 0 MB.");
|
||||
expect(existsSync(output)).toBe(false);
|
||||
});
|
||||
|
||||
it("extracts 16 kHz mono WAV for transcription, optionally a range", async () => {
|
||||
const output = join(clips.dir, "speech.wav");
|
||||
await extractAudio(clips.landscape, output, { mimeType: "video/mp4", startMs: 1000 });
|
||||
const probe = await probeMedia(output, { mimeType: "audio/wav" });
|
||||
expect(probe.audio).toEqual({ codec: "pcm_s16le", sampleRate: 16_000, channels: 1 });
|
||||
expect(probe.video).toBeUndefined();
|
||||
expect(Math.abs(probe.durationMs - 2000)).toBeLessThan(100);
|
||||
await expect(
|
||||
extractAudio(clips.silentPortrait, join(clips.dir, "none.wav"), { mimeType: "video/mp4" }),
|
||||
).rejects.toThrow("The file has no audio track.");
|
||||
});
|
||||
|
||||
it("renders a poster frame, falling back to the first frame past the end", async () => {
|
||||
const poster = join(clips.dir, "poster.jpg");
|
||||
await renderPoster(clips.landscape, poster, { mimeType: "video/mp4", atMs: 60_000 });
|
||||
const probe = await probeMedia(poster);
|
||||
expect(probe.video).toMatchObject({ codec: "mjpeg", width: 320, height: 180 });
|
||||
});
|
||||
});
|
||||
603
packages/adapters/src/media/ffmpeg.ts
Normal file
603
packages/adapters/src/media/ffmpeg.ts
Normal file
@ -0,0 +1,603 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { constants } from "node:fs";
|
||||
import { access, rm, stat } from "node:fs/promises";
|
||||
import { setPriority } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
|
||||
/**
|
||||
* ffmpeg and ffprobe from PATH. Every call passes an argument array (never a shell), opens
|
||||
* inputs through the `file` protocol only, and forces the demuxer from the artifact's MIME type
|
||||
* so a crafted upload cannot make ffmpeg probe it as a playlist that reads other files or URLs.
|
||||
*/
|
||||
|
||||
export interface FfmpegInfo {
|
||||
ffmpegPath: string;
|
||||
ffprobePath: string;
|
||||
version: string;
|
||||
}
|
||||
|
||||
export interface MediaProbe {
|
||||
durationMs: number;
|
||||
formatName: string;
|
||||
/** Width and height as displayed: swapped when the stream is rotated by 90 or 270 degrees. */
|
||||
video?: {
|
||||
codec: string;
|
||||
width: number;
|
||||
height: number;
|
||||
rotation: 0 | 90 | 180 | 270;
|
||||
/** Frames per second; 0 when unknown or implausible (variable frame rate files). */
|
||||
fps: number;
|
||||
/** Pixel aspect ratio of the displayed frame; 1 for square pixels. */
|
||||
sampleAspectRatio: number;
|
||||
};
|
||||
audio?: { codec: string; sampleRate: number; channels: number };
|
||||
hasAudio: boolean;
|
||||
}
|
||||
|
||||
export interface FfmpegProgress {
|
||||
outTimeUs?: number;
|
||||
totalSize?: number;
|
||||
done: boolean;
|
||||
}
|
||||
|
||||
export interface RunFfmpegOptions {
|
||||
signal?: AbortSignal;
|
||||
/** Expected output duration; progress is reported as out_time / durationMs. */
|
||||
durationMs?: number;
|
||||
onProgress?: (fraction: number) => void;
|
||||
cwd?: string;
|
||||
/** Kills ffmpeg once the output grows past this size. */
|
||||
maxOutputBytes?: number;
|
||||
/** Files removed when the run fails or is aborted. */
|
||||
outputs?: readonly string[];
|
||||
/** Kills ffmpeg when it reports no progress for this long. */
|
||||
stallTimeoutMs?: number;
|
||||
ffmpegPath?: string;
|
||||
}
|
||||
|
||||
export class FfmpegError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly exitCode: number | null,
|
||||
readonly stderrTail: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "FfmpegError";
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_STALL_TIMEOUT_MS = 2 * 60_000;
|
||||
const PROBE_TIMEOUT_MS = 60_000;
|
||||
const STDERR_TAIL_BYTES = 8 * 1024;
|
||||
const KILL_GRACE_MS = 5_000;
|
||||
const MAX_PLAUSIBLE_FPS = 240;
|
||||
|
||||
const DEMUXERS: Record<string, string> = {
|
||||
"video/mp4": "mov",
|
||||
"video/quicktime": "mov",
|
||||
"audio/mp4": "mov",
|
||||
"video/webm": "matroska",
|
||||
"audio/webm": "matroska",
|
||||
"audio/mpeg": "mp3",
|
||||
"audio/wav": "wav",
|
||||
"audio/ogg": "ogg",
|
||||
};
|
||||
|
||||
let detection: Promise<FfmpegInfo | null> | undefined;
|
||||
|
||||
/** Finds ffmpeg and ffprobe on PATH once per process; null when either is missing or broken. */
|
||||
export function detectFfmpeg(): Promise<FfmpegInfo | null> {
|
||||
detection ??= locateFfmpeg();
|
||||
return detection;
|
||||
}
|
||||
|
||||
async function locateFfmpeg(): Promise<FfmpegInfo | null> {
|
||||
const [ffmpegPath, ffprobePath] = await Promise.all([
|
||||
findExecutable("ffmpeg"),
|
||||
findExecutable("ffprobe"),
|
||||
]);
|
||||
if (!ffmpegPath || !ffprobePath) return null;
|
||||
try {
|
||||
const [ffmpeg, ffprobe] = await Promise.all([
|
||||
capture(ffmpegPath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }),
|
||||
capture(ffprobePath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }),
|
||||
]);
|
||||
if (ffmpeg.code !== 0 || ffprobe.code !== 0) return null;
|
||||
const version = /version (\S+)/.exec(ffmpeg.stdout)?.[1] ?? "unknown";
|
||||
return { ffmpegPath, ffprobePath, version };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function findExecutable(name: string): Promise<string | null> {
|
||||
const extensions =
|
||||
process.platform === "win32"
|
||||
? (process.env.PATHEXT ?? ".EXE").split(";").filter(Boolean)
|
||||
: [""];
|
||||
for (const directory of (process.env.PATH ?? "").split(delimiter)) {
|
||||
if (!directory) continue;
|
||||
for (const extension of extensions) {
|
||||
const candidate = join(directory, `${name}${extension}`);
|
||||
try {
|
||||
await access(candidate, constants.X_OK);
|
||||
if ((await stat(candidate)).isFile()) return candidate;
|
||||
} catch {
|
||||
// Not in this directory.
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function requireFfmpeg(): Promise<FfmpegInfo> {
|
||||
const info = await detectFfmpeg();
|
||||
if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server.");
|
||||
return info;
|
||||
}
|
||||
|
||||
/** ffmpeg time argument with millisecond precision and no exponent notation. */
|
||||
export function ffmpegSeconds(seconds: number): string {
|
||||
return String(Number(Math.max(0, seconds).toFixed(3)));
|
||||
}
|
||||
|
||||
/** Input arguments for a local file: optional seek, file protocol only, forced demuxer. */
|
||||
export function ffmpegInputArgs(
|
||||
path: string,
|
||||
options: { mimeType?: string; startSeconds?: number; durationSeconds?: number } = {},
|
||||
): string[] {
|
||||
const demuxer = options.mimeType ? DEMUXERS[options.mimeType] : undefined;
|
||||
return [
|
||||
...(options.startSeconds ? ["-ss", ffmpegSeconds(options.startSeconds)] : []),
|
||||
...(options.durationSeconds !== undefined
|
||||
? ["-t", ffmpegSeconds(options.durationSeconds)]
|
||||
: []),
|
||||
"-protocol_whitelist",
|
||||
"file",
|
||||
...(demuxer ? ["-f", demuxer] : []),
|
||||
"-i",
|
||||
`file:${path}`,
|
||||
];
|
||||
}
|
||||
|
||||
export async function probeMedia(
|
||||
path: string,
|
||||
options: { signal?: AbortSignal; mimeType?: string; ffprobePath?: string } = {},
|
||||
): Promise<MediaProbe> {
|
||||
const ffprobePath = options.ffprobePath ?? (await requireFfmpeg()).ffprobePath;
|
||||
const input = ffmpegInputArgs(path, { mimeType: options.mimeType });
|
||||
const result = await capture(
|
||||
ffprobePath,
|
||||
["-v", "error", ...input, "-print_format", "json", "-show_format", "-show_streams"],
|
||||
{ signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS },
|
||||
);
|
||||
if (result.code !== 0) throw ffmpegFailure("ffprobe", result.code, result.stderr);
|
||||
const probe = parseProbeOutput(result.stdout);
|
||||
if (probe.durationMs > 0 || (!probe.video && !probe.audio)) return probe;
|
||||
// Recordings streamed by browsers often carry no duration header; read it from the packets.
|
||||
const packets = await capture(
|
||||
ffprobePath,
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
...input,
|
||||
"-select_streams",
|
||||
probe.video ? "v:0" : "a:0",
|
||||
"-show_entries",
|
||||
"packet=pts_time,duration_time",
|
||||
"-of",
|
||||
"csv=p=0",
|
||||
],
|
||||
{ signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS * 5 },
|
||||
);
|
||||
if (packets.code !== 0) throw ffmpegFailure("ffprobe", packets.code, packets.stderr);
|
||||
return { ...probe, durationMs: packetDurationMs(packets.stdout) };
|
||||
}
|
||||
|
||||
/** End of the last packet, from ffprobe `packet=pts_time,duration_time` CSV lines. */
|
||||
export function packetDurationMs(csv: string): number {
|
||||
let end = 0;
|
||||
for (const line of csv.split("\n")) {
|
||||
const [pts, duration] = line.split(",").map(Number);
|
||||
if (pts === undefined || !Number.isFinite(pts)) continue;
|
||||
end = Math.max(end, pts + (duration !== undefined && Number.isFinite(duration) ? duration : 0));
|
||||
}
|
||||
return Math.round(end * 1000);
|
||||
}
|
||||
|
||||
interface ProbeStream {
|
||||
codec_type?: string;
|
||||
codec_name?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
avg_frame_rate?: string;
|
||||
r_frame_rate?: string;
|
||||
sample_aspect_ratio?: string;
|
||||
sample_rate?: string;
|
||||
channels?: number;
|
||||
duration?: string;
|
||||
disposition?: { attached_pic?: number };
|
||||
tags?: { rotate?: string };
|
||||
side_data_list?: { rotation?: number }[];
|
||||
}
|
||||
|
||||
/** Parses `ffprobe -print_format json -show_format -show_streams` output. */
|
||||
export function parseProbeOutput(json: string): MediaProbe {
|
||||
const data = JSON.parse(json) as {
|
||||
format?: { duration?: string; format_name?: string };
|
||||
streams?: ProbeStream[];
|
||||
};
|
||||
const streams = data.streams ?? [];
|
||||
// Cover art in audio files is a single attached picture, not a video track.
|
||||
const video = streams.find(
|
||||
(stream) => stream.codec_type === "video" && stream.disposition?.attached_pic !== 1,
|
||||
);
|
||||
const audio = streams.find((stream) => stream.codec_type === "audio");
|
||||
const seconds =
|
||||
positive(Number(data.format?.duration)) ??
|
||||
Math.max(0, ...streams.map((stream) => positive(Number(stream.duration)) ?? 0));
|
||||
return {
|
||||
durationMs: Math.round(seconds * 1000),
|
||||
formatName: data.format?.format_name ?? "",
|
||||
...(video?.width && video.height
|
||||
? { video: probeVideo(video, video.width, video.height) }
|
||||
: {}),
|
||||
...(audio
|
||||
? {
|
||||
audio: {
|
||||
codec: audio.codec_name ?? "",
|
||||
sampleRate: Number(audio.sample_rate) || 0,
|
||||
channels: audio.channels ?? 0,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
hasAudio: Boolean(audio),
|
||||
};
|
||||
}
|
||||
|
||||
function probeVideo(stream: ProbeStream, width: number, height: number) {
|
||||
const rotation = normalizeRotation(
|
||||
stream.side_data_list?.find((entry) => typeof entry.rotation === "number")?.rotation ??
|
||||
Number(stream.tags?.rotate ?? 0),
|
||||
);
|
||||
const sideways = rotation === 90 || rotation === 270;
|
||||
const sar = ratio(stream.sample_aspect_ratio, ":") || 1;
|
||||
const fps = ratio(stream.avg_frame_rate, "/") || ratio(stream.r_frame_rate, "/");
|
||||
return {
|
||||
codec: stream.codec_name ?? "",
|
||||
width: sideways ? height : width,
|
||||
height: sideways ? width : height,
|
||||
rotation,
|
||||
fps: fps > 0 && fps <= MAX_PLAUSIBLE_FPS ? fps : 0,
|
||||
sampleAspectRatio: sideways ? 1 / sar : sar,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeRotation(degrees: number): 0 | 90 | 180 | 270 {
|
||||
if (!Number.isFinite(degrees)) return 0;
|
||||
const quarter = (((Math.round(degrees / 90) % 4) + 4) % 4) as 0 | 1 | 2 | 3;
|
||||
return ([0, 90, 180, 270] as const)[quarter];
|
||||
}
|
||||
|
||||
function ratio(value: string | undefined, separator: string): number {
|
||||
if (!value) return 0;
|
||||
const [numerator, denominator] = value.split(separator).map(Number);
|
||||
if (!numerator || !denominator || !Number.isFinite(numerator / denominator)) return 0;
|
||||
return numerator / denominator;
|
||||
}
|
||||
|
||||
function positive(value: number): number | undefined {
|
||||
return Number.isFinite(value) && value > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
/** Feeds ffmpeg `-progress` output (key=value lines) and reports each completed block. */
|
||||
export function createFfmpegProgressParser(
|
||||
onUpdate: (progress: FfmpegProgress) => void,
|
||||
): (chunk: string) => void {
|
||||
let buffer = "";
|
||||
let current: Omit<FfmpegProgress, "done"> = {};
|
||||
return (chunk) => {
|
||||
buffer += chunk;
|
||||
for (let newline = buffer.indexOf("\n"); newline >= 0; newline = buffer.indexOf("\n")) {
|
||||
const line = buffer.slice(0, newline).trim();
|
||||
buffer = buffer.slice(newline + 1);
|
||||
const separator = line.indexOf("=");
|
||||
if (separator < 0) continue;
|
||||
const key = line.slice(0, separator);
|
||||
const value = line.slice(separator + 1).trim();
|
||||
const number = Number(value);
|
||||
// out_time_ms is microseconds too (a long-standing ffmpeg naming bug).
|
||||
if ((key === "out_time_us" || key === "out_time_ms") && value && number >= 0) {
|
||||
current.outTimeUs = number;
|
||||
} else if (key === "total_size" && value && number >= 0) {
|
||||
current.totalSize = number;
|
||||
} else if (key === "progress") {
|
||||
onUpdate({ ...current, done: value === "end" });
|
||||
current = {};
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Runs ffmpeg with progress on stdout; rejects with the stderr tail when it fails. */
|
||||
export async function runFfmpeg(
|
||||
args: readonly string[],
|
||||
options: RunFfmpegOptions = {},
|
||||
): Promise<void> {
|
||||
options.signal?.throwIfAborted();
|
||||
const ffmpegPath = options.ffmpegPath ?? (await requireFfmpeg()).ffmpegPath;
|
||||
try {
|
||||
await spawnFfmpeg(ffmpegPath, args, options);
|
||||
} catch (error) {
|
||||
await Promise.all((options.outputs ?? []).map((path) => rm(path, { force: true })));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function spawnFfmpeg(
|
||||
ffmpegPath: string,
|
||||
args: readonly string[],
|
||||
options: RunFfmpegOptions,
|
||||
): Promise<void> {
|
||||
const { signal, durationMs, onProgress, maxOutputBytes } = options;
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(
|
||||
ffmpegPath,
|
||||
[
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-nostats",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-y",
|
||||
...args,
|
||||
],
|
||||
{ cwd: options.cwd, stdio: ["ignore", "pipe", "pipe"], windowsHide: true },
|
||||
);
|
||||
if (child.pid) {
|
||||
try {
|
||||
// Long encodes must not starve the API process that shares this machine.
|
||||
setPriority(child.pid, 10);
|
||||
} catch {
|
||||
// Lowering priority is best effort.
|
||||
}
|
||||
}
|
||||
let stderr = "";
|
||||
let settled = false;
|
||||
let reported = 0;
|
||||
let failure: unknown;
|
||||
let stallTimer: NodeJS.Timeout | undefined;
|
||||
const finish = (error?: unknown) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(stallTimer);
|
||||
signal?.removeEventListener("abort", abort);
|
||||
if (error === undefined) resolve();
|
||||
else reject(error);
|
||||
};
|
||||
// Settles once the process is gone, so callers can delete its files right away.
|
||||
const kill = (error: unknown) => {
|
||||
if (failure !== undefined || settled) return;
|
||||
failure = error;
|
||||
clearTimeout(stallTimer);
|
||||
child.kill("SIGKILL");
|
||||
setTimeout(() => finish(error), KILL_GRACE_MS).unref?.();
|
||||
};
|
||||
const abort = () => kill(signal?.reason ?? new Error("aborted"));
|
||||
const stallTimeoutMs = options.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;
|
||||
const armStallTimer = () => {
|
||||
clearTimeout(stallTimer);
|
||||
stallTimer = setTimeout(
|
||||
() => kill(new FfmpegError("ffmpeg stopped making progress.", null, tail(stderr))),
|
||||
stallTimeoutMs,
|
||||
);
|
||||
stallTimer.unref?.();
|
||||
};
|
||||
const parse = createFfmpegProgressParser((progress) => {
|
||||
if (failure !== undefined) return;
|
||||
if (maxOutputBytes !== undefined && (progress.totalSize ?? 0) > maxOutputBytes) {
|
||||
kill(new Error(outputTooLargeMessage(maxOutputBytes)));
|
||||
return;
|
||||
}
|
||||
if (!onProgress) return;
|
||||
const fraction = progress.done
|
||||
? 1
|
||||
: durationMs && progress.outTimeUs !== undefined
|
||||
? Math.min(1, progress.outTimeUs / (durationMs * 1000))
|
||||
: 0;
|
||||
if (fraction > reported) {
|
||||
reported = fraction;
|
||||
onProgress(fraction);
|
||||
}
|
||||
});
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
armStallTimer();
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
if (failure === undefined) armStallTimer();
|
||||
parse(chunk);
|
||||
});
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
stderr = tail(stderr + chunk);
|
||||
});
|
||||
child.on("error", (error) => finish(failure ?? error));
|
||||
child.on("close", (code) => {
|
||||
if (failure !== undefined) finish(failure);
|
||||
else if (code === 0) finish();
|
||||
else finish(ffmpegFailure("ffmpeg", code, stderr));
|
||||
});
|
||||
if (signal?.aborted) abort();
|
||||
});
|
||||
}
|
||||
|
||||
export function outputTooLargeMessage(maxBytes: number): string {
|
||||
return `The result would exceed ${Math.round(maxBytes / (1024 * 1024))} MB.`;
|
||||
}
|
||||
|
||||
function tail(text: string): string {
|
||||
return text.length > STDERR_TAIL_BYTES ? text.slice(-STDERR_TAIL_BYTES) : text;
|
||||
}
|
||||
|
||||
function ffmpegFailure(tool: string, code: number | null, stderr: string): FfmpegError {
|
||||
// The cause is usually the line before ffmpeg's generic summary ("Conversion failed!").
|
||||
const lines = stderr
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-2);
|
||||
// Name files by their base name only; the job's temp paths mean nothing to the reader.
|
||||
const detail = lines.join(" ").replaceAll(/(?:file:)?(?:[A-Za-z]:)?[\\/][^\s'"]*[\\/]/g, "");
|
||||
return new FfmpegError(
|
||||
detail ? `${tool} failed: ${detail}` : `${tool} failed with exit code ${code}.`,
|
||||
code,
|
||||
tail(stderr),
|
||||
);
|
||||
}
|
||||
|
||||
/** Extracts the first audio track as 16 kHz mono 16-bit WAV, the format speech models expect. */
|
||||
export async function extractAudio(
|
||||
input: string,
|
||||
output: string,
|
||||
options: {
|
||||
signal?: AbortSignal;
|
||||
mimeType?: string;
|
||||
startMs?: number;
|
||||
endMs?: number;
|
||||
onProgress?: (fraction: number) => void;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
const tools = await requireFfmpeg();
|
||||
const probe = await probeMedia(input, {
|
||||
signal: options.signal,
|
||||
mimeType: options.mimeType,
|
||||
ffprobePath: tools.ffprobePath,
|
||||
});
|
||||
if (!probe.hasAudio) throw new Error("The file has no audio track.");
|
||||
const startMs = Math.max(0, options.startMs ?? 0);
|
||||
const endMs = Math.min(probe.durationMs, options.endMs ?? probe.durationMs);
|
||||
if (endMs <= startMs) throw new Error("The audio range is empty.");
|
||||
await runFfmpeg(
|
||||
[
|
||||
...ffmpegInputArgs(input, {
|
||||
mimeType: options.mimeType,
|
||||
startSeconds: startMs / 1000,
|
||||
durationSeconds: (endMs - startMs) / 1000,
|
||||
}),
|
||||
"-map",
|
||||
"0:a:0",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
"-map_metadata",
|
||||
"-1",
|
||||
"-f",
|
||||
"wav",
|
||||
`file:${output}`,
|
||||
],
|
||||
{
|
||||
signal: options.signal,
|
||||
durationMs: endMs - startMs,
|
||||
onProgress: options.onProgress,
|
||||
outputs: [output],
|
||||
ffmpegPath: tools.ffmpegPath,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Writes one JPEG frame at atMs (default 0), falling back to the first frame past the end. */
|
||||
export async function renderPoster(
|
||||
input: string,
|
||||
output: string,
|
||||
options: { signal?: AbortSignal; mimeType?: string; atMs?: number } = {},
|
||||
): Promise<void> {
|
||||
const ffmpegPath = (await requireFfmpeg()).ffmpegPath;
|
||||
const render = (atMs: number) =>
|
||||
runFfmpeg(
|
||||
[
|
||||
...ffmpegInputArgs(input, { mimeType: options.mimeType, startSeconds: atMs / 1000 }),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-c:v",
|
||||
"mjpeg",
|
||||
"-q:v",
|
||||
"3",
|
||||
"-map_metadata",
|
||||
"-1",
|
||||
"-f",
|
||||
"image2",
|
||||
"-update",
|
||||
"1",
|
||||
`file:${output}`,
|
||||
],
|
||||
{ signal: options.signal, outputs: [output], ffmpegPath },
|
||||
);
|
||||
const atMs = Math.max(0, options.atMs ?? 0);
|
||||
await render(atMs);
|
||||
// Seeking into the last frame's gap writes nothing; the first frame is better than no poster.
|
||||
if (atMs > 0 && !(await nonEmptyFile(output))) await render(0);
|
||||
if (!(await nonEmptyFile(output))) throw new Error("Could not render a poster frame.");
|
||||
}
|
||||
|
||||
async function nonEmptyFile(path: string): Promise<boolean> {
|
||||
try {
|
||||
return (await stat(path)).size > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
interface CaptureResult {
|
||||
code: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
function capture(
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
options: { signal?: AbortSignal; timeoutMs: number },
|
||||
): Promise<CaptureResult> {
|
||||
options.signal?.throwIfAborted();
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], windowsHide: true });
|
||||
const stdout: Buffer[] = [];
|
||||
let stderr = "";
|
||||
let settled = false;
|
||||
const finish = (error: unknown, result?: CaptureResult) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
options.signal?.removeEventListener("abort", abort);
|
||||
if (result) resolve(result);
|
||||
else reject(error);
|
||||
};
|
||||
const kill = (error: unknown) => {
|
||||
child.kill("SIGKILL");
|
||||
finish(error);
|
||||
};
|
||||
const abort = () => kill(options.signal?.reason ?? new Error("aborted"));
|
||||
const timer = setTimeout(
|
||||
() => kill(new Error(`${command} timed out after ${options.timeoutMs} ms`)),
|
||||
options.timeoutMs,
|
||||
);
|
||||
timer.unref?.();
|
||||
options.signal?.addEventListener("abort", abort, { once: true });
|
||||
child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk));
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
stderr = tail(stderr + chunk);
|
||||
});
|
||||
child.on("error", (error) => finish(error));
|
||||
child.on("close", (code) =>
|
||||
finish(undefined, { code, stdout: Buffer.concat(stdout).toString("utf8"), stderr }),
|
||||
);
|
||||
});
|
||||
}
|
||||
106
packages/adapters/src/media/media-test-support.ts
Normal file
106
packages/adapters/src/media/media-test-support.ts
Normal file
@ -0,0 +1,106 @@
|
||||
import { copyFile, mkdtemp, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit";
|
||||
import type { MediaOutput, MediaOutputKind } from "@rakazo/contracts";
|
||||
import { attachmentKindOf } from "@rakazo/contracts";
|
||||
import { runFfmpeg } from "./ffmpeg.js";
|
||||
|
||||
export interface TestClips {
|
||||
dir: string;
|
||||
/** 320×180, 25 fps, 3 s, 440 Hz tone. */
|
||||
landscape: string;
|
||||
/** 180×320, 30 fps, 2 s, no audio. */
|
||||
silentPortrait: string;
|
||||
srt: string;
|
||||
vtt: string;
|
||||
}
|
||||
|
||||
/** Tiny generated clips; nothing binary is committed. */
|
||||
export async function createTestClips(): Promise<TestClips> {
|
||||
const dir = await mkdtemp(join(tmpdir(), "evoo-media-test-"));
|
||||
const landscape = join(dir, "landscape.mp4");
|
||||
const silentPortrait = join(dir, "silent-portrait.mp4");
|
||||
const encode = ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"];
|
||||
await runFfmpeg([
|
||||
...["-f", "lavfi", "-i", "testsrc=size=320x180:rate=25:duration=3"],
|
||||
...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=3"],
|
||||
...encode,
|
||||
...["-c:a", "aac", "-shortest", `file:${landscape}`],
|
||||
]);
|
||||
await runFfmpeg([
|
||||
...["-f", "lavfi", "-i", "testsrc2=size=180x320:rate=30:duration=2"],
|
||||
...encode,
|
||||
`file:${silentPortrait}`,
|
||||
]);
|
||||
const srt = join(dir, "captions.srt");
|
||||
await writeFile(
|
||||
srt,
|
||||
"1\n00:00:00,100 --> 00:00:01,500\nHello from the subtitles\n\n2\n00:00:01,500 --> 00:00:02,900\nZweite Zeile äöü\n",
|
||||
);
|
||||
const vtt = join(dir, "captions.vtt");
|
||||
await writeFile(vtt, "WEBVTT\n\n00:00.100 --> 00:01.500\nHello from WebVTT\n");
|
||||
return { dir, landscape, silentPortrait, srt, vtt };
|
||||
}
|
||||
|
||||
export interface TestArtifact {
|
||||
path: string;
|
||||
name: string;
|
||||
mimeType: string;
|
||||
kind?: MediaOutputKind;
|
||||
}
|
||||
|
||||
/** A MediaJobContext over local files that records progress and outputs. */
|
||||
export async function fakeMediaJobContext(input: unknown, artifacts: Record<string, TestArtifact>) {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), "evoo-media-job-"));
|
||||
const controller = new AbortController();
|
||||
const progress: number[] = [];
|
||||
const outputs: (MediaJobOutputFile & { size: number; posterSize?: number })[] = [];
|
||||
const context: MediaJobContext = {
|
||||
jobId: "job-1",
|
||||
spaceId: "space-1",
|
||||
userId: "user-1",
|
||||
botId: null,
|
||||
input,
|
||||
tempDir,
|
||||
signal: controller.signal,
|
||||
progress: async (fraction) => {
|
||||
progress.push(fraction);
|
||||
},
|
||||
fetchArtifact: async (artifactId): Promise<MediaJobInputFile> => {
|
||||
const artifact = artifacts[artifactId];
|
||||
if (!artifact) throw new Error(`Artifact ${artifactId} not found.`);
|
||||
const path = join(tempDir, `input-${artifactId}`);
|
||||
await copyFile(artifact.path, path);
|
||||
return {
|
||||
artifactId,
|
||||
path,
|
||||
name: artifact.name,
|
||||
mimeType: artifact.mimeType,
|
||||
kind: artifact.kind ?? attachmentKindOf(artifact.mimeType),
|
||||
size: (await stat(path)).size,
|
||||
};
|
||||
},
|
||||
addOutput: async (file): Promise<MediaOutput> => {
|
||||
// Files must exist when handed over; the runner stores them before the job ends.
|
||||
const size = (await stat(file.path)).size;
|
||||
const posterSize = file.posterPath ? (await stat(file.posterPath)).size : undefined;
|
||||
outputs.push({ ...file, size, ...(posterSize !== undefined ? { posterSize } : {}) });
|
||||
const index = outputs.length;
|
||||
return {
|
||||
artifactId: `output-${index}`,
|
||||
kind: attachmentKindOf(file.mimeType),
|
||||
mimeType: file.mimeType,
|
||||
name: file.name,
|
||||
size,
|
||||
...(file.width ? { width: file.width } : {}),
|
||||
...(file.height ? { height: file.height } : {}),
|
||||
...(file.durationMs !== undefined ? { durationMs: file.durationMs } : {}),
|
||||
...(file.posterPath ? { posterArtifactId: `poster-${index}` } : {}),
|
||||
...(file.parentArtifactId ? { parentArtifactId: file.parentArtifactId } : {}),
|
||||
...(file.preset ? { preset: file.preset } : {}),
|
||||
};
|
||||
},
|
||||
};
|
||||
return { context, controller, progress, outputs, tempDir };
|
||||
}
|
||||
301
packages/adapters/src/media/video-edit-plan.test.ts
Normal file
301
packages/adapters/src/media/video-edit-plan.test.ts
Normal file
@ -0,0 +1,301 @@
|
||||
import type { EditVideoStep } from "@rakazo/contracts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { MediaProbe } from "./ffmpeg.js";
|
||||
import type { PlanInput } from "./video-edit-plan.js";
|
||||
import { atempoFilters, planVideoEdit, subtitleStyle } from "./video-edit-plan.js";
|
||||
|
||||
function probe(
|
||||
overrides: Partial<NonNullable<MediaProbe["video"]>> & {
|
||||
durationMs?: number;
|
||||
hasAudio?: boolean;
|
||||
} = {},
|
||||
): MediaProbe {
|
||||
const { durationMs = 10_000, hasAudio = true, ...video } = overrides;
|
||||
return {
|
||||
durationMs,
|
||||
formatName: "mov,mp4,m4a,3gp,3g2,mj2",
|
||||
video: {
|
||||
codec: "h264",
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
rotation: 0,
|
||||
fps: 25,
|
||||
sampleAspectRatio: 1,
|
||||
...video,
|
||||
},
|
||||
...(hasAudio ? { audio: { codec: "aac", sampleRate: 44_100, channels: 2 } } : {}),
|
||||
hasAudio,
|
||||
};
|
||||
}
|
||||
|
||||
const source: PlanInput = {
|
||||
path: "/jobs/1/input-source",
|
||||
mimeType: "video/mp4",
|
||||
name: "holiday.mp4",
|
||||
probe: probe(),
|
||||
};
|
||||
|
||||
function plan(steps: EditVideoStep[], extra: Partial<Parameters<typeof planVideoEdit>[0]> = {}) {
|
||||
return planVideoEdit({ source, steps, outputPath: "/jobs/1/output.mp4", ...extra });
|
||||
}
|
||||
|
||||
function graphOf(args: string[]): string {
|
||||
return args[args.indexOf("-filter_complex") + 1]!;
|
||||
}
|
||||
|
||||
describe("planVideoEdit", () => {
|
||||
it("seeks the input for a leading trim and encodes H.264/AAC MP4", () => {
|
||||
const result = plan([{ op: "trim", start: 1.5, end: 4 }]);
|
||||
expect(result.args).toEqual([
|
||||
...["-ss", "1.5", "-t", "2.5", "-protocol_whitelist", "file", "-f", "mov"],
|
||||
...["-i", "file:/jobs/1/input-source"],
|
||||
"-filter_complex",
|
||||
"[0:v:0]format=yuv420p[v1];[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a2]",
|
||||
...["-map", "[v1]", "-map", "[a2]", "-c:a", "aac", "-b:a", "160k"],
|
||||
...["-c:v", "libx264", "-preset", "veryfast", "-crf", "21", "-pix_fmt", "yuv420p"],
|
||||
...["-movflags", "+faststart", "-map_metadata", "-1", "-map_chapters", "-1"],
|
||||
...["-f", "mp4", "file:/jobs/1/output.mp4"],
|
||||
]);
|
||||
expect(result).toMatchObject({ kind: "video", durationMs: 2500, width: 1920, height: 1080 });
|
||||
});
|
||||
|
||||
it("keeps sorted, merged segments from one seek with split, trims and concat", () => {
|
||||
const result = plan([
|
||||
{
|
||||
op: "keep",
|
||||
segments: [
|
||||
{ start: 6, end: 8 },
|
||||
{ start: 2, end: 3 },
|
||||
{ start: 2.5, end: 4 },
|
||||
],
|
||||
},
|
||||
]);
|
||||
expect(result.args.slice(0, 4)).toEqual(["-ss", "2", "-t", "6"]);
|
||||
const graph = graphOf(result.args);
|
||||
expect(graph).toContain("[0:v:0]split=2[v1][v2]");
|
||||
expect(graph).toContain("[v1]trim=start=0:end=2,setpts=PTS-STARTPTS[v3]");
|
||||
expect(graph).toContain("[v2]trim=start=4:end=6,setpts=PTS-STARTPTS[v4]");
|
||||
expect(graph).toContain("[0:a:0]asplit=2[a5][a6]");
|
||||
expect(graph).toContain("[v3][a7][v4][a8]concat=n=2:v=1:a=1[v9][a10]");
|
||||
expect(result.durationMs).toBe(4000);
|
||||
});
|
||||
|
||||
it("applies later cuts on the timeline left by earlier steps", () => {
|
||||
const result = plan([
|
||||
{ op: "speed", factor: 2 },
|
||||
{ op: "trim", start: 1, end: 99 },
|
||||
]);
|
||||
expect(graphOf(result.args)).toContain(
|
||||
"[0:v:0]setpts=PTS/2,fps=25,trim=start=1:end=5,setpts=PTS-STARTPTS,format=yuv420p[v1]",
|
||||
);
|
||||
expect(result.durationMs).toBe(4000);
|
||||
});
|
||||
|
||||
it("rejects ranges outside the video and too short ones", () => {
|
||||
expect(() => plan([{ op: "trim", start: 12, end: 18 }])).toThrow(
|
||||
"trim 12–18 s is outside the 10 s video.",
|
||||
);
|
||||
expect(() =>
|
||||
plan([
|
||||
{ op: "speed", factor: 2 },
|
||||
{
|
||||
op: "keep",
|
||||
segments: [
|
||||
{ start: 1, end: 2 },
|
||||
{ start: 6, end: 7 },
|
||||
],
|
||||
},
|
||||
]),
|
||||
).toThrow("keep 6–7 s is outside the 5 s video.");
|
||||
expect(() => plan([{ op: "trim", start: 9.95, end: 12 }])).toThrow("shorter than 0.1 s");
|
||||
});
|
||||
|
||||
it("appends clips fitted to the frame, with silence for clips without audio", () => {
|
||||
const clip: PlanInput = {
|
||||
path: "/jobs/1/input-clip",
|
||||
mimeType: "video/webm",
|
||||
name: "clip.webm",
|
||||
probe: probe({ width: 1080, height: 1920, fps: 30, durationMs: 2000, hasAudio: false }),
|
||||
};
|
||||
const result = plan([{ op: "concat", artifact_ids: ["clip"] }], {
|
||||
clips: new Map([["clip", clip]]),
|
||||
});
|
||||
expect(result.args).toContain("matroska");
|
||||
const graph = graphOf(result.args);
|
||||
expect(graph).toContain("[0:v:0]setsar=1,format=yuv420p[v1]");
|
||||
expect(graph).toContain(
|
||||
"[1:v:0]scale=1920:1080:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1920:1080:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=25,format=yuv420p[v3]",
|
||||
);
|
||||
expect(graph).toContain(
|
||||
"anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=2,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a4]",
|
||||
);
|
||||
expect(graph).toContain("[v1][a2][v3][a4]concat=n=2:v=1:a=1");
|
||||
expect(result.durationMs).toBe(12_000);
|
||||
});
|
||||
|
||||
it("adds silence to a silent source when an appended clip has audio", () => {
|
||||
const silent = { ...source, probe: probe({ hasAudio: false }) };
|
||||
const result = planVideoEdit({
|
||||
source: silent,
|
||||
steps: [{ op: "concat", artifact_ids: ["clip"] }],
|
||||
clips: new Map([["clip", source]]),
|
||||
outputPath: "/out.mp4",
|
||||
});
|
||||
expect(graphOf(result.args)).toContain(
|
||||
"anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=10",
|
||||
);
|
||||
expect(result.args.filter((arg) => arg === "-map")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("appends clips with the last reframe's fit", () => {
|
||||
const result = plan(
|
||||
[
|
||||
{ op: "reframe", preset: "vertical", fit: "crop" },
|
||||
{ op: "concat", artifact_ids: ["clip"] },
|
||||
],
|
||||
{ clips: new Map([["clip", source]]) },
|
||||
);
|
||||
expect(graphOf(result.args)).toContain(
|
||||
"[1:v:0]scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920,setsar=1,fps=25",
|
||||
);
|
||||
});
|
||||
|
||||
it("reframes by crop, pad or blur, and only scales when the aspect already matches", () => {
|
||||
expect(graphOf(plan([{ op: "reframe", preset: "square", fit: "crop" }]).args)).toContain(
|
||||
"scale=1080:1080:force_original_aspect_ratio=increase,crop=1080:1080,setsar=1",
|
||||
);
|
||||
expect(graphOf(plan([{ op: "reframe", preset: "portrait", fit: "pad" }]).args)).toContain(
|
||||
"pad=1080:1350:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1",
|
||||
);
|
||||
const blur = plan([{ op: "reframe", preset: "vertical", fit: "blur" }]);
|
||||
expect(graphOf(blur.args)).toBe(
|
||||
[
|
||||
"[0:v:0]split=2[v1][v2]",
|
||||
"[v1]scale=270:480:force_original_aspect_ratio=increase,crop=270:480,boxblur=27:2,scale=1080:1920,setsar=1[v3]",
|
||||
"[v2]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,setsar=1[v4]",
|
||||
"[v3][v4]overlay=(W-w)/2:(H-h)/2[v5]",
|
||||
"[v5]format=yuv420p[v6]",
|
||||
"[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a7]",
|
||||
].join(";"),
|
||||
);
|
||||
expect(blur).toMatchObject({ width: 1080, height: 1920 });
|
||||
expect(graphOf(plan([{ op: "reframe", preset: "landscape", fit: "blur" }]).args)).toContain(
|
||||
"[0:v:0]scale=1920:1080,setsar=1,format=yuv420p",
|
||||
);
|
||||
});
|
||||
|
||||
it("changes speed with atempo chains and keeps the source frame rate", () => {
|
||||
const fast = plan([{ op: "speed", factor: 4 }]);
|
||||
expect(graphOf(fast.args)).toContain("setpts=PTS/4,fps=25");
|
||||
expect(graphOf(fast.args)).toContain("atempo=2,atempo=2");
|
||||
expect(fast.durationMs).toBe(2500);
|
||||
expect(atempoFilters(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]);
|
||||
expect(atempoFilters(3)).toEqual(["atempo=2", "atempo=1.5"]);
|
||||
expect(atempoFilters(0.8)).toEqual(["atempo=0.8"]);
|
||||
});
|
||||
|
||||
it("changes volume, and mute drops the audio track", () => {
|
||||
expect(graphOf(plan([{ op: "volume", gain: 0.5 }]).args)).toContain(
|
||||
"[0:a:0]volume=0.5,aformat",
|
||||
);
|
||||
const muted = plan([{ op: "volume", gain: 2 }, { op: "mute" }]);
|
||||
expect(muted.args.filter((arg) => arg === "-map")).toHaveLength(1);
|
||||
expect(muted.args).not.toContain("aac");
|
||||
});
|
||||
|
||||
it("burns in subtitles styled for the current frame from a generated file name", () => {
|
||||
const result = plan(
|
||||
[
|
||||
{ op: "reframe", preset: "vertical", fit: "crop" },
|
||||
{ op: "subtitles", artifact_id: "subs" },
|
||||
],
|
||||
{ subtitleFiles: new Map([["subs", "subtitles-0.srt"]]) },
|
||||
);
|
||||
expect(graphOf(result.args)).toContain(
|
||||
`subtitles=filename=subtitles-0.srt:force_style='${subtitleStyle(1080, 1920)}'`,
|
||||
);
|
||||
expect(() =>
|
||||
plan([{ op: "subtitles", artifact_id: "subs" }], {
|
||||
subtitleFiles: new Map([["subs", "x.srt:force_style='a'"]]),
|
||||
}),
|
||||
).toThrow("Unsafe subtitle file name.");
|
||||
});
|
||||
|
||||
it("styles subtitles above the controls of tall frames", () => {
|
||||
expect(subtitleStyle(1080, 1920)).toContain("MarginV=384");
|
||||
expect(subtitleStyle(1080, 1920)).toContain("FontSize=65");
|
||||
expect(subtitleStyle(1920, 1080)).toContain("MarginV=65");
|
||||
expect(subtitleStyle(1920, 1080)).toContain("PlayResX=1920,PlayResY=1080");
|
||||
});
|
||||
|
||||
it("extracts audio without decoding video work", () => {
|
||||
const result = planVideoEdit({
|
||||
source,
|
||||
steps: [
|
||||
{ op: "trim", start: 0, end: 5 },
|
||||
{ op: "reframe", preset: "vertical", fit: "blur" },
|
||||
{ op: "subtitles", artifact_id: "subs" },
|
||||
{ op: "extract_audio" },
|
||||
],
|
||||
subtitleFiles: new Map([["subs", "subtitles-0.srt"]]),
|
||||
outputPath: "/out.m4a",
|
||||
});
|
||||
expect(graphOf(result.args)).toBe(
|
||||
"[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a1]",
|
||||
);
|
||||
expect(result.args).toEqual(expect.arrayContaining(["-map", "[a1]", "-b:a", "192k"]));
|
||||
expect(result.args).not.toContain("libx264");
|
||||
expect(result).toMatchObject({ kind: "audio", durationMs: 5000 });
|
||||
expect(() =>
|
||||
planVideoEdit({
|
||||
source: { ...source, probe: probe({ hasAudio: false }) },
|
||||
steps: [{ op: "extract_audio" }],
|
||||
outputPath: "/out.m4a",
|
||||
}),
|
||||
).toThrow("holiday.mp4 has no audio.");
|
||||
});
|
||||
|
||||
it("validates the poster time against the result", () => {
|
||||
expect(
|
||||
plan([
|
||||
{ op: "trim", start: 0, end: 4 },
|
||||
{ op: "poster", at: 3 },
|
||||
]).posterAtMs,
|
||||
).toBe(3000);
|
||||
expect(() =>
|
||||
plan([
|
||||
{ op: "trim", start: 0, end: 4 },
|
||||
{ op: "poster", at: 5 },
|
||||
]),
|
||||
).toThrow("poster at 5 s is past the end of the 4 s result.");
|
||||
});
|
||||
|
||||
it("keeps file paths out of the filter graph", () => {
|
||||
const hostile = { ...source, path: "/jobs/1/a'b[c]:d;e,f.mp4", name: "x'];[0:v]drawtext=" };
|
||||
const result = planVideoEdit({
|
||||
source: hostile,
|
||||
steps: [{ op: "reframe", preset: "square", fit: "blur" }],
|
||||
outputPath: "/jobs/1/out.mp4",
|
||||
});
|
||||
expect(graphOf(result.args)).not.toContain("a'b");
|
||||
expect(graphOf(result.args)).not.toContain("drawtext");
|
||||
expect(result.args).toContain("file:/jobs/1/a'b[c]:d;e,f.mp4");
|
||||
});
|
||||
|
||||
it("squares pixels and evens odd frame sizes", () => {
|
||||
const anamorphic = {
|
||||
...source,
|
||||
probe: probe({ width: 1440, height: 1080, sampleAspectRatio: 4 / 3 }),
|
||||
};
|
||||
const result = planVideoEdit({
|
||||
source: anamorphic,
|
||||
steps: [{ op: "mute" }],
|
||||
outputPath: "/o.mp4",
|
||||
});
|
||||
expect(graphOf(result.args)).toBe("[0:v:0]scale=1920:1080,setsar=1,format=yuv420p[v1]");
|
||||
expect(result).toMatchObject({ width: 1920, height: 1080 });
|
||||
const odd = { ...source, probe: probe({ width: 721, height: 405 }) };
|
||||
const evened = planVideoEdit({ source: odd, steps: [{ op: "mute" }], outputPath: "/o.mp4" });
|
||||
expect(graphOf(evened.args)).toBe("[0:v:0]crop=720:404:0:0,format=yuv420p[v1]");
|
||||
});
|
||||
});
|
||||
571
packages/adapters/src/media/video-edit-plan.ts
Normal file
571
packages/adapters/src/media/video-edit-plan.ts
Normal file
@ -0,0 +1,571 @@
|
||||
import type { EditVideoStep, MediaFit } from "@rakazo/contracts";
|
||||
import { MEDIA_PRESETS } from "@rakazo/contracts";
|
||||
import type { MediaProbe } from "./ffmpeg.js";
|
||||
import { ffmpegInputArgs, ffmpegSeconds } from "./ffmpeg.js";
|
||||
|
||||
/**
|
||||
* Turns an ordered list of edit steps into one ffmpeg invocation with a single filter_complex
|
||||
* pass. Pure: the caller probes the inputs and runs the arguments.
|
||||
*/
|
||||
|
||||
export interface PlanInput {
|
||||
path: string;
|
||||
mimeType: string;
|
||||
name: string;
|
||||
probe: MediaProbe;
|
||||
}
|
||||
|
||||
export interface VideoEditPlanRequest {
|
||||
source: PlanInput;
|
||||
steps: readonly EditVideoStep[];
|
||||
/** Videos named by concat steps, by artifact id. */
|
||||
clips?: ReadonlyMap<string, PlanInput>;
|
||||
/** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */
|
||||
subtitleFiles?: ReadonlyMap<string, string>;
|
||||
outputPath: string;
|
||||
}
|
||||
|
||||
export interface VideoEditPlan {
|
||||
args: string[];
|
||||
kind: "video" | "audio";
|
||||
durationMs: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
posterAtMs?: number;
|
||||
}
|
||||
|
||||
export const VIDEO_ENCODE_ARGS = [
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"21",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
];
|
||||
const AUDIO_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "160k"];
|
||||
const AUDIO_ONLY_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"];
|
||||
/** Location and other camera metadata stay private; chapters do not survive cuts anyway. */
|
||||
const STRIP_METADATA_ARGS = ["-map_metadata", "-1", "-map_chapters", "-1"];
|
||||
const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo";
|
||||
const SAFE_FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
||||
const MIN_SEGMENT_SECONDS = 0.1;
|
||||
const FALLBACK_FPS = 30;
|
||||
|
||||
interface Track {
|
||||
label: string;
|
||||
filters: string[];
|
||||
}
|
||||
|
||||
interface Timeline {
|
||||
/** Null when the result is audio only: video work is skipped. */
|
||||
video: Track | null;
|
||||
audio: Track | null;
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
seconds: number;
|
||||
/** How appended clips fill the frame: the last reframe's fit, otherwise bars. */
|
||||
fit: MediaFit;
|
||||
}
|
||||
|
||||
class FilterGraph {
|
||||
private readonly chains: string[] = [];
|
||||
private count = 0;
|
||||
|
||||
label(kind: "v" | "a"): string {
|
||||
this.count += 1;
|
||||
return `${kind}${this.count}`;
|
||||
}
|
||||
|
||||
add(inputs: readonly string[], filters: readonly string[], outputs: readonly string[]) {
|
||||
const pads = (labels: readonly string[]) => labels.map((label) => `[${label}]`).join("");
|
||||
this.chains.push(`${pads(inputs)}${filters.join(",")}${pads(outputs)}`);
|
||||
}
|
||||
|
||||
/** Ends a track's pending filters in a labelled pad other chains can consume. */
|
||||
flush(track: Track, kind: "v" | "a"): string {
|
||||
if (track.filters.length === 0) return track.label;
|
||||
const output = this.label(kind);
|
||||
this.add([track.label], track.filters, [output]);
|
||||
track.label = output;
|
||||
track.filters = [];
|
||||
return output;
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
return this.chains.join(";");
|
||||
}
|
||||
}
|
||||
|
||||
export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan {
|
||||
const { source } = request;
|
||||
const video = source.probe.video;
|
||||
if (!video) throw new Error(`${source.name} has no video.`);
|
||||
const sourceSeconds = source.probe.durationMs / 1000;
|
||||
const audioOnly = request.steps.at(-1)?.op === "extract_audio";
|
||||
const graph = new FilterGraph();
|
||||
const inputs: string[][] = [];
|
||||
const addInput = (input: PlanInput, seek?: { start: number; end: number }) => {
|
||||
inputs.push(
|
||||
ffmpegInputArgs(input.path, {
|
||||
mimeType: input.mimeType,
|
||||
...(seek ? { startSeconds: seek.start, durationSeconds: seek.end - seek.start } : {}),
|
||||
}),
|
||||
);
|
||||
return inputs.length - 1;
|
||||
};
|
||||
|
||||
let steps = request.steps;
|
||||
// A leading cut seeks the input instead of decoding everything before it.
|
||||
const first = steps[0];
|
||||
let segments: Segment[] | undefined;
|
||||
if (first?.op === "trim" || first?.op === "keep") {
|
||||
segments = normalizeSegments(
|
||||
first.op === "trim" ? [first] : first.segments,
|
||||
sourceSeconds,
|
||||
first.op,
|
||||
);
|
||||
steps = steps.slice(1);
|
||||
}
|
||||
const seek = segments && { start: segments[0]!.start, end: segments.at(-1)!.end };
|
||||
const index = addInput(source, seek);
|
||||
const size = displaySize(video);
|
||||
const timeline: Timeline = {
|
||||
video: audioOnly ? null : { label: `${index}:v:0`, filters: sourceFilters(video) },
|
||||
audio: source.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null,
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
fps: video.fps,
|
||||
seconds: seek ? seek.end - seek.start : sourceSeconds,
|
||||
fit: "pad",
|
||||
};
|
||||
if (segments && seek && segments.length > 1) {
|
||||
keepSegments(
|
||||
graph,
|
||||
timeline,
|
||||
segments.map((segment) => ({
|
||||
start: segment.start - seek.start,
|
||||
end: segment.end - seek.start,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
let posterAt: number | undefined;
|
||||
for (const step of steps) {
|
||||
switch (step.op) {
|
||||
case "trim":
|
||||
case "keep":
|
||||
keepSegments(
|
||||
graph,
|
||||
timeline,
|
||||
normalizeSegments(step.op === "trim" ? [step] : step.segments, timeline.seconds, step.op),
|
||||
);
|
||||
break;
|
||||
case "concat":
|
||||
appendClips(
|
||||
graph,
|
||||
timeline,
|
||||
step.artifact_ids.map((id) => {
|
||||
const clip = request.clips?.get(id);
|
||||
if (!clip) throw new Error(`Video ${id} is not available.`);
|
||||
return { clip, index: addInput(clip) };
|
||||
}),
|
||||
);
|
||||
break;
|
||||
case "reframe": {
|
||||
const preset = MEDIA_PRESETS[step.preset];
|
||||
if (timeline.video) {
|
||||
timeline.video = fitFrame(
|
||||
graph,
|
||||
timeline.video,
|
||||
timeline,
|
||||
{ width: preset.width, height: preset.height },
|
||||
step.fit,
|
||||
);
|
||||
}
|
||||
timeline.width = preset.width;
|
||||
timeline.height = preset.height;
|
||||
timeline.fit = step.fit;
|
||||
break;
|
||||
}
|
||||
case "speed":
|
||||
timeline.video?.filters.push(
|
||||
`setpts=PTS/${number(step.factor)}`,
|
||||
...(timeline.fps ? [`fps=${number(timeline.fps)}`] : []),
|
||||
);
|
||||
timeline.audio?.filters.push(...atempoFilters(step.factor));
|
||||
timeline.seconds /= step.factor;
|
||||
break;
|
||||
case "volume":
|
||||
timeline.audio?.filters.push(`volume=${number(step.gain)}`);
|
||||
break;
|
||||
case "mute":
|
||||
timeline.audio = null;
|
||||
break;
|
||||
case "subtitles": {
|
||||
const file = request.subtitleFiles?.get(step.artifact_id);
|
||||
if (!file) throw new Error(`Subtitles ${step.artifact_id} are not available.`);
|
||||
// Only generated names reach the filter graph, so nothing here needs escaping.
|
||||
if (!SAFE_FILE_NAME.test(file)) throw new Error("Unsafe subtitle file name.");
|
||||
timeline.video?.filters.push(
|
||||
`subtitles=filename=${file}:force_style='${subtitleStyle(timeline.width, timeline.height)}'`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
case "extract_audio":
|
||||
break;
|
||||
case "poster":
|
||||
posterAt = step.at;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (posterAt !== undefined && posterAt >= timeline.seconds) {
|
||||
throw new Error(
|
||||
`poster at ${seconds(posterAt)} s is past the end of the ${seconds(timeline.seconds)} s result.`,
|
||||
);
|
||||
}
|
||||
const durationMs = Math.round(timeline.seconds * 1000);
|
||||
const output = `file:${request.outputPath}`;
|
||||
if (audioOnly || !timeline.video) {
|
||||
if (!timeline.audio) throw new Error(`${source.name} has no audio.`);
|
||||
timeline.audio.filters.push(AUDIO_FORMAT);
|
||||
const audio = graph.flush(timeline.audio, "a");
|
||||
return {
|
||||
args: [
|
||||
...inputs.flat(),
|
||||
"-filter_complex",
|
||||
graph.toString(),
|
||||
"-map",
|
||||
`[${audio}]`,
|
||||
...AUDIO_ONLY_ENCODE_ARGS,
|
||||
...STRIP_METADATA_ARGS,
|
||||
"-f",
|
||||
"mp4",
|
||||
output,
|
||||
],
|
||||
kind: "audio",
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
timeline.video.filters.push("format=yuv420p");
|
||||
const videoOut = graph.flush(timeline.video, "v");
|
||||
let audioOut: string | undefined;
|
||||
if (timeline.audio) {
|
||||
timeline.audio.filters.push(AUDIO_FORMAT);
|
||||
audioOut = graph.flush(timeline.audio, "a");
|
||||
}
|
||||
return {
|
||||
args: [
|
||||
...inputs.flat(),
|
||||
"-filter_complex",
|
||||
graph.toString(),
|
||||
"-map",
|
||||
`[${videoOut}]`,
|
||||
...(audioOut ? ["-map", `[${audioOut}]`, ...AUDIO_ENCODE_ARGS] : []),
|
||||
...VIDEO_ENCODE_ARGS,
|
||||
...STRIP_METADATA_ARGS,
|
||||
"-f",
|
||||
"mp4",
|
||||
output,
|
||||
],
|
||||
kind: "video",
|
||||
durationMs,
|
||||
width: timeline.width,
|
||||
height: timeline.height,
|
||||
...(posterAt !== undefined ? { posterAtMs: Math.round(posterAt * 1000) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
interface Segment {
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
|
||||
/** Validates ranges against the timeline, clamps ends to it, sorts and merges overlaps. */
|
||||
function normalizeSegments(
|
||||
ranges: readonly Segment[],
|
||||
duration: number,
|
||||
op: "trim" | "keep",
|
||||
): Segment[] {
|
||||
const clamped = ranges.map((range) => {
|
||||
if (range.start >= duration) {
|
||||
throw new Error(
|
||||
`${op} ${seconds(range.start)}–${seconds(range.end)} s is outside the ${seconds(duration)} s video.`,
|
||||
);
|
||||
}
|
||||
const end = Math.min(range.end, duration);
|
||||
if (end - range.start < MIN_SEGMENT_SECONDS) {
|
||||
throw new Error(
|
||||
`${op} ${seconds(range.start)}–${seconds(end)} s is shorter than ${MIN_SEGMENT_SECONDS} s.`,
|
||||
);
|
||||
}
|
||||
return { start: range.start, end };
|
||||
});
|
||||
clamped.sort((a, b) => a.start - b.start);
|
||||
const merged: Segment[] = [];
|
||||
for (const segment of clamped) {
|
||||
const last = merged.at(-1);
|
||||
if (last && segment.start <= last.end) last.end = Math.max(last.end, segment.end);
|
||||
else merged.push({ ...segment });
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/** Keeps sorted, disjoint segments of the timeline: one trim, or split + trims + concat. */
|
||||
function keepSegments(graph: FilterGraph, timeline: Timeline, segments: readonly Segment[]) {
|
||||
const [only] = segments;
|
||||
if (segments.length === 1 && only) {
|
||||
const range = `start=${ffmpegSeconds(only.start)}:end=${ffmpegSeconds(only.end)}`;
|
||||
timeline.video?.filters.push(`trim=${range}`, "setpts=PTS-STARTPTS");
|
||||
timeline.audio?.filters.push(`atrim=${range}`, "asetpts=PTS-STARTPTS");
|
||||
timeline.seconds = only.end - only.start;
|
||||
return;
|
||||
}
|
||||
const count = segments.length;
|
||||
const branch = (track: Track | null, kind: "v" | "a"): string[] => {
|
||||
if (!track) return [];
|
||||
const input = graph.flush(track, kind);
|
||||
const splits = segments.map(() => graph.label(kind));
|
||||
graph.add([input], [`${kind === "v" ? "split" : "asplit"}=${count}`], splits);
|
||||
return segments.map((segment, index) => {
|
||||
const output = graph.label(kind);
|
||||
const range = `start=${ffmpegSeconds(segment.start)}:end=${ffmpegSeconds(segment.end)}`;
|
||||
graph.add(
|
||||
[splits[index]!],
|
||||
kind === "v"
|
||||
? [`trim=${range}`, "setpts=PTS-STARTPTS"]
|
||||
: [`atrim=${range}`, "asetpts=PTS-STARTPTS"],
|
||||
[output],
|
||||
);
|
||||
return output;
|
||||
});
|
||||
};
|
||||
const videoParts = branch(timeline.video, "v");
|
||||
const audioParts = branch(timeline.audio, "a");
|
||||
concatParts(graph, timeline, videoParts, audioParts, count);
|
||||
timeline.seconds = segments.reduce((total, segment) => total + segment.end - segment.start, 0);
|
||||
}
|
||||
|
||||
/** Appends clips normalized to the timeline: frame size, fit, fps and 48 kHz stereo audio. */
|
||||
function appendClips(
|
||||
graph: FilterGraph,
|
||||
timeline: Timeline,
|
||||
clips: readonly { clip: PlanInput; index: number }[],
|
||||
) {
|
||||
const fps = timeline.fps || FALLBACK_FPS;
|
||||
const needsAudio = timeline.audio !== null || clips.some(({ clip }) => clip.probe.hasAudio);
|
||||
const videoParts: string[] = [];
|
||||
const audioParts: string[] = [];
|
||||
if (timeline.video) {
|
||||
timeline.video.filters.push("setsar=1", "format=yuv420p");
|
||||
videoParts.push(graph.flush(timeline.video, "v"));
|
||||
}
|
||||
if (needsAudio) audioParts.push(audioPart(graph, timeline.audio, timeline.seconds));
|
||||
let seconds = timeline.seconds;
|
||||
for (const { clip, index } of clips) {
|
||||
const video = clip.probe.video;
|
||||
if (!video) throw new Error(`${clip.name} has no video.`);
|
||||
const clipSeconds = clip.probe.durationMs / 1000;
|
||||
if (clipSeconds <= 0) throw new Error(`Could not read the length of ${clip.name}.`);
|
||||
if (timeline.video) {
|
||||
const track = fitFrame(
|
||||
graph,
|
||||
{ label: `${index}:v:0`, filters: sourceFilters(video) },
|
||||
displaySize(video),
|
||||
timeline,
|
||||
timeline.fit,
|
||||
);
|
||||
track.filters.push(`fps=${number(fps)}`, "format=yuv420p");
|
||||
videoParts.push(graph.flush(track, "v"));
|
||||
}
|
||||
if (needsAudio) {
|
||||
const track = clip.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null;
|
||||
audioParts.push(audioPart(graph, track, clipSeconds));
|
||||
}
|
||||
seconds += clipSeconds;
|
||||
}
|
||||
concatParts(graph, timeline, videoParts, audioParts, clips.length + 1);
|
||||
timeline.seconds = seconds;
|
||||
}
|
||||
|
||||
/** A clip's audio in the common format, or silence as long as the clip when it has none. */
|
||||
function audioPart(graph: FilterGraph, track: Track | null, seconds: number): string {
|
||||
if (track) {
|
||||
track.filters.push(AUDIO_FORMAT);
|
||||
return graph.flush(track, "a");
|
||||
}
|
||||
const output = graph.label("a");
|
||||
graph.add(
|
||||
[],
|
||||
[
|
||||
"anullsrc=channel_layout=stereo:sample_rate=48000",
|
||||
`atrim=end=${ffmpegSeconds(seconds)}`,
|
||||
AUDIO_FORMAT,
|
||||
],
|
||||
[output],
|
||||
);
|
||||
return output;
|
||||
}
|
||||
|
||||
function concatParts(
|
||||
graph: FilterGraph,
|
||||
timeline: Timeline,
|
||||
videoParts: readonly string[],
|
||||
audioParts: readonly string[],
|
||||
count: number,
|
||||
) {
|
||||
const inputs = Array.from({ length: count }, (_, index) => [
|
||||
...(videoParts[index] ? [videoParts[index]] : []),
|
||||
...(audioParts[index] ? [audioParts[index]] : []),
|
||||
]).flat();
|
||||
const video = videoParts.length > 0 ? graph.label("v") : undefined;
|
||||
const audio = audioParts.length > 0 ? graph.label("a") : undefined;
|
||||
graph.add(
|
||||
inputs,
|
||||
[`concat=n=${count}:v=${video ? 1 : 0}:a=${audio ? 1 : 0}`],
|
||||
[...(video ? [video] : []), ...(audio ? [audio] : [])],
|
||||
);
|
||||
timeline.video = video ? { label: video, filters: [] } : null;
|
||||
timeline.audio = audio ? { label: audio, filters: [] } : null;
|
||||
}
|
||||
|
||||
interface Size {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/** Scales a frame of size `from` into `to`: center crop, bars, or a blurred copy behind it. */
|
||||
function fitFrame(graph: FilterGraph, track: Track, from: Size, to: Size, fit: MediaFit): Track {
|
||||
const { width, height } = to;
|
||||
if (Math.abs(from.width / from.height - width / height) < 0.01) {
|
||||
track.filters.push(`scale=${width}:${height}`, "setsar=1");
|
||||
return track;
|
||||
}
|
||||
const contain = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`;
|
||||
if (fit === "crop") {
|
||||
track.filters.push(
|
||||
`scale=${width}:${height}:force_original_aspect_ratio=increase`,
|
||||
`crop=${width}:${height}`,
|
||||
"setsar=1",
|
||||
);
|
||||
return track;
|
||||
}
|
||||
if (fit === "pad") {
|
||||
track.filters.push(
|
||||
contain,
|
||||
`pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`,
|
||||
"setsar=1",
|
||||
);
|
||||
return track;
|
||||
}
|
||||
// Blur a small copy and scale it up: as smooth as a large blur at a fraction of the cost.
|
||||
const input = graph.flush(track, "v");
|
||||
const [background, foreground, blurred, contained, output] = [
|
||||
graph.label("v"),
|
||||
graph.label("v"),
|
||||
graph.label("v"),
|
||||
graph.label("v"),
|
||||
graph.label("v"),
|
||||
];
|
||||
const smallWidth = even(width / 4);
|
||||
const smallHeight = even(height / 4);
|
||||
const radius = Math.max(2, Math.floor(Math.min(smallWidth, smallHeight) / 10));
|
||||
graph.add([input], ["split=2"], [background, foreground]);
|
||||
graph.add(
|
||||
[background],
|
||||
[
|
||||
`scale=${smallWidth}:${smallHeight}:force_original_aspect_ratio=increase`,
|
||||
`crop=${smallWidth}:${smallHeight}`,
|
||||
`boxblur=${radius}:2`,
|
||||
`scale=${width}:${height}`,
|
||||
"setsar=1",
|
||||
],
|
||||
[blurred],
|
||||
);
|
||||
graph.add([foreground], [contain, "setsar=1"], [contained]);
|
||||
graph.add([blurred, contained], ["overlay=(W-w)/2:(H-h)/2"], [output]);
|
||||
return { label: output, filters: [] };
|
||||
}
|
||||
|
||||
/** Square pixels and even dimensions, which yuv420p H.264 requires. */
|
||||
function sourceFilters(video: NonNullable<MediaProbe["video"]>): string[] {
|
||||
const size = displaySize(video);
|
||||
if (Math.abs(video.sampleAspectRatio - 1) > 0.01) {
|
||||
return [`scale=${size.width}:${size.height}`, "setsar=1"];
|
||||
}
|
||||
if (size.width !== video.width || size.height !== video.height) {
|
||||
return [`crop=${size.width}:${size.height}:0:0`];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function displaySize(video: NonNullable<MediaProbe["video"]>): Size {
|
||||
if (Math.abs(video.sampleAspectRatio - 1) > 0.01) {
|
||||
return {
|
||||
width: even(Math.round(video.width * video.sampleAspectRatio)),
|
||||
height: even(video.height),
|
||||
};
|
||||
}
|
||||
return { width: even(video.width), height: even(video.height) };
|
||||
}
|
||||
|
||||
/** atempo takes 0.5–2 per instance on older ffmpeg builds, so larger changes chain it. */
|
||||
export function atempoFilters(factor: number): string[] {
|
||||
const filters: string[] = [];
|
||||
let rest = factor;
|
||||
while (rest > 2) {
|
||||
filters.push("atempo=2");
|
||||
rest /= 2;
|
||||
}
|
||||
while (rest < 0.5) {
|
||||
filters.push("atempo=0.5");
|
||||
rest /= 0.5;
|
||||
}
|
||||
filters.push(`atempo=${number(rest)}`);
|
||||
return filters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Readable burned-in subtitles: bold white text with a dark outline, sized to the frame, and
|
||||
* lifted above the bottom fifth of tall frames where 9:16 apps place their own controls.
|
||||
*/
|
||||
export function subtitleStyle(width: number, height: number): string {
|
||||
const tall = height >= width * 1.5;
|
||||
const fontSize = Math.round(Math.min(width, height) * 0.06);
|
||||
const marginX = Math.round(width * (tall ? 0.1 : 0.05));
|
||||
const marginY = Math.round(height * (tall ? 0.2 : 0.06));
|
||||
return [
|
||||
`PlayResX=${width}`,
|
||||
`PlayResY=${height}`,
|
||||
"FontName=DejaVu Sans",
|
||||
`FontSize=${fontSize}`,
|
||||
"Bold=1",
|
||||
"PrimaryColour=&H00FFFFFF",
|
||||
"OutlineColour=&H00000000",
|
||||
"BackColour=&H80000000",
|
||||
"BorderStyle=1",
|
||||
`Outline=${Math.max(2, Math.round(fontSize / 16))}`,
|
||||
"Shadow=1",
|
||||
"Alignment=2",
|
||||
`MarginL=${marginX}`,
|
||||
`MarginR=${marginX}`,
|
||||
`MarginV=${marginY}`,
|
||||
].join(",");
|
||||
}
|
||||
|
||||
function even(value: number): number {
|
||||
return Math.max(2, 2 * Math.floor(value / 2));
|
||||
}
|
||||
|
||||
function number(value: number): string {
|
||||
return String(Number(value.toFixed(6)));
|
||||
}
|
||||
|
||||
function seconds(value: number): string {
|
||||
return String(Number(value.toFixed(2)));
|
||||
}
|
||||
331
packages/adapters/src/media/video-operations.ts
Normal file
331
packages/adapters/src/media/video-operations.ts
Normal file
@ -0,0 +1,331 @@
|
||||
import { copyFile, mkdtemp, open, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type {
|
||||
MediaJobContext,
|
||||
MediaJobInputFile,
|
||||
MediaOperationHandler,
|
||||
MediaOperationResult,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts";
|
||||
import {
|
||||
ARTIFACT_MAX_BYTES_BY_KIND,
|
||||
EditVideoInputSchema,
|
||||
ExportVideoInputSchema,
|
||||
MEDIA_PRESETS,
|
||||
} from "@rakazo/contracts";
|
||||
import type { FfmpegInfo } from "./ffmpeg.js";
|
||||
import {
|
||||
detectFfmpeg,
|
||||
outputTooLargeMessage,
|
||||
probeMedia,
|
||||
renderPoster,
|
||||
runFfmpeg,
|
||||
} from "./ffmpeg.js";
|
||||
import type { PlanInput, VideoEditPlan } from "./video-edit-plan.js";
|
||||
import { planVideoEdit } from "./video-edit-plan.js";
|
||||
|
||||
export interface VideoOperationsDeps {
|
||||
/** Finds ffmpeg; defaults to PATH lookup. */
|
||||
detect?: () => Promise<FfmpegInfo | null>;
|
||||
}
|
||||
|
||||
const PROGRESS_INTERVAL_MS = 1_000;
|
||||
/** Share of each output's progress spent encoding; the rest is probing and the poster. */
|
||||
const ENCODE_SHARE = 0.95;
|
||||
const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
/** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */
|
||||
export function createVideoOperations(deps: VideoOperationsDeps = {}) {
|
||||
const detect = deps.detect ?? detectFfmpeg;
|
||||
const available = async () => (await detect()) !== null;
|
||||
const tools = async () => {
|
||||
const info = await detect();
|
||||
if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server.");
|
||||
return info;
|
||||
};
|
||||
const editVideo: MediaOperationHandler = {
|
||||
operation: "edit_video",
|
||||
run: async (context) => runEditVideo(context, await tools()),
|
||||
};
|
||||
const exportVideo: MediaOperationHandler = {
|
||||
operation: "export_video",
|
||||
run: async (context) => runExportVideo(context, await tools()),
|
||||
};
|
||||
return [
|
||||
{
|
||||
operation: "edit_video" as const,
|
||||
inputSchema: EditVideoInputSchema,
|
||||
title: (input: EditVideoInput) => input.title,
|
||||
available,
|
||||
handler: editVideo,
|
||||
},
|
||||
{
|
||||
operation: "export_video" as const,
|
||||
inputSchema: ExportVideoInputSchema,
|
||||
title: (input: ExportVideoInput) =>
|
||||
input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "),
|
||||
available,
|
||||
handler: exportVideo,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
async function runEditVideo(
|
||||
context: MediaJobContext,
|
||||
tools: FfmpegInfo,
|
||||
): Promise<MediaOperationResult> {
|
||||
const input = EditVideoInputSchema.parse(context.input);
|
||||
const workDir = await mkdtemp(join(context.tempDir, "edit-"));
|
||||
const progress = createProgressReporter(context);
|
||||
const source = await loadVideo(context, tools, input.artifact_id);
|
||||
const name = cleanFileName(input.title, cleanFileName(withoutExtension(source.name), "video"));
|
||||
|
||||
if (input.steps.every((step) => step.op === "poster")) {
|
||||
const at = input.steps[0]?.op === "poster" ? input.steps[0].at : 0;
|
||||
if (at * 1000 >= source.probe.durationMs) {
|
||||
throw new Error(
|
||||
`poster at ${at} s is past the end of the ${source.probe.durationMs / 1000} s video.`,
|
||||
);
|
||||
}
|
||||
const posterPath = join(workDir, "poster.jpg");
|
||||
await renderPoster(source.path, posterPath, {
|
||||
signal: context.signal,
|
||||
mimeType: source.mimeType,
|
||||
atMs: at * 1000,
|
||||
});
|
||||
const output = await context.addOutput({
|
||||
path: posterPath,
|
||||
name: `${name}.jpg`,
|
||||
mimeType: "image/jpeg",
|
||||
width: source.probe.video?.width,
|
||||
height: source.probe.video?.height,
|
||||
parentArtifactId: source.artifactId,
|
||||
});
|
||||
await progress.done();
|
||||
return { summary: `Saved the frame at ${at} s of ${source.name} as ${describe(output)}.` };
|
||||
}
|
||||
|
||||
const clips = new Map<string, PlanInput>();
|
||||
const subtitleFiles = new Map<string, string>();
|
||||
for (const step of input.steps) {
|
||||
if (step.op === "concat") {
|
||||
for (const id of step.artifact_ids) {
|
||||
if (!clips.has(id)) clips.set(id, await loadVideo(context, tools, id));
|
||||
}
|
||||
} else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) {
|
||||
subtitleFiles.set(
|
||||
step.artifact_id,
|
||||
await prepareSubtitles(context, step.artifact_id, workDir, subtitleFiles.size),
|
||||
);
|
||||
}
|
||||
}
|
||||
const audioOnly = input.steps.at(-1)?.op === "extract_audio";
|
||||
const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4");
|
||||
const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath });
|
||||
const output = await encode(context, tools, plan, outputPath, workDir, progress.range(0, 1), {
|
||||
name: `${name}${audioOnly ? ".m4a" : ".mp4"}`,
|
||||
parentArtifactId: source.artifactId,
|
||||
});
|
||||
await progress.done();
|
||||
return {
|
||||
summary: `Edited ${source.name} into ${describe(output)}.`,
|
||||
};
|
||||
}
|
||||
|
||||
async function runExportVideo(
|
||||
context: MediaJobContext,
|
||||
tools: FfmpegInfo,
|
||||
): Promise<MediaOperationResult> {
|
||||
const input = ExportVideoInputSchema.parse(context.input);
|
||||
const presets = [...new Set(input.presets)];
|
||||
const workDir = await mkdtemp(join(context.tempDir, "export-"));
|
||||
const progress = createProgressReporter(context);
|
||||
const source = await loadVideo(context, tools, input.artifact_id);
|
||||
const base = cleanFileName(withoutExtension(source.name), "video");
|
||||
const outputs: MediaOutput[] = [];
|
||||
for (const [index, preset] of presets.entries()) {
|
||||
const outputPath = join(workDir, `${preset}.mp4`);
|
||||
const plan = planVideoEdit({
|
||||
source,
|
||||
steps: [{ op: "reframe", preset, fit: input.fit }],
|
||||
outputPath,
|
||||
});
|
||||
const share = 1 / presets.length;
|
||||
outputs.push(
|
||||
await encode(
|
||||
context,
|
||||
tools,
|
||||
plan,
|
||||
outputPath,
|
||||
workDir,
|
||||
progress.range(index * share, (index + 1) * share),
|
||||
{
|
||||
name: `${base}-${MEDIA_PRESETS[preset].aspect.replace(":", "x")}.mp4`,
|
||||
parentArtifactId: source.artifactId,
|
||||
preset,
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
await progress.done();
|
||||
return { summary: `Exported ${source.name} as ${outputs.map(describe).join(", ")}.` };
|
||||
}
|
||||
|
||||
/** Runs a plan, checks the result, renders its poster and stores it as an output. */
|
||||
async function encode(
|
||||
context: MediaJobContext,
|
||||
tools: FfmpegInfo,
|
||||
plan: VideoEditPlan,
|
||||
outputPath: string,
|
||||
workDir: string,
|
||||
report: (fraction: number) => void,
|
||||
file: Pick<Parameters<MediaJobContext["addOutput"]>[0], "name" | "parentArtifactId" | "preset">,
|
||||
): Promise<MediaOutput> {
|
||||
const maxBytes = ARTIFACT_MAX_BYTES_BY_KIND[plan.kind];
|
||||
await runFfmpeg(plan.args, {
|
||||
signal: context.signal,
|
||||
cwd: workDir,
|
||||
durationMs: plan.durationMs,
|
||||
onProgress: (fraction) => report(fraction * ENCODE_SHARE),
|
||||
maxOutputBytes: maxBytes,
|
||||
outputs: [outputPath],
|
||||
ffmpegPath: tools.ffmpegPath,
|
||||
});
|
||||
if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes));
|
||||
const probe = await probeMedia(outputPath, {
|
||||
signal: context.signal,
|
||||
mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4",
|
||||
ffprobePath: tools.ffprobePath,
|
||||
});
|
||||
if (plan.kind === "audio") {
|
||||
const output = await context.addOutput({
|
||||
...file,
|
||||
path: outputPath,
|
||||
mimeType: "audio/mp4",
|
||||
durationMs: probe.durationMs,
|
||||
});
|
||||
report(1);
|
||||
return output;
|
||||
}
|
||||
const posterPath = outputPath.replace(/\.mp4$/, ".jpg");
|
||||
await renderPoster(outputPath, posterPath, {
|
||||
signal: context.signal,
|
||||
mimeType: "video/mp4",
|
||||
atMs: plan.posterAtMs ?? Math.min(1_000, probe.durationMs / 4),
|
||||
});
|
||||
const output = await context.addOutput({
|
||||
...file,
|
||||
path: outputPath,
|
||||
mimeType: "video/mp4",
|
||||
width: probe.video?.width ?? plan.width,
|
||||
height: probe.video?.height ?? plan.height,
|
||||
durationMs: probe.durationMs,
|
||||
posterPath,
|
||||
});
|
||||
report(1);
|
||||
return output;
|
||||
}
|
||||
|
||||
async function loadVideo(
|
||||
context: MediaJobContext,
|
||||
tools: FfmpegInfo,
|
||||
artifactId: string,
|
||||
): Promise<PlanInput & MediaJobInputFile> {
|
||||
const file = await context.fetchArtifact(artifactId);
|
||||
if (file.kind !== "video") throw new Error(`${file.name} is not a video.`);
|
||||
const probe = await probeMedia(file.path, {
|
||||
signal: context.signal,
|
||||
mimeType: file.mimeType,
|
||||
ffprobePath: tools.ffprobePath,
|
||||
});
|
||||
if (!probe.video) throw new Error(`${file.name} has no video.`);
|
||||
if (probe.durationMs <= 0) throw new Error(`Could not read the length of ${file.name}.`);
|
||||
return { ...file, probe };
|
||||
}
|
||||
|
||||
/** Copies an SRT/VTT artifact to a generated name, so no user text reaches the filter graph. */
|
||||
async function prepareSubtitles(
|
||||
context: MediaJobContext,
|
||||
artifactId: string,
|
||||
workDir: string,
|
||||
index: number,
|
||||
): Promise<string> {
|
||||
const file = await context.fetchArtifact(artifactId);
|
||||
const format = file.size <= SUBTITLE_MAX_BYTES ? subtitleFormat(await readHead(file.path)) : null;
|
||||
if (!format) throw new Error(`${file.name} is not an SRT or VTT file.`);
|
||||
const name = `subtitles-${index}.${format}`;
|
||||
await copyFile(file.path, join(workDir, name));
|
||||
return name;
|
||||
}
|
||||
|
||||
async function readHead(path: string): Promise<string> {
|
||||
const handle = await open(path, "r");
|
||||
try {
|
||||
const buffer = Buffer.alloc(1024);
|
||||
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
|
||||
return buffer.subarray(0, bytesRead).toString("utf8");
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognizes WebVTT and SubRip by content. Anything else is refused, so ffmpeg never probes
|
||||
* an arbitrary file (a playlist, say) handed in as subtitles.
|
||||
*/
|
||||
export function subtitleFormat(head: string): "srt" | "vtt" | null {
|
||||
const text = head.replace(/^/, "").trimStart();
|
||||
if (/^WEBVTT(?:[ \t\r\n]|$)/.test(text)) return "vtt";
|
||||
if (/^(?:\d+[ \t]*\r?\n)?\d{1,2}:\d{2}:\d{2}[,.]\d{1,3}[ \t]*-->/.test(text)) return "srt";
|
||||
return null;
|
||||
}
|
||||
|
||||
/** A name for produced files: no path separators or control characters, at most 80 chars. */
|
||||
export function cleanFileName(text: string, fallback: string): string {
|
||||
const name = text
|
||||
.replaceAll(/[\p{Cc}\\/:*?"<>|]+/gu, " ")
|
||||
.replaceAll(/\s+/g, " ")
|
||||
.replace(/^[\s.]+/, "")
|
||||
.slice(0, 80)
|
||||
.trim();
|
||||
return name || fallback;
|
||||
}
|
||||
|
||||
function withoutExtension(name: string): string {
|
||||
return name.replace(/\.[A-Za-z0-9]{1,5}$/, "");
|
||||
}
|
||||
|
||||
function describe(output: MediaOutput): string {
|
||||
const size = output.width && output.height ? ` ${output.width}×${output.height}` : "";
|
||||
const length =
|
||||
output.durationMs !== undefined ? ` ${Number((output.durationMs / 1000).toFixed(1))} s` : "";
|
||||
return `${output.name}${size}${length} (artifact ${output.artifactId})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps ffmpeg progress onto the card: monotonic, at most once a second, written in order and
|
||||
* never failing the job when a write fails.
|
||||
*/
|
||||
function createProgressReporter(context: MediaJobContext) {
|
||||
let reported = 0;
|
||||
let reportedAt = 0;
|
||||
let writes = Promise.resolve();
|
||||
const write = (fraction: number) => {
|
||||
reported = fraction;
|
||||
reportedAt = Date.now();
|
||||
writes = writes.then(() => context.progress(fraction)).catch(() => undefined);
|
||||
};
|
||||
return {
|
||||
range: (from: number, to: number) => (fraction: number) => {
|
||||
const value = Math.min(1, from + (to - from) * Math.min(1, Math.max(0, fraction)));
|
||||
const boundary = fraction >= 1;
|
||||
if (value <= reported) return;
|
||||
if (!boundary && Date.now() - reportedAt < PROGRESS_INTERVAL_MS) return;
|
||||
write(value);
|
||||
},
|
||||
done: async () => {
|
||||
if (reported < 1) write(1);
|
||||
await writes;
|
||||
},
|
||||
};
|
||||
}
|
||||
@ -10,6 +10,7 @@ export * from "./integration-settings.js";
|
||||
export * from "./local-settings.js";
|
||||
export * from "./mcp.js";
|
||||
export * from "./media.js";
|
||||
export * from "./media-video.js";
|
||||
export * from "./model-tiers.js";
|
||||
export * from "./openai-compatible-ui.js";
|
||||
export * from "./project-goals.js";
|
||||
|
||||
80
packages/contracts/src/media-video.ts
Normal file
80
packages/contracts/src/media-video.ts
Normal file
@ -0,0 +1,80 @@
|
||||
import * as z from "zod";
|
||||
import { Id } from "./ids.js";
|
||||
import { MediaFitSchema, MediaPresetIdSchema } from "./media.js";
|
||||
|
||||
/**
|
||||
* Inputs of the `edit_video` and `export_video` media operations, which are also the arguments
|
||||
* of the tools with those names. Times are seconds on the timeline left by the previous steps.
|
||||
*/
|
||||
|
||||
const Seconds = z.number().min(0).max(86_400);
|
||||
|
||||
const TimeRangeSchema = z
|
||||
.object({ start: Seconds, end: Seconds })
|
||||
.refine((range) => range.end > range.start, "end must be after start");
|
||||
|
||||
/** Written by the bot in the user's language; shown on the media card and used as file name. */
|
||||
const MediaTitleSchema = z.string().trim().min(1).max(120);
|
||||
|
||||
export const EditVideoStepSchema = z.discriminatedUnion("op", [
|
||||
z
|
||||
.object({ op: z.literal("trim"), start: Seconds, end: Seconds })
|
||||
.refine((step) => step.end > step.start, "end must be after start"),
|
||||
/** Keeps these parts and cuts everything between them. */
|
||||
z.object({ op: z.literal("keep"), segments: z.array(TimeRangeSchema).min(1).max(50) }),
|
||||
/** Appends these videos, fitted to the current frame. */
|
||||
z.object({ op: z.literal("concat"), artifact_ids: z.array(Id).min(1).max(20) }),
|
||||
z.object({
|
||||
op: z.literal("reframe"),
|
||||
preset: MediaPresetIdSchema,
|
||||
fit: MediaFitSchema.default("crop"),
|
||||
}),
|
||||
z.object({ op: z.literal("speed"), factor: z.number().min(0.25).max(4) }),
|
||||
/** Linear gain: 0.5 halves the loudness, 2 doubles it. */
|
||||
z.object({ op: z.literal("volume"), gain: z.number().min(0).max(8) }),
|
||||
z.object({ op: z.literal("mute") }),
|
||||
/** Burns in an SRT or VTT artifact. */
|
||||
z.object({ op: z.literal("subtitles"), artifact_id: Id }),
|
||||
/** Last step only: the result is the audio as M4A. */
|
||||
z.object({ op: z.literal("extract_audio") }),
|
||||
/** Cover frame at this second of the result; as the only kind of step, the result is that image. */
|
||||
z.object({ op: z.literal("poster"), at: Seconds }),
|
||||
]);
|
||||
export type EditVideoStep = z.infer<typeof EditVideoStepSchema>;
|
||||
|
||||
export const EditVideoInputSchema = z
|
||||
.object({
|
||||
artifact_id: Id,
|
||||
steps: z.array(EditVideoStepSchema).min(1).max(20),
|
||||
title: MediaTitleSchema,
|
||||
})
|
||||
.superRefine((input, context) => {
|
||||
const ops = input.steps.map((step) => step.op);
|
||||
const extract = ops.indexOf("extract_audio");
|
||||
if (extract >= 0 && extract !== ops.length - 1) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["steps", extract],
|
||||
message: "extract_audio must be the last step",
|
||||
});
|
||||
}
|
||||
if (extract >= 0 && ops.includes("poster")) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["steps"],
|
||||
message: "poster cannot be combined with extract_audio",
|
||||
});
|
||||
}
|
||||
if (ops.filter((op) => op === "poster").length > 1) {
|
||||
context.addIssue({ code: "custom", path: ["steps"], message: "use at most one poster step" });
|
||||
}
|
||||
});
|
||||
export type EditVideoInput = z.infer<typeof EditVideoInputSchema>;
|
||||
|
||||
export const ExportVideoInputSchema = z.object({
|
||||
artifact_id: Id,
|
||||
presets: z.array(MediaPresetIdSchema).min(1).max(6),
|
||||
fit: MediaFitSchema.default("crop"),
|
||||
title: MediaTitleSchema.optional(),
|
||||
});
|
||||
export type ExportVideoInput = z.infer<typeof ExportVideoInputSchema>;
|
||||
Loading…
x
Reference in New Issue
Block a user