feat(media): edit and export video with ffmpeg
Bots can now cut, reframe and export videos for every common format without a hosted service: ffmpeg and ffprobe from PATH do the work on the server. - ffmpeg wrapper: PATH detection, ffprobe JSON probing (rotation, pixel aspect, frame rate, audio, packet scan when the container has no duration), runs with -progress parsing, abort and stall kills, an output size cap, partial-output cleanup and short errors without temp paths. Inputs go through the file protocol with the demuxer forced from the MIME type, so a crafted upload cannot be probed as a playlist that reads other files or URLs. extractAudio (16 kHz mono WAV) and renderPoster for transcription and media cards. - A pure planner turns edit steps (trim, keep, concat, reframe, speed, volume, mute, subtitles, extract_audio, poster) into one filter_complex pass: leading cuts seek the input, appended clips are normalized to the frame, fps and 48 kHz stereo (silence where a clip has none), reframes crop, pad or blur to the presets, speed chains atempo, subtitles burn in from generated file names with a readable style that stays clear of the controls on 9:16 frames. H.264/AAC MP4 with yuv420p and faststart; location metadata is stripped. - edit_video and export_video operations and tool definitions; the inputs are shared contracts so the media card can start exports. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
parent
7ff93f08bb
commit
485090c391
@ -2,6 +2,8 @@ import type { ConnectorTool } from "@rakazo/adapter-kit";
|
|||||||
import {
|
import {
|
||||||
BotSecretDestination,
|
BotSecretDestination,
|
||||||
BotSecretName,
|
BotSecretName,
|
||||||
|
EditVideoInputSchema,
|
||||||
|
ExportVideoInputSchema,
|
||||||
MODEL_TIERS,
|
MODEL_TIERS,
|
||||||
SecretAskPurpose,
|
SecretAskPurpose,
|
||||||
SecretHttpRequest,
|
SecretHttpRequest,
|
||||||
@ -12,6 +14,25 @@ import { z } from "zod";
|
|||||||
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
|
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
|
||||||
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
|
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
|
||||||
|
|
||||||
|
/** A media tool's arguments: its operation's input plus whether to wake the bot when it ends. */
|
||||||
|
function mediaToolSchema(input: z.ZodType): Record<string, unknown> {
|
||||||
|
const schema = z.toJSONSchema(input, { io: "input" });
|
||||||
|
return {
|
||||||
|
...schema,
|
||||||
|
properties: {
|
||||||
|
...schema.properties,
|
||||||
|
continue_after: {
|
||||||
|
type: "boolean",
|
||||||
|
description:
|
||||||
|
"true to be woken with the result when the job ends, to continue a multi-step task. Default false: the card alone shows the result.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const VIDEO_PRESETS_AND_FITS =
|
||||||
|
"Presets: vertical 9:16, landscape 16:9, square 1:1, portrait 4:5, classic 4:3, cinema 21:9. fit: crop (default) fills the frame and cuts the edges, pad adds bars, blur fills the bars with a blurred copy.";
|
||||||
|
|
||||||
export const DELEGATION_TOOL_NAMES = new Set([
|
export const DELEGATION_TOOL_NAMES = new Set([
|
||||||
"run_subagent",
|
"run_subagent",
|
||||||
"spawn_bot",
|
"spawn_bot",
|
||||||
@ -326,6 +347,16 @@ export const builtinAgentTools: ConnectorTool[] = [
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "edit_video",
|
||||||
|
description: `Edit a video artifact in a background job; a media card shows progress and the result, a new version of the source. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4 with a poster. title: a few words in the user's language for the card and file name.`,
|
||||||
|
inputSchema: mediaToolSchema(EditVideoInputSchema),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "export_video",
|
||||||
|
description: `Export a video artifact to one or more presets in a background job: one H.264/AAC MP4 with a poster per preset, each a new version of the source, shown on a media card. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`,
|
||||||
|
inputSchema: mediaToolSchema(ExportVideoInputSchema),
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "add_mcp_server",
|
name: "add_mcp_server",
|
||||||
description:
|
description:
|
||||||
|
|||||||
@ -76,6 +76,8 @@ export * from "./mcp-emulator.js";
|
|||||||
export * from "./mcp-oauth.js";
|
export * from "./mcp-oauth.js";
|
||||||
export * from "./mcp-server-tool.js";
|
export * from "./mcp-server-tool.js";
|
||||||
export * from "./mcp-transport.js";
|
export * from "./mcp-transport.js";
|
||||||
|
export * from "./media/ffmpeg.js";
|
||||||
|
export * from "./media/video-operations.js";
|
||||||
export * from "./memory-provider-factory.js";
|
export * from "./memory-provider-factory.js";
|
||||||
export * from "./messaging-context.js";
|
export * from "./messaging-context.js";
|
||||||
export * from "./messaging-delivery.js";
|
export * from "./messaging-delivery.js";
|
||||||
|
|||||||
256
packages/adapters/src/media/ffmpeg.test.ts
Normal file
256
packages/adapters/src/media/ffmpeg.test.ts
Normal file
@ -0,0 +1,256 @@
|
|||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import { rm, writeFile } from "node:fs/promises";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import type { FfmpegProgress } from "./ffmpeg.js";
|
||||||
|
import {
|
||||||
|
createFfmpegProgressParser,
|
||||||
|
detectFfmpeg,
|
||||||
|
extractAudio,
|
||||||
|
FfmpegError,
|
||||||
|
ffmpegInputArgs,
|
||||||
|
ffmpegSeconds,
|
||||||
|
packetDurationMs,
|
||||||
|
parseProbeOutput,
|
||||||
|
probeMedia,
|
||||||
|
renderPoster,
|
||||||
|
runFfmpeg,
|
||||||
|
} from "./ffmpeg.js";
|
||||||
|
import type { TestClips } from "./media-test-support.js";
|
||||||
|
import { createTestClips } from "./media-test-support.js";
|
||||||
|
|
||||||
|
const ffmpeg = await detectFfmpeg();
|
||||||
|
|
||||||
|
describe("ffmpeg progress", () => {
|
||||||
|
it("reports each block, tolerates split chunks and N/A times, and marks the end", () => {
|
||||||
|
const updates: FfmpegProgress[] = [];
|
||||||
|
const feed = createFfmpegProgressParser((update) => updates.push(update));
|
||||||
|
feed("frame=0\nout_time_us=N/A\ntotal_size=48\nprogress=cont");
|
||||||
|
feed("inue\nframe=75\nout_time_us=2995");
|
||||||
|
feed("374\nout_time_ms=2995374\ntotal_size=42300\nprogress=end\n");
|
||||||
|
expect(updates).toEqual([
|
||||||
|
{ totalSize: 48, done: false },
|
||||||
|
{ outTimeUs: 2_995_374, totalSize: 42_300, done: true },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("probe parsing", () => {
|
||||||
|
const stream = (fields: Record<string, unknown>) => ({
|
||||||
|
codec_type: "video",
|
||||||
|
codec_name: "h264",
|
||||||
|
...fields,
|
||||||
|
});
|
||||||
|
|
||||||
|
it("swaps the size of rotated video and reads frame rate and audio", () => {
|
||||||
|
const probe = parseProbeOutput(
|
||||||
|
JSON.stringify({
|
||||||
|
format: { duration: "12.345", format_name: "mov,mp4,m4a,3gp,3g2,mj2" },
|
||||||
|
streams: [
|
||||||
|
stream({
|
||||||
|
width: 1920,
|
||||||
|
height: 1080,
|
||||||
|
avg_frame_rate: "30000/1001",
|
||||||
|
sample_aspect_ratio: "1:1",
|
||||||
|
side_data_list: [{ side_data_type: "Display Matrix", rotation: -90 }],
|
||||||
|
}),
|
||||||
|
{ codec_type: "audio", codec_name: "aac", sample_rate: "48000", channels: 2 },
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(probe).toEqual({
|
||||||
|
durationMs: 12_345,
|
||||||
|
formatName: "mov,mp4,m4a,3gp,3g2,mj2",
|
||||||
|
video: {
|
||||||
|
codec: "h264",
|
||||||
|
width: 1080,
|
||||||
|
height: 1920,
|
||||||
|
rotation: 270,
|
||||||
|
fps: 30000 / 1001,
|
||||||
|
sampleAspectRatio: 1,
|
||||||
|
},
|
||||||
|
audio: { codec: "aac", sampleRate: 48_000, channels: 2 },
|
||||||
|
hasAudio: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads legacy rotate tags, skips cover art and falls back to stream durations", () => {
|
||||||
|
const probe = parseProbeOutput(
|
||||||
|
JSON.stringify({
|
||||||
|
format: { format_name: "mov" },
|
||||||
|
streams: [
|
||||||
|
stream({ width: 600, height: 600, disposition: { attached_pic: 1 } }),
|
||||||
|
stream({
|
||||||
|
width: 640,
|
||||||
|
height: 480,
|
||||||
|
avg_frame_rate: "0/0",
|
||||||
|
r_frame_rate: "90000/1",
|
||||||
|
sample_aspect_ratio: "4:3",
|
||||||
|
tags: { rotate: "90" },
|
||||||
|
duration: "4.5",
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(probe.durationMs).toBe(4500);
|
||||||
|
expect(probe.hasAudio).toBe(false);
|
||||||
|
expect(probe.video).toMatchObject({
|
||||||
|
width: 480,
|
||||||
|
height: 640,
|
||||||
|
rotation: 90,
|
||||||
|
fps: 0,
|
||||||
|
sampleAspectRatio: 3 / 4,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads the end of the last packet when the container has no duration", () => {
|
||||||
|
expect(packetDurationMs("0.000000,0.040000\n2.960000,0.040000\nN/A,N/A\n")).toBe(3000);
|
||||||
|
expect(packetDurationMs("")).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("ffmpeg arguments", () => {
|
||||||
|
it("opens inputs through the file protocol with the demuxer of their MIME type", () => {
|
||||||
|
expect(
|
||||||
|
ffmpegInputArgs("/a/b.webm", {
|
||||||
|
mimeType: "video/webm",
|
||||||
|
startSeconds: 2,
|
||||||
|
durationSeconds: 1.5,
|
||||||
|
}),
|
||||||
|
).toEqual([
|
||||||
|
...["-ss", "2", "-t", "1.5", "-protocol_whitelist", "file", "-f", "matroska"],
|
||||||
|
...["-i", "file:/a/b.webm"],
|
||||||
|
]);
|
||||||
|
expect(ffmpegInputArgs("/a/b", { mimeType: "application/octet-stream" })).toEqual([
|
||||||
|
...["-protocol_whitelist", "file", "-i", "file:/a/b"],
|
||||||
|
]);
|
||||||
|
expect(ffmpegSeconds(1e-7)).toBe("0");
|
||||||
|
expect(ffmpegSeconds(12.34567)).toBe("12.346");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// CI runners may not have ffmpeg; these run wherever it is installed.
|
||||||
|
describe.skipIf(!ffmpeg)("ffmpeg integration", () => {
|
||||||
|
let clips: TestClips;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
clips = await createTestClips();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
if (clips) await rm(clips.dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("probes duration, size, frame rate and audio", async () => {
|
||||||
|
const probe = await probeMedia(clips.landscape, { mimeType: "video/mp4" });
|
||||||
|
expect(probe.durationMs).toBeGreaterThanOrEqual(2900);
|
||||||
|
expect(probe.durationMs).toBeLessThanOrEqual(3100);
|
||||||
|
expect(probe.video).toMatchObject({ codec: "h264", width: 320, height: 180, fps: 25 });
|
||||||
|
expect(probe.hasAudio).toBe(true);
|
||||||
|
const silent = await probeMedia(clips.silentPortrait, { mimeType: "video/mp4" });
|
||||||
|
expect(silent.hasAudio).toBe(false);
|
||||||
|
expect(silent.video).toMatchObject({ width: 180, height: 320 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses a playlist disguised as an MP4 instead of following its URLs", async () => {
|
||||||
|
const playlist = join(clips.dir, "playlist.mp4");
|
||||||
|
await writeFile(
|
||||||
|
playlist,
|
||||||
|
`#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:1.0,\nfile:${clips.srt}\n#EXT-X-ENDLIST\n`,
|
||||||
|
);
|
||||||
|
await expect(probeMedia(playlist, { mimeType: "video/mp4" })).rejects.toThrow(FfmpegError);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports progress up to 1 for a run", async () => {
|
||||||
|
const output = join(clips.dir, "progress.mp4");
|
||||||
|
const fractions: number[] = [];
|
||||||
|
await runFfmpeg(
|
||||||
|
[
|
||||||
|
...ffmpegInputArgs(clips.landscape, { mimeType: "video/mp4" }),
|
||||||
|
...[
|
||||||
|
"-c:v",
|
||||||
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"ultrafast",
|
||||||
|
"-c:a",
|
||||||
|
"aac",
|
||||||
|
"-f",
|
||||||
|
"mp4",
|
||||||
|
`file:${output}`,
|
||||||
|
],
|
||||||
|
],
|
||||||
|
{ durationMs: 3000, onProgress: (fraction) => fractions.push(fraction) },
|
||||||
|
);
|
||||||
|
expect(fractions.at(-1)).toBe(1);
|
||||||
|
expect(fractions).toEqual([...fractions].sort((a, b) => a - b));
|
||||||
|
expect(existsSync(output)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails with the last stderr line, without directory paths, and removes outputs", async () => {
|
||||||
|
const output = join(clips.dir, "broken.mp4");
|
||||||
|
await writeFile(output, "partial");
|
||||||
|
const failure = runFfmpeg(
|
||||||
|
[
|
||||||
|
...ffmpegInputArgs(join(clips.dir, "missing.mp4"), { mimeType: "video/mp4" }),
|
||||||
|
`file:${output}`,
|
||||||
|
],
|
||||||
|
{ outputs: [output] },
|
||||||
|
);
|
||||||
|
const error = await failure.catch((caught: unknown) => caught);
|
||||||
|
expect(error).toBeInstanceOf(FfmpegError);
|
||||||
|
expect((error as Error).message).toMatch(/^ffmpeg failed: .*missing\.mp4.*No such file/);
|
||||||
|
expect((error as Error).message).not.toContain(clips.dir);
|
||||||
|
expect(existsSync(output)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("kills ffmpeg on abort and removes the partial output", async () => {
|
||||||
|
const output = join(clips.dir, "aborted.mp4");
|
||||||
|
const controller = new AbortController();
|
||||||
|
const run = runFfmpeg(
|
||||||
|
[
|
||||||
|
...["-re", "-f", "lavfi", "-i", "testsrc=size=320x180:rate=25"],
|
||||||
|
...["-t", "60", "-c:v", "libx264", "-preset", "ultrafast", "-f", "mp4", `file:${output}`],
|
||||||
|
],
|
||||||
|
{
|
||||||
|
signal: controller.signal,
|
||||||
|
durationMs: 60_000,
|
||||||
|
onProgress: () => controller.abort(new Error("cancelled")),
|
||||||
|
outputs: [output],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
await expect(run).rejects.toThrow("cancelled");
|
||||||
|
expect(existsSync(output)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stops when the output outgrows its cap", async () => {
|
||||||
|
const output = join(clips.dir, "too-large.mp4");
|
||||||
|
const run = runFfmpeg(
|
||||||
|
[
|
||||||
|
...["-f", "lavfi", "-i", "testsrc=size=640x360:rate=25:duration=20"],
|
||||||
|
...["-c:v", "libx264", "-preset", "ultrafast", "-crf", "0", "-f", "mp4", `file:${output}`],
|
||||||
|
],
|
||||||
|
{ maxOutputBytes: 64 * 1024, outputs: [output] },
|
||||||
|
);
|
||||||
|
await expect(run).rejects.toThrow("The result would exceed 0 MB.");
|
||||||
|
expect(existsSync(output)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts 16 kHz mono WAV for transcription, optionally a range", async () => {
|
||||||
|
const output = join(clips.dir, "speech.wav");
|
||||||
|
await extractAudio(clips.landscape, output, { mimeType: "video/mp4", startMs: 1000 });
|
||||||
|
const probe = await probeMedia(output, { mimeType: "audio/wav" });
|
||||||
|
expect(probe.audio).toEqual({ codec: "pcm_s16le", sampleRate: 16_000, channels: 1 });
|
||||||
|
expect(probe.video).toBeUndefined();
|
||||||
|
expect(Math.abs(probe.durationMs - 2000)).toBeLessThan(100);
|
||||||
|
await expect(
|
||||||
|
extractAudio(clips.silentPortrait, join(clips.dir, "none.wav"), { mimeType: "video/mp4" }),
|
||||||
|
).rejects.toThrow("The file has no audio track.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders a poster frame, falling back to the first frame past the end", async () => {
|
||||||
|
const poster = join(clips.dir, "poster.jpg");
|
||||||
|
await renderPoster(clips.landscape, poster, { mimeType: "video/mp4", atMs: 60_000 });
|
||||||
|
const probe = await probeMedia(poster);
|
||||||
|
expect(probe.video).toMatchObject({ codec: "mjpeg", width: 320, height: 180 });
|
||||||
|
});
|
||||||
|
});
|
||||||
603
packages/adapters/src/media/ffmpeg.ts
Normal file
603
packages/adapters/src/media/ffmpeg.ts
Normal file
@ -0,0 +1,603 @@
|
|||||||
|
import { spawn } from "node:child_process";
|
||||||
|
import { constants } from "node:fs";
|
||||||
|
import { access, rm, stat } from "node:fs/promises";
|
||||||
|
import { setPriority } from "node:os";
|
||||||
|
import { delimiter, join } from "node:path";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ffmpeg and ffprobe from PATH. Every call passes an argument array (never a shell), opens
|
||||||
|
* inputs through the `file` protocol only, and forces the demuxer from the artifact's MIME type
|
||||||
|
* so a crafted upload cannot make ffmpeg probe it as a playlist that reads other files or URLs.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface FfmpegInfo {
|
||||||
|
ffmpegPath: string;
|
||||||
|
ffprobePath: string;
|
||||||
|
version: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MediaProbe {
|
||||||
|
durationMs: number;
|
||||||
|
formatName: string;
|
||||||
|
/** Width and height as displayed: swapped when the stream is rotated by 90 or 270 degrees. */
|
||||||
|
video?: {
|
||||||
|
codec: string;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
rotation: 0 | 90 | 180 | 270;
|
||||||
|
/** Frames per second; 0 when unknown or implausible (variable frame rate files). */
|
||||||
|
fps: number;
|
||||||
|
/** Pixel aspect ratio of the displayed frame; 1 for square pixels. */
|
||||||
|
sampleAspectRatio: number;
|
||||||
|
};
|
||||||
|
audio?: { codec: string; sampleRate: number; channels: number };
|
||||||
|
hasAudio: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FfmpegProgress {
|
||||||
|
outTimeUs?: number;
|
||||||
|
totalSize?: number;
|
||||||
|
done: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RunFfmpegOptions {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
/** Expected output duration; progress is reported as out_time / durationMs. */
|
||||||
|
durationMs?: number;
|
||||||
|
onProgress?: (fraction: number) => void;
|
||||||
|
cwd?: string;
|
||||||
|
/** Kills ffmpeg once the output grows past this size. */
|
||||||
|
maxOutputBytes?: number;
|
||||||
|
/** Files removed when the run fails or is aborted. */
|
||||||
|
outputs?: readonly string[];
|
||||||
|
/** Kills ffmpeg when it reports no progress for this long. */
|
||||||
|
stallTimeoutMs?: number;
|
||||||
|
ffmpegPath?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class FfmpegError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
readonly exitCode: number | null,
|
||||||
|
readonly stderrTail: string,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "FfmpegError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_STALL_TIMEOUT_MS = 2 * 60_000;
|
||||||
|
const PROBE_TIMEOUT_MS = 60_000;
|
||||||
|
const STDERR_TAIL_BYTES = 8 * 1024;
|
||||||
|
const KILL_GRACE_MS = 5_000;
|
||||||
|
const MAX_PLAUSIBLE_FPS = 240;
|
||||||
|
|
||||||
|
const DEMUXERS: Record<string, string> = {
|
||||||
|
"video/mp4": "mov",
|
||||||
|
"video/quicktime": "mov",
|
||||||
|
"audio/mp4": "mov",
|
||||||
|
"video/webm": "matroska",
|
||||||
|
"audio/webm": "matroska",
|
||||||
|
"audio/mpeg": "mp3",
|
||||||
|
"audio/wav": "wav",
|
||||||
|
"audio/ogg": "ogg",
|
||||||
|
};
|
||||||
|
|
||||||
|
let detection: Promise<FfmpegInfo | null> | undefined;
|
||||||
|
|
||||||
|
/** Finds ffmpeg and ffprobe on PATH once per process; null when either is missing or broken. */
|
||||||
|
export function detectFfmpeg(): Promise<FfmpegInfo | null> {
|
||||||
|
detection ??= locateFfmpeg();
|
||||||
|
return detection;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function locateFfmpeg(): Promise<FfmpegInfo | null> {
|
||||||
|
const [ffmpegPath, ffprobePath] = await Promise.all([
|
||||||
|
findExecutable("ffmpeg"),
|
||||||
|
findExecutable("ffprobe"),
|
||||||
|
]);
|
||||||
|
if (!ffmpegPath || !ffprobePath) return null;
|
||||||
|
try {
|
||||||
|
const [ffmpeg, ffprobe] = await Promise.all([
|
||||||
|
capture(ffmpegPath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }),
|
||||||
|
capture(ffprobePath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }),
|
||||||
|
]);
|
||||||
|
if (ffmpeg.code !== 0 || ffprobe.code !== 0) return null;
|
||||||
|
const version = /version (\S+)/.exec(ffmpeg.stdout)?.[1] ?? "unknown";
|
||||||
|
return { ffmpegPath, ffprobePath, version };
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function findExecutable(name: string): Promise<string | null> {
|
||||||
|
const extensions =
|
||||||
|
process.platform === "win32"
|
||||||
|
? (process.env.PATHEXT ?? ".EXE").split(";").filter(Boolean)
|
||||||
|
: [""];
|
||||||
|
for (const directory of (process.env.PATH ?? "").split(delimiter)) {
|
||||||
|
if (!directory) continue;
|
||||||
|
for (const extension of extensions) {
|
||||||
|
const candidate = join(directory, `${name}${extension}`);
|
||||||
|
try {
|
||||||
|
await access(candidate, constants.X_OK);
|
||||||
|
if ((await stat(candidate)).isFile()) return candidate;
|
||||||
|
} catch {
|
||||||
|
// Not in this directory.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function requireFfmpeg(): Promise<FfmpegInfo> {
|
||||||
|
const info = await detectFfmpeg();
|
||||||
|
if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server.");
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ffmpeg time argument with millisecond precision and no exponent notation. */
|
||||||
|
export function ffmpegSeconds(seconds: number): string {
|
||||||
|
return String(Number(Math.max(0, seconds).toFixed(3)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Input arguments for a local file: optional seek, file protocol only, forced demuxer. */
|
||||||
|
export function ffmpegInputArgs(
|
||||||
|
path: string,
|
||||||
|
options: { mimeType?: string; startSeconds?: number; durationSeconds?: number } = {},
|
||||||
|
): string[] {
|
||||||
|
const demuxer = options.mimeType ? DEMUXERS[options.mimeType] : undefined;
|
||||||
|
return [
|
||||||
|
...(options.startSeconds ? ["-ss", ffmpegSeconds(options.startSeconds)] : []),
|
||||||
|
...(options.durationSeconds !== undefined
|
||||||
|
? ["-t", ffmpegSeconds(options.durationSeconds)]
|
||||||
|
: []),
|
||||||
|
"-protocol_whitelist",
|
||||||
|
"file",
|
||||||
|
...(demuxer ? ["-f", demuxer] : []),
|
||||||
|
"-i",
|
||||||
|
`file:${path}`,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function probeMedia(
|
||||||
|
path: string,
|
||||||
|
options: { signal?: AbortSignal; mimeType?: string; ffprobePath?: string } = {},
|
||||||
|
): Promise<MediaProbe> {
|
||||||
|
const ffprobePath = options.ffprobePath ?? (await requireFfmpeg()).ffprobePath;
|
||||||
|
const input = ffmpegInputArgs(path, { mimeType: options.mimeType });
|
||||||
|
const result = await capture(
|
||||||
|
ffprobePath,
|
||||||
|
["-v", "error", ...input, "-print_format", "json", "-show_format", "-show_streams"],
|
||||||
|
{ signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS },
|
||||||
|
);
|
||||||
|
if (result.code !== 0) throw ffmpegFailure("ffprobe", result.code, result.stderr);
|
||||||
|
const probe = parseProbeOutput(result.stdout);
|
||||||
|
if (probe.durationMs > 0 || (!probe.video && !probe.audio)) return probe;
|
||||||
|
// Recordings streamed by browsers often carry no duration header; read it from the packets.
|
||||||
|
const packets = await capture(
|
||||||
|
ffprobePath,
|
||||||
|
[
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
...input,
|
||||||
|
"-select_streams",
|
||||||
|
probe.video ? "v:0" : "a:0",
|
||||||
|
"-show_entries",
|
||||||
|
"packet=pts_time,duration_time",
|
||||||
|
"-of",
|
||||||
|
"csv=p=0",
|
||||||
|
],
|
||||||
|
{ signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS * 5 },
|
||||||
|
);
|
||||||
|
if (packets.code !== 0) throw ffmpegFailure("ffprobe", packets.code, packets.stderr);
|
||||||
|
return { ...probe, durationMs: packetDurationMs(packets.stdout) };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** End of the last packet, from ffprobe `packet=pts_time,duration_time` CSV lines. */
|
||||||
|
export function packetDurationMs(csv: string): number {
|
||||||
|
let end = 0;
|
||||||
|
for (const line of csv.split("\n")) {
|
||||||
|
const [pts, duration] = line.split(",").map(Number);
|
||||||
|
if (pts === undefined || !Number.isFinite(pts)) continue;
|
||||||
|
end = Math.max(end, pts + (duration !== undefined && Number.isFinite(duration) ? duration : 0));
|
||||||
|
}
|
||||||
|
return Math.round(end * 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProbeStream {
|
||||||
|
codec_type?: string;
|
||||||
|
codec_name?: string;
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
avg_frame_rate?: string;
|
||||||
|
r_frame_rate?: string;
|
||||||
|
sample_aspect_ratio?: string;
|
||||||
|
sample_rate?: string;
|
||||||
|
channels?: number;
|
||||||
|
duration?: string;
|
||||||
|
disposition?: { attached_pic?: number };
|
||||||
|
tags?: { rotate?: string };
|
||||||
|
side_data_list?: { rotation?: number }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Parses `ffprobe -print_format json -show_format -show_streams` output. */
|
||||||
|
export function parseProbeOutput(json: string): MediaProbe {
|
||||||
|
const data = JSON.parse(json) as {
|
||||||
|
format?: { duration?: string; format_name?: string };
|
||||||
|
streams?: ProbeStream[];
|
||||||
|
};
|
||||||
|
const streams = data.streams ?? [];
|
||||||
|
// Cover art in audio files is a single attached picture, not a video track.
|
||||||
|
const video = streams.find(
|
||||||
|
(stream) => stream.codec_type === "video" && stream.disposition?.attached_pic !== 1,
|
||||||
|
);
|
||||||
|
const audio = streams.find((stream) => stream.codec_type === "audio");
|
||||||
|
const seconds =
|
||||||
|
positive(Number(data.format?.duration)) ??
|
||||||
|
Math.max(0, ...streams.map((stream) => positive(Number(stream.duration)) ?? 0));
|
||||||
|
return {
|
||||||
|
durationMs: Math.round(seconds * 1000),
|
||||||
|
formatName: data.format?.format_name ?? "",
|
||||||
|
...(video?.width && video.height
|
||||||
|
? { video: probeVideo(video, video.width, video.height) }
|
||||||
|
: {}),
|
||||||
|
...(audio
|
||||||
|
? {
|
||||||
|
audio: {
|
||||||
|
codec: audio.codec_name ?? "",
|
||||||
|
sampleRate: Number(audio.sample_rate) || 0,
|
||||||
|
channels: audio.channels ?? 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
hasAudio: Boolean(audio),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function probeVideo(stream: ProbeStream, width: number, height: number) {
|
||||||
|
const rotation = normalizeRotation(
|
||||||
|
stream.side_data_list?.find((entry) => typeof entry.rotation === "number")?.rotation ??
|
||||||
|
Number(stream.tags?.rotate ?? 0),
|
||||||
|
);
|
||||||
|
const sideways = rotation === 90 || rotation === 270;
|
||||||
|
const sar = ratio(stream.sample_aspect_ratio, ":") || 1;
|
||||||
|
const fps = ratio(stream.avg_frame_rate, "/") || ratio(stream.r_frame_rate, "/");
|
||||||
|
return {
|
||||||
|
codec: stream.codec_name ?? "",
|
||||||
|
width: sideways ? height : width,
|
||||||
|
height: sideways ? width : height,
|
||||||
|
rotation,
|
||||||
|
fps: fps > 0 && fps <= MAX_PLAUSIBLE_FPS ? fps : 0,
|
||||||
|
sampleAspectRatio: sideways ? 1 / sar : sar,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeRotation(degrees: number): 0 | 90 | 180 | 270 {
|
||||||
|
if (!Number.isFinite(degrees)) return 0;
|
||||||
|
const quarter = (((Math.round(degrees / 90) % 4) + 4) % 4) as 0 | 1 | 2 | 3;
|
||||||
|
return ([0, 90, 180, 270] as const)[quarter];
|
||||||
|
}
|
||||||
|
|
||||||
|
function ratio(value: string | undefined, separator: string): number {
|
||||||
|
if (!value) return 0;
|
||||||
|
const [numerator, denominator] = value.split(separator).map(Number);
|
||||||
|
if (!numerator || !denominator || !Number.isFinite(numerator / denominator)) return 0;
|
||||||
|
return numerator / denominator;
|
||||||
|
}
|
||||||
|
|
||||||
|
function positive(value: number): number | undefined {
|
||||||
|
return Number.isFinite(value) && value > 0 ? value : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Feeds ffmpeg `-progress` output (key=value lines) and reports each completed block. */
|
||||||
|
export function createFfmpegProgressParser(
|
||||||
|
onUpdate: (progress: FfmpegProgress) => void,
|
||||||
|
): (chunk: string) => void {
|
||||||
|
let buffer = "";
|
||||||
|
let current: Omit<FfmpegProgress, "done"> = {};
|
||||||
|
return (chunk) => {
|
||||||
|
buffer += chunk;
|
||||||
|
for (let newline = buffer.indexOf("\n"); newline >= 0; newline = buffer.indexOf("\n")) {
|
||||||
|
const line = buffer.slice(0, newline).trim();
|
||||||
|
buffer = buffer.slice(newline + 1);
|
||||||
|
const separator = line.indexOf("=");
|
||||||
|
if (separator < 0) continue;
|
||||||
|
const key = line.slice(0, separator);
|
||||||
|
const value = line.slice(separator + 1).trim();
|
||||||
|
const number = Number(value);
|
||||||
|
// out_time_ms is microseconds too (a long-standing ffmpeg naming bug).
|
||||||
|
if ((key === "out_time_us" || key === "out_time_ms") && value && number >= 0) {
|
||||||
|
current.outTimeUs = number;
|
||||||
|
} else if (key === "total_size" && value && number >= 0) {
|
||||||
|
current.totalSize = number;
|
||||||
|
} else if (key === "progress") {
|
||||||
|
onUpdate({ ...current, done: value === "end" });
|
||||||
|
current = {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Runs ffmpeg with progress on stdout; rejects with the stderr tail when it fails. */
|
||||||
|
export async function runFfmpeg(
|
||||||
|
args: readonly string[],
|
||||||
|
options: RunFfmpegOptions = {},
|
||||||
|
): Promise<void> {
|
||||||
|
options.signal?.throwIfAborted();
|
||||||
|
const ffmpegPath = options.ffmpegPath ?? (await requireFfmpeg()).ffmpegPath;
|
||||||
|
try {
|
||||||
|
await spawnFfmpeg(ffmpegPath, args, options);
|
||||||
|
} catch (error) {
|
||||||
|
await Promise.all((options.outputs ?? []).map((path) => rm(path, { force: true })));
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function spawnFfmpeg(
|
||||||
|
ffmpegPath: string,
|
||||||
|
args: readonly string[],
|
||||||
|
options: RunFfmpegOptions,
|
||||||
|
): Promise<void> {
|
||||||
|
const { signal, durationMs, onProgress, maxOutputBytes } = options;
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const child = spawn(
|
||||||
|
ffmpegPath,
|
||||||
|
[
|
||||||
|
"-hide_banner",
|
||||||
|
"-nostdin",
|
||||||
|
"-nostats",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-progress",
|
||||||
|
"pipe:1",
|
||||||
|
"-y",
|
||||||
|
...args,
|
||||||
|
],
|
||||||
|
{ cwd: options.cwd, stdio: ["ignore", "pipe", "pipe"], windowsHide: true },
|
||||||
|
);
|
||||||
|
if (child.pid) {
|
||||||
|
try {
|
||||||
|
// Long encodes must not starve the API process that shares this machine.
|
||||||
|
setPriority(child.pid, 10);
|
||||||
|
} catch {
|
||||||
|
// Lowering priority is best effort.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let stderr = "";
|
||||||
|
let settled = false;
|
||||||
|
let reported = 0;
|
||||||
|
let failure: unknown;
|
||||||
|
let stallTimer: NodeJS.Timeout | undefined;
|
||||||
|
const finish = (error?: unknown) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(stallTimer);
|
||||||
|
signal?.removeEventListener("abort", abort);
|
||||||
|
if (error === undefined) resolve();
|
||||||
|
else reject(error);
|
||||||
|
};
|
||||||
|
// Settles once the process is gone, so callers can delete its files right away.
|
||||||
|
const kill = (error: unknown) => {
|
||||||
|
if (failure !== undefined || settled) return;
|
||||||
|
failure = error;
|
||||||
|
clearTimeout(stallTimer);
|
||||||
|
child.kill("SIGKILL");
|
||||||
|
setTimeout(() => finish(error), KILL_GRACE_MS).unref?.();
|
||||||
|
};
|
||||||
|
const abort = () => kill(signal?.reason ?? new Error("aborted"));
|
||||||
|
const stallTimeoutMs = options.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;
|
||||||
|
const armStallTimer = () => {
|
||||||
|
clearTimeout(stallTimer);
|
||||||
|
stallTimer = setTimeout(
|
||||||
|
() => kill(new FfmpegError("ffmpeg stopped making progress.", null, tail(stderr))),
|
||||||
|
stallTimeoutMs,
|
||||||
|
);
|
||||||
|
stallTimer.unref?.();
|
||||||
|
};
|
||||||
|
const parse = createFfmpegProgressParser((progress) => {
|
||||||
|
if (failure !== undefined) return;
|
||||||
|
if (maxOutputBytes !== undefined && (progress.totalSize ?? 0) > maxOutputBytes) {
|
||||||
|
kill(new Error(outputTooLargeMessage(maxOutputBytes)));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!onProgress) return;
|
||||||
|
const fraction = progress.done
|
||||||
|
? 1
|
||||||
|
: durationMs && progress.outTimeUs !== undefined
|
||||||
|
? Math.min(1, progress.outTimeUs / (durationMs * 1000))
|
||||||
|
: 0;
|
||||||
|
if (fraction > reported) {
|
||||||
|
reported = fraction;
|
||||||
|
onProgress(fraction);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
signal?.addEventListener("abort", abort, { once: true });
|
||||||
|
armStallTimer();
|
||||||
|
child.stdout.setEncoding("utf8");
|
||||||
|
child.stdout.on("data", (chunk: string) => {
|
||||||
|
if (failure === undefined) armStallTimer();
|
||||||
|
parse(chunk);
|
||||||
|
});
|
||||||
|
child.stderr.setEncoding("utf8");
|
||||||
|
child.stderr.on("data", (chunk: string) => {
|
||||||
|
stderr = tail(stderr + chunk);
|
||||||
|
});
|
||||||
|
child.on("error", (error) => finish(failure ?? error));
|
||||||
|
child.on("close", (code) => {
|
||||||
|
if (failure !== undefined) finish(failure);
|
||||||
|
else if (code === 0) finish();
|
||||||
|
else finish(ffmpegFailure("ffmpeg", code, stderr));
|
||||||
|
});
|
||||||
|
if (signal?.aborted) abort();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function outputTooLargeMessage(maxBytes: number): string {
|
||||||
|
return `The result would exceed ${Math.round(maxBytes / (1024 * 1024))} MB.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tail(text: string): string {
|
||||||
|
return text.length > STDERR_TAIL_BYTES ? text.slice(-STDERR_TAIL_BYTES) : text;
|
||||||
|
}
|
||||||
|
|
||||||
|
function ffmpegFailure(tool: string, code: number | null, stderr: string): FfmpegError {
|
||||||
|
// The cause is usually the line before ffmpeg's generic summary ("Conversion failed!").
|
||||||
|
const lines = stderr
|
||||||
|
.split("\n")
|
||||||
|
.map((line) => line.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.slice(-2);
|
||||||
|
// Name files by their base name only; the job's temp paths mean nothing to the reader.
|
||||||
|
const detail = lines.join(" ").replaceAll(/(?:file:)?(?:[A-Za-z]:)?[\\/][^\s'"]*[\\/]/g, "");
|
||||||
|
return new FfmpegError(
|
||||||
|
detail ? `${tool} failed: ${detail}` : `${tool} failed with exit code ${code}.`,
|
||||||
|
code,
|
||||||
|
tail(stderr),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Extracts the first audio track as 16 kHz mono 16-bit WAV, the format speech models expect. */
|
||||||
|
export async function extractAudio(
|
||||||
|
input: string,
|
||||||
|
output: string,
|
||||||
|
options: {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
mimeType?: string;
|
||||||
|
startMs?: number;
|
||||||
|
endMs?: number;
|
||||||
|
onProgress?: (fraction: number) => void;
|
||||||
|
} = {},
|
||||||
|
): Promise<void> {
|
||||||
|
const tools = await requireFfmpeg();
|
||||||
|
const probe = await probeMedia(input, {
|
||||||
|
signal: options.signal,
|
||||||
|
mimeType: options.mimeType,
|
||||||
|
ffprobePath: tools.ffprobePath,
|
||||||
|
});
|
||||||
|
if (!probe.hasAudio) throw new Error("The file has no audio track.");
|
||||||
|
const startMs = Math.max(0, options.startMs ?? 0);
|
||||||
|
const endMs = Math.min(probe.durationMs, options.endMs ?? probe.durationMs);
|
||||||
|
if (endMs <= startMs) throw new Error("The audio range is empty.");
|
||||||
|
await runFfmpeg(
|
||||||
|
[
|
||||||
|
...ffmpegInputArgs(input, {
|
||||||
|
mimeType: options.mimeType,
|
||||||
|
startSeconds: startMs / 1000,
|
||||||
|
durationSeconds: (endMs - startMs) / 1000,
|
||||||
|
}),
|
||||||
|
"-map",
|
||||||
|
"0:a:0",
|
||||||
|
"-ac",
|
||||||
|
"1",
|
||||||
|
"-ar",
|
||||||
|
"16000",
|
||||||
|
"-c:a",
|
||||||
|
"pcm_s16le",
|
||||||
|
"-map_metadata",
|
||||||
|
"-1",
|
||||||
|
"-f",
|
||||||
|
"wav",
|
||||||
|
`file:${output}`,
|
||||||
|
],
|
||||||
|
{
|
||||||
|
signal: options.signal,
|
||||||
|
durationMs: endMs - startMs,
|
||||||
|
onProgress: options.onProgress,
|
||||||
|
outputs: [output],
|
||||||
|
ffmpegPath: tools.ffmpegPath,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Writes one JPEG frame at atMs (default 0), falling back to the first frame past the end. */
|
||||||
|
export async function renderPoster(
|
||||||
|
input: string,
|
||||||
|
output: string,
|
||||||
|
options: { signal?: AbortSignal; mimeType?: string; atMs?: number } = {},
|
||||||
|
): Promise<void> {
|
||||||
|
const ffmpegPath = (await requireFfmpeg()).ffmpegPath;
|
||||||
|
const render = (atMs: number) =>
|
||||||
|
runFfmpeg(
|
||||||
|
[
|
||||||
|
...ffmpegInputArgs(input, { mimeType: options.mimeType, startSeconds: atMs / 1000 }),
|
||||||
|
"-map",
|
||||||
|
"0:v:0",
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-c:v",
|
||||||
|
"mjpeg",
|
||||||
|
"-q:v",
|
||||||
|
"3",
|
||||||
|
"-map_metadata",
|
||||||
|
"-1",
|
||||||
|
"-f",
|
||||||
|
"image2",
|
||||||
|
"-update",
|
||||||
|
"1",
|
||||||
|
`file:${output}`,
|
||||||
|
],
|
||||||
|
{ signal: options.signal, outputs: [output], ffmpegPath },
|
||||||
|
);
|
||||||
|
const atMs = Math.max(0, options.atMs ?? 0);
|
||||||
|
await render(atMs);
|
||||||
|
// Seeking into the last frame's gap writes nothing; the first frame is better than no poster.
|
||||||
|
if (atMs > 0 && !(await nonEmptyFile(output))) await render(0);
|
||||||
|
if (!(await nonEmptyFile(output))) throw new Error("Could not render a poster frame.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function nonEmptyFile(path: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
return (await stat(path)).size > 0;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CaptureResult {
|
||||||
|
code: number | null;
|
||||||
|
stdout: string;
|
||||||
|
stderr: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function capture(
|
||||||
|
command: string,
|
||||||
|
args: readonly string[],
|
||||||
|
options: { signal?: AbortSignal; timeoutMs: number },
|
||||||
|
): Promise<CaptureResult> {
|
||||||
|
options.signal?.throwIfAborted();
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], windowsHide: true });
|
||||||
|
const stdout: Buffer[] = [];
|
||||||
|
let stderr = "";
|
||||||
|
let settled = false;
|
||||||
|
const finish = (error: unknown, result?: CaptureResult) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
options.signal?.removeEventListener("abort", abort);
|
||||||
|
if (result) resolve(result);
|
||||||
|
else reject(error);
|
||||||
|
};
|
||||||
|
const kill = (error: unknown) => {
|
||||||
|
child.kill("SIGKILL");
|
||||||
|
finish(error);
|
||||||
|
};
|
||||||
|
const abort = () => kill(options.signal?.reason ?? new Error("aborted"));
|
||||||
|
const timer = setTimeout(
|
||||||
|
() => kill(new Error(`${command} timed out after ${options.timeoutMs} ms`)),
|
||||||
|
options.timeoutMs,
|
||||||
|
);
|
||||||
|
timer.unref?.();
|
||||||
|
options.signal?.addEventListener("abort", abort, { once: true });
|
||||||
|
child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk));
|
||||||
|
child.stderr.setEncoding("utf8");
|
||||||
|
child.stderr.on("data", (chunk: string) => {
|
||||||
|
stderr = tail(stderr + chunk);
|
||||||
|
});
|
||||||
|
child.on("error", (error) => finish(error));
|
||||||
|
child.on("close", (code) =>
|
||||||
|
finish(undefined, { code, stdout: Buffer.concat(stdout).toString("utf8"), stderr }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
106
packages/adapters/src/media/media-test-support.ts
Normal file
106
packages/adapters/src/media/media-test-support.ts
Normal file
@ -0,0 +1,106 @@
|
|||||||
|
import { copyFile, mkdtemp, stat, writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit";
|
||||||
|
import type { MediaOutput, MediaOutputKind } from "@rakazo/contracts";
|
||||||
|
import { attachmentKindOf } from "@rakazo/contracts";
|
||||||
|
import { runFfmpeg } from "./ffmpeg.js";
|
||||||
|
|
||||||
|
export interface TestClips {
|
||||||
|
dir: string;
|
||||||
|
/** 320×180, 25 fps, 3 s, 440 Hz tone. */
|
||||||
|
landscape: string;
|
||||||
|
/** 180×320, 30 fps, 2 s, no audio. */
|
||||||
|
silentPortrait: string;
|
||||||
|
srt: string;
|
||||||
|
vtt: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Tiny generated clips; nothing binary is committed. */
|
||||||
|
export async function createTestClips(): Promise<TestClips> {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "evoo-media-test-"));
|
||||||
|
const landscape = join(dir, "landscape.mp4");
|
||||||
|
const silentPortrait = join(dir, "silent-portrait.mp4");
|
||||||
|
const encode = ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"];
|
||||||
|
await runFfmpeg([
|
||||||
|
...["-f", "lavfi", "-i", "testsrc=size=320x180:rate=25:duration=3"],
|
||||||
|
...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=3"],
|
||||||
|
...encode,
|
||||||
|
...["-c:a", "aac", "-shortest", `file:${landscape}`],
|
||||||
|
]);
|
||||||
|
await runFfmpeg([
|
||||||
|
...["-f", "lavfi", "-i", "testsrc2=size=180x320:rate=30:duration=2"],
|
||||||
|
...encode,
|
||||||
|
`file:${silentPortrait}`,
|
||||||
|
]);
|
||||||
|
const srt = join(dir, "captions.srt");
|
||||||
|
await writeFile(
|
||||||
|
srt,
|
||||||
|
"1\n00:00:00,100 --> 00:00:01,500\nHello from the subtitles\n\n2\n00:00:01,500 --> 00:00:02,900\nZweite Zeile äöü\n",
|
||||||
|
);
|
||||||
|
const vtt = join(dir, "captions.vtt");
|
||||||
|
await writeFile(vtt, "WEBVTT\n\n00:00.100 --> 00:01.500\nHello from WebVTT\n");
|
||||||
|
return { dir, landscape, silentPortrait, srt, vtt };
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TestArtifact {
|
||||||
|
path: string;
|
||||||
|
name: string;
|
||||||
|
mimeType: string;
|
||||||
|
kind?: MediaOutputKind;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A MediaJobContext over local files that records progress and outputs. */
|
||||||
|
export async function fakeMediaJobContext(input: unknown, artifacts: Record<string, TestArtifact>) {
|
||||||
|
const tempDir = await mkdtemp(join(tmpdir(), "evoo-media-job-"));
|
||||||
|
const controller = new AbortController();
|
||||||
|
const progress: number[] = [];
|
||||||
|
const outputs: (MediaJobOutputFile & { size: number; posterSize?: number })[] = [];
|
||||||
|
const context: MediaJobContext = {
|
||||||
|
jobId: "job-1",
|
||||||
|
spaceId: "space-1",
|
||||||
|
userId: "user-1",
|
||||||
|
botId: null,
|
||||||
|
input,
|
||||||
|
tempDir,
|
||||||
|
signal: controller.signal,
|
||||||
|
progress: async (fraction) => {
|
||||||
|
progress.push(fraction);
|
||||||
|
},
|
||||||
|
fetchArtifact: async (artifactId): Promise<MediaJobInputFile> => {
|
||||||
|
const artifact = artifacts[artifactId];
|
||||||
|
if (!artifact) throw new Error(`Artifact ${artifactId} not found.`);
|
||||||
|
const path = join(tempDir, `input-${artifactId}`);
|
||||||
|
await copyFile(artifact.path, path);
|
||||||
|
return {
|
||||||
|
artifactId,
|
||||||
|
path,
|
||||||
|
name: artifact.name,
|
||||||
|
mimeType: artifact.mimeType,
|
||||||
|
kind: artifact.kind ?? attachmentKindOf(artifact.mimeType),
|
||||||
|
size: (await stat(path)).size,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
addOutput: async (file): Promise<MediaOutput> => {
|
||||||
|
// Files must exist when handed over; the runner stores them before the job ends.
|
||||||
|
const size = (await stat(file.path)).size;
|
||||||
|
const posterSize = file.posterPath ? (await stat(file.posterPath)).size : undefined;
|
||||||
|
outputs.push({ ...file, size, ...(posterSize !== undefined ? { posterSize } : {}) });
|
||||||
|
const index = outputs.length;
|
||||||
|
return {
|
||||||
|
artifactId: `output-${index}`,
|
||||||
|
kind: attachmentKindOf(file.mimeType),
|
||||||
|
mimeType: file.mimeType,
|
||||||
|
name: file.name,
|
||||||
|
size,
|
||||||
|
...(file.width ? { width: file.width } : {}),
|
||||||
|
...(file.height ? { height: file.height } : {}),
|
||||||
|
...(file.durationMs !== undefined ? { durationMs: file.durationMs } : {}),
|
||||||
|
...(file.posterPath ? { posterArtifactId: `poster-${index}` } : {}),
|
||||||
|
...(file.parentArtifactId ? { parentArtifactId: file.parentArtifactId } : {}),
|
||||||
|
...(file.preset ? { preset: file.preset } : {}),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return { context, controller, progress, outputs, tempDir };
|
||||||
|
}
|
||||||
301
packages/adapters/src/media/video-edit-plan.test.ts
Normal file
301
packages/adapters/src/media/video-edit-plan.test.ts
Normal file
@ -0,0 +1,301 @@
|
|||||||
|
import type { EditVideoStep } from "@rakazo/contracts";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { MediaProbe } from "./ffmpeg.js";
|
||||||
|
import type { PlanInput } from "./video-edit-plan.js";
|
||||||
|
import { atempoFilters, planVideoEdit, subtitleStyle } from "./video-edit-plan.js";
|
||||||
|
|
||||||
|
function probe(
|
||||||
|
overrides: Partial<NonNullable<MediaProbe["video"]>> & {
|
||||||
|
durationMs?: number;
|
||||||
|
hasAudio?: boolean;
|
||||||
|
} = {},
|
||||||
|
): MediaProbe {
|
||||||
|
const { durationMs = 10_000, hasAudio = true, ...video } = overrides;
|
||||||
|
return {
|
||||||
|
durationMs,
|
||||||
|
formatName: "mov,mp4,m4a,3gp,3g2,mj2",
|
||||||
|
video: {
|
||||||
|
codec: "h264",
|
||||||
|
width: 1920,
|
||||||
|
height: 1080,
|
||||||
|
rotation: 0,
|
||||||
|
fps: 25,
|
||||||
|
sampleAspectRatio: 1,
|
||||||
|
...video,
|
||||||
|
},
|
||||||
|
...(hasAudio ? { audio: { codec: "aac", sampleRate: 44_100, channels: 2 } } : {}),
|
||||||
|
hasAudio,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const source: PlanInput = {
|
||||||
|
path: "/jobs/1/input-source",
|
||||||
|
mimeType: "video/mp4",
|
||||||
|
name: "holiday.mp4",
|
||||||
|
probe: probe(),
|
||||||
|
};
|
||||||
|
|
||||||
|
function plan(steps: EditVideoStep[], extra: Partial<Parameters<typeof planVideoEdit>[0]> = {}) {
|
||||||
|
return planVideoEdit({ source, steps, outputPath: "/jobs/1/output.mp4", ...extra });
|
||||||
|
}
|
||||||
|
|
||||||
|
function graphOf(args: string[]): string {
|
||||||
|
return args[args.indexOf("-filter_complex") + 1]!;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("planVideoEdit", () => {
|
||||||
|
it("seeks the input for a leading trim and encodes H.264/AAC MP4", () => {
|
||||||
|
const result = plan([{ op: "trim", start: 1.5, end: 4 }]);
|
||||||
|
expect(result.args).toEqual([
|
||||||
|
...["-ss", "1.5", "-t", "2.5", "-protocol_whitelist", "file", "-f", "mov"],
|
||||||
|
...["-i", "file:/jobs/1/input-source"],
|
||||||
|
"-filter_complex",
|
||||||
|
"[0:v:0]format=yuv420p[v1];[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a2]",
|
||||||
|
...["-map", "[v1]", "-map", "[a2]", "-c:a", "aac", "-b:a", "160k"],
|
||||||
|
...["-c:v", "libx264", "-preset", "veryfast", "-crf", "21", "-pix_fmt", "yuv420p"],
|
||||||
|
...["-movflags", "+faststart", "-map_metadata", "-1", "-map_chapters", "-1"],
|
||||||
|
...["-f", "mp4", "file:/jobs/1/output.mp4"],
|
||||||
|
]);
|
||||||
|
expect(result).toMatchObject({ kind: "video", durationMs: 2500, width: 1920, height: 1080 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps sorted, merged segments from one seek with split, trims and concat", () => {
|
||||||
|
const result = plan([
|
||||||
|
{
|
||||||
|
op: "keep",
|
||||||
|
segments: [
|
||||||
|
{ start: 6, end: 8 },
|
||||||
|
{ start: 2, end: 3 },
|
||||||
|
{ start: 2.5, end: 4 },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(result.args.slice(0, 4)).toEqual(["-ss", "2", "-t", "6"]);
|
||||||
|
const graph = graphOf(result.args);
|
||||||
|
expect(graph).toContain("[0:v:0]split=2[v1][v2]");
|
||||||
|
expect(graph).toContain("[v1]trim=start=0:end=2,setpts=PTS-STARTPTS[v3]");
|
||||||
|
expect(graph).toContain("[v2]trim=start=4:end=6,setpts=PTS-STARTPTS[v4]");
|
||||||
|
expect(graph).toContain("[0:a:0]asplit=2[a5][a6]");
|
||||||
|
expect(graph).toContain("[v3][a7][v4][a8]concat=n=2:v=1:a=1[v9][a10]");
|
||||||
|
expect(result.durationMs).toBe(4000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("applies later cuts on the timeline left by earlier steps", () => {
|
||||||
|
const result = plan([
|
||||||
|
{ op: "speed", factor: 2 },
|
||||||
|
{ op: "trim", start: 1, end: 99 },
|
||||||
|
]);
|
||||||
|
expect(graphOf(result.args)).toContain(
|
||||||
|
"[0:v:0]setpts=PTS/2,fps=25,trim=start=1:end=5,setpts=PTS-STARTPTS,format=yuv420p[v1]",
|
||||||
|
);
|
||||||
|
expect(result.durationMs).toBe(4000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects ranges outside the video and too short ones", () => {
|
||||||
|
expect(() => plan([{ op: "trim", start: 12, end: 18 }])).toThrow(
|
||||||
|
"trim 12–18 s is outside the 10 s video.",
|
||||||
|
);
|
||||||
|
expect(() =>
|
||||||
|
plan([
|
||||||
|
{ op: "speed", factor: 2 },
|
||||||
|
{
|
||||||
|
op: "keep",
|
||||||
|
segments: [
|
||||||
|
{ start: 1, end: 2 },
|
||||||
|
{ start: 6, end: 7 },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
).toThrow("keep 6–7 s is outside the 5 s video.");
|
||||||
|
expect(() => plan([{ op: "trim", start: 9.95, end: 12 }])).toThrow("shorter than 0.1 s");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("appends clips fitted to the frame, with silence for clips without audio", () => {
|
||||||
|
const clip: PlanInput = {
|
||||||
|
path: "/jobs/1/input-clip",
|
||||||
|
mimeType: "video/webm",
|
||||||
|
name: "clip.webm",
|
||||||
|
probe: probe({ width: 1080, height: 1920, fps: 30, durationMs: 2000, hasAudio: false }),
|
||||||
|
};
|
||||||
|
const result = plan([{ op: "concat", artifact_ids: ["clip"] }], {
|
||||||
|
clips: new Map([["clip", clip]]),
|
||||||
|
});
|
||||||
|
expect(result.args).toContain("matroska");
|
||||||
|
const graph = graphOf(result.args);
|
||||||
|
expect(graph).toContain("[0:v:0]setsar=1,format=yuv420p[v1]");
|
||||||
|
expect(graph).toContain(
|
||||||
|
"[1:v:0]scale=1920:1080:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1920:1080:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=25,format=yuv420p[v3]",
|
||||||
|
);
|
||||||
|
expect(graph).toContain(
|
||||||
|
"anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=2,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a4]",
|
||||||
|
);
|
||||||
|
expect(graph).toContain("[v1][a2][v3][a4]concat=n=2:v=1:a=1");
|
||||||
|
expect(result.durationMs).toBe(12_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("adds silence to a silent source when an appended clip has audio", () => {
|
||||||
|
const silent = { ...source, probe: probe({ hasAudio: false }) };
|
||||||
|
const result = planVideoEdit({
|
||||||
|
source: silent,
|
||||||
|
steps: [{ op: "concat", artifact_ids: ["clip"] }],
|
||||||
|
clips: new Map([["clip", source]]),
|
||||||
|
outputPath: "/out.mp4",
|
||||||
|
});
|
||||||
|
expect(graphOf(result.args)).toContain(
|
||||||
|
"anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=10",
|
||||||
|
);
|
||||||
|
expect(result.args.filter((arg) => arg === "-map")).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("appends clips with the last reframe's fit", () => {
|
||||||
|
const result = plan(
|
||||||
|
[
|
||||||
|
{ op: "reframe", preset: "vertical", fit: "crop" },
|
||||||
|
{ op: "concat", artifact_ids: ["clip"] },
|
||||||
|
],
|
||||||
|
{ clips: new Map([["clip", source]]) },
|
||||||
|
);
|
||||||
|
expect(graphOf(result.args)).toContain(
|
||||||
|
"[1:v:0]scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920,setsar=1,fps=25",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reframes by crop, pad or blur, and only scales when the aspect already matches", () => {
|
||||||
|
expect(graphOf(plan([{ op: "reframe", preset: "square", fit: "crop" }]).args)).toContain(
|
||||||
|
"scale=1080:1080:force_original_aspect_ratio=increase,crop=1080:1080,setsar=1",
|
||||||
|
);
|
||||||
|
expect(graphOf(plan([{ op: "reframe", preset: "portrait", fit: "pad" }]).args)).toContain(
|
||||||
|
"pad=1080:1350:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1",
|
||||||
|
);
|
||||||
|
const blur = plan([{ op: "reframe", preset: "vertical", fit: "blur" }]);
|
||||||
|
expect(graphOf(blur.args)).toBe(
|
||||||
|
[
|
||||||
|
"[0:v:0]split=2[v1][v2]",
|
||||||
|
"[v1]scale=270:480:force_original_aspect_ratio=increase,crop=270:480,boxblur=27:2,scale=1080:1920,setsar=1[v3]",
|
||||||
|
"[v2]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,setsar=1[v4]",
|
||||||
|
"[v3][v4]overlay=(W-w)/2:(H-h)/2[v5]",
|
||||||
|
"[v5]format=yuv420p[v6]",
|
||||||
|
"[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a7]",
|
||||||
|
].join(";"),
|
||||||
|
);
|
||||||
|
expect(blur).toMatchObject({ width: 1080, height: 1920 });
|
||||||
|
expect(graphOf(plan([{ op: "reframe", preset: "landscape", fit: "blur" }]).args)).toContain(
|
||||||
|
"[0:v:0]scale=1920:1080,setsar=1,format=yuv420p",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("changes speed with atempo chains and keeps the source frame rate", () => {
|
||||||
|
const fast = plan([{ op: "speed", factor: 4 }]);
|
||||||
|
expect(graphOf(fast.args)).toContain("setpts=PTS/4,fps=25");
|
||||||
|
expect(graphOf(fast.args)).toContain("atempo=2,atempo=2");
|
||||||
|
expect(fast.durationMs).toBe(2500);
|
||||||
|
expect(atempoFilters(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]);
|
||||||
|
expect(atempoFilters(3)).toEqual(["atempo=2", "atempo=1.5"]);
|
||||||
|
expect(atempoFilters(0.8)).toEqual(["atempo=0.8"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("changes volume, and mute drops the audio track", () => {
|
||||||
|
expect(graphOf(plan([{ op: "volume", gain: 0.5 }]).args)).toContain(
|
||||||
|
"[0:a:0]volume=0.5,aformat",
|
||||||
|
);
|
||||||
|
const muted = plan([{ op: "volume", gain: 2 }, { op: "mute" }]);
|
||||||
|
expect(muted.args.filter((arg) => arg === "-map")).toHaveLength(1);
|
||||||
|
expect(muted.args).not.toContain("aac");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("burns in subtitles styled for the current frame from a generated file name", () => {
|
||||||
|
const result = plan(
|
||||||
|
[
|
||||||
|
{ op: "reframe", preset: "vertical", fit: "crop" },
|
||||||
|
{ op: "subtitles", artifact_id: "subs" },
|
||||||
|
],
|
||||||
|
{ subtitleFiles: new Map([["subs", "subtitles-0.srt"]]) },
|
||||||
|
);
|
||||||
|
expect(graphOf(result.args)).toContain(
|
||||||
|
`subtitles=filename=subtitles-0.srt:force_style='${subtitleStyle(1080, 1920)}'`,
|
||||||
|
);
|
||||||
|
expect(() =>
|
||||||
|
plan([{ op: "subtitles", artifact_id: "subs" }], {
|
||||||
|
subtitleFiles: new Map([["subs", "x.srt:force_style='a'"]]),
|
||||||
|
}),
|
||||||
|
).toThrow("Unsafe subtitle file name.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("styles subtitles above the controls of tall frames", () => {
|
||||||
|
expect(subtitleStyle(1080, 1920)).toContain("MarginV=384");
|
||||||
|
expect(subtitleStyle(1080, 1920)).toContain("FontSize=65");
|
||||||
|
expect(subtitleStyle(1920, 1080)).toContain("MarginV=65");
|
||||||
|
expect(subtitleStyle(1920, 1080)).toContain("PlayResX=1920,PlayResY=1080");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts audio without decoding video work", () => {
|
||||||
|
const result = planVideoEdit({
|
||||||
|
source,
|
||||||
|
steps: [
|
||||||
|
{ op: "trim", start: 0, end: 5 },
|
||||||
|
{ op: "reframe", preset: "vertical", fit: "blur" },
|
||||||
|
{ op: "subtitles", artifact_id: "subs" },
|
||||||
|
{ op: "extract_audio" },
|
||||||
|
],
|
||||||
|
subtitleFiles: new Map([["subs", "subtitles-0.srt"]]),
|
||||||
|
outputPath: "/out.m4a",
|
||||||
|
});
|
||||||
|
expect(graphOf(result.args)).toBe(
|
||||||
|
"[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a1]",
|
||||||
|
);
|
||||||
|
expect(result.args).toEqual(expect.arrayContaining(["-map", "[a1]", "-b:a", "192k"]));
|
||||||
|
expect(result.args).not.toContain("libx264");
|
||||||
|
expect(result).toMatchObject({ kind: "audio", durationMs: 5000 });
|
||||||
|
expect(() =>
|
||||||
|
planVideoEdit({
|
||||||
|
source: { ...source, probe: probe({ hasAudio: false }) },
|
||||||
|
steps: [{ op: "extract_audio" }],
|
||||||
|
outputPath: "/out.m4a",
|
||||||
|
}),
|
||||||
|
).toThrow("holiday.mp4 has no audio.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("validates the poster time against the result", () => {
|
||||||
|
expect(
|
||||||
|
plan([
|
||||||
|
{ op: "trim", start: 0, end: 4 },
|
||||||
|
{ op: "poster", at: 3 },
|
||||||
|
]).posterAtMs,
|
||||||
|
).toBe(3000);
|
||||||
|
expect(() =>
|
||||||
|
plan([
|
||||||
|
{ op: "trim", start: 0, end: 4 },
|
||||||
|
{ op: "poster", at: 5 },
|
||||||
|
]),
|
||||||
|
).toThrow("poster at 5 s is past the end of the 4 s result.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps file paths out of the filter graph", () => {
|
||||||
|
const hostile = { ...source, path: "/jobs/1/a'b[c]:d;e,f.mp4", name: "x'];[0:v]drawtext=" };
|
||||||
|
const result = planVideoEdit({
|
||||||
|
source: hostile,
|
||||||
|
steps: [{ op: "reframe", preset: "square", fit: "blur" }],
|
||||||
|
outputPath: "/jobs/1/out.mp4",
|
||||||
|
});
|
||||||
|
expect(graphOf(result.args)).not.toContain("a'b");
|
||||||
|
expect(graphOf(result.args)).not.toContain("drawtext");
|
||||||
|
expect(result.args).toContain("file:/jobs/1/a'b[c]:d;e,f.mp4");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("squares pixels and evens odd frame sizes", () => {
|
||||||
|
const anamorphic = {
|
||||||
|
...source,
|
||||||
|
probe: probe({ width: 1440, height: 1080, sampleAspectRatio: 4 / 3 }),
|
||||||
|
};
|
||||||
|
const result = planVideoEdit({
|
||||||
|
source: anamorphic,
|
||||||
|
steps: [{ op: "mute" }],
|
||||||
|
outputPath: "/o.mp4",
|
||||||
|
});
|
||||||
|
expect(graphOf(result.args)).toBe("[0:v:0]scale=1920:1080,setsar=1,format=yuv420p[v1]");
|
||||||
|
expect(result).toMatchObject({ width: 1920, height: 1080 });
|
||||||
|
const odd = { ...source, probe: probe({ width: 721, height: 405 }) };
|
||||||
|
const evened = planVideoEdit({ source: odd, steps: [{ op: "mute" }], outputPath: "/o.mp4" });
|
||||||
|
expect(graphOf(evened.args)).toBe("[0:v:0]crop=720:404:0:0,format=yuv420p[v1]");
|
||||||
|
});
|
||||||
|
});
|
||||||
571
packages/adapters/src/media/video-edit-plan.ts
Normal file
571
packages/adapters/src/media/video-edit-plan.ts
Normal file
@ -0,0 +1,571 @@
|
|||||||
|
import type { EditVideoStep, MediaFit } from "@rakazo/contracts";
|
||||||
|
import { MEDIA_PRESETS } from "@rakazo/contracts";
|
||||||
|
import type { MediaProbe } from "./ffmpeg.js";
|
||||||
|
import { ffmpegInputArgs, ffmpegSeconds } from "./ffmpeg.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Turns an ordered list of edit steps into one ffmpeg invocation with a single filter_complex
|
||||||
|
* pass. Pure: the caller probes the inputs and runs the arguments.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface PlanInput {
|
||||||
|
path: string;
|
||||||
|
mimeType: string;
|
||||||
|
name: string;
|
||||||
|
probe: MediaProbe;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VideoEditPlanRequest {
|
||||||
|
source: PlanInput;
|
||||||
|
steps: readonly EditVideoStep[];
|
||||||
|
/** Videos named by concat steps, by artifact id. */
|
||||||
|
clips?: ReadonlyMap<string, PlanInput>;
|
||||||
|
/** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */
|
||||||
|
subtitleFiles?: ReadonlyMap<string, string>;
|
||||||
|
outputPath: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VideoEditPlan {
|
||||||
|
args: string[];
|
||||||
|
kind: "video" | "audio";
|
||||||
|
durationMs: number;
|
||||||
|
width?: number;
|
||||||
|
height?: number;
|
||||||
|
posterAtMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const VIDEO_ENCODE_ARGS = [
|
||||||
|
"-c:v",
|
||||||
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"veryfast",
|
||||||
|
"-crf",
|
||||||
|
"21",
|
||||||
|
"-pix_fmt",
|
||||||
|
"yuv420p",
|
||||||
|
"-movflags",
|
||||||
|
"+faststart",
|
||||||
|
];
|
||||||
|
const AUDIO_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "160k"];
|
||||||
|
const AUDIO_ONLY_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"];
|
||||||
|
/** Location and other camera metadata stay private; chapters do not survive cuts anyway. */
|
||||||
|
const STRIP_METADATA_ARGS = ["-map_metadata", "-1", "-map_chapters", "-1"];
|
||||||
|
const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo";
|
||||||
|
const SAFE_FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
||||||
|
const MIN_SEGMENT_SECONDS = 0.1;
|
||||||
|
const FALLBACK_FPS = 30;
|
||||||
|
|
||||||
|
interface Track {
|
||||||
|
label: string;
|
||||||
|
filters: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Timeline {
|
||||||
|
/** Null when the result is audio only: video work is skipped. */
|
||||||
|
video: Track | null;
|
||||||
|
audio: Track | null;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
fps: number;
|
||||||
|
seconds: number;
|
||||||
|
/** How appended clips fill the frame: the last reframe's fit, otherwise bars. */
|
||||||
|
fit: MediaFit;
|
||||||
|
}
|
||||||
|
|
||||||
|
class FilterGraph {
|
||||||
|
private readonly chains: string[] = [];
|
||||||
|
private count = 0;
|
||||||
|
|
||||||
|
label(kind: "v" | "a"): string {
|
||||||
|
this.count += 1;
|
||||||
|
return `${kind}${this.count}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
add(inputs: readonly string[], filters: readonly string[], outputs: readonly string[]) {
|
||||||
|
const pads = (labels: readonly string[]) => labels.map((label) => `[${label}]`).join("");
|
||||||
|
this.chains.push(`${pads(inputs)}${filters.join(",")}${pads(outputs)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Ends a track's pending filters in a labelled pad other chains can consume. */
|
||||||
|
flush(track: Track, kind: "v" | "a"): string {
|
||||||
|
if (track.filters.length === 0) return track.label;
|
||||||
|
const output = this.label(kind);
|
||||||
|
this.add([track.label], track.filters, [output]);
|
||||||
|
track.label = output;
|
||||||
|
track.filters = [];
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
toString(): string {
|
||||||
|
return this.chains.join(";");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan {
|
||||||
|
const { source } = request;
|
||||||
|
const video = source.probe.video;
|
||||||
|
if (!video) throw new Error(`${source.name} has no video.`);
|
||||||
|
const sourceSeconds = source.probe.durationMs / 1000;
|
||||||
|
const audioOnly = request.steps.at(-1)?.op === "extract_audio";
|
||||||
|
const graph = new FilterGraph();
|
||||||
|
const inputs: string[][] = [];
|
||||||
|
const addInput = (input: PlanInput, seek?: { start: number; end: number }) => {
|
||||||
|
inputs.push(
|
||||||
|
ffmpegInputArgs(input.path, {
|
||||||
|
mimeType: input.mimeType,
|
||||||
|
...(seek ? { startSeconds: seek.start, durationSeconds: seek.end - seek.start } : {}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return inputs.length - 1;
|
||||||
|
};
|
||||||
|
|
||||||
|
let steps = request.steps;
|
||||||
|
// A leading cut seeks the input instead of decoding everything before it.
|
||||||
|
const first = steps[0];
|
||||||
|
let segments: Segment[] | undefined;
|
||||||
|
if (first?.op === "trim" || first?.op === "keep") {
|
||||||
|
segments = normalizeSegments(
|
||||||
|
first.op === "trim" ? [first] : first.segments,
|
||||||
|
sourceSeconds,
|
||||||
|
first.op,
|
||||||
|
);
|
||||||
|
steps = steps.slice(1);
|
||||||
|
}
|
||||||
|
const seek = segments && { start: segments[0]!.start, end: segments.at(-1)!.end };
|
||||||
|
const index = addInput(source, seek);
|
||||||
|
const size = displaySize(video);
|
||||||
|
const timeline: Timeline = {
|
||||||
|
video: audioOnly ? null : { label: `${index}:v:0`, filters: sourceFilters(video) },
|
||||||
|
audio: source.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null,
|
||||||
|
width: size.width,
|
||||||
|
height: size.height,
|
||||||
|
fps: video.fps,
|
||||||
|
seconds: seek ? seek.end - seek.start : sourceSeconds,
|
||||||
|
fit: "pad",
|
||||||
|
};
|
||||||
|
if (segments && seek && segments.length > 1) {
|
||||||
|
keepSegments(
|
||||||
|
graph,
|
||||||
|
timeline,
|
||||||
|
segments.map((segment) => ({
|
||||||
|
start: segment.start - seek.start,
|
||||||
|
end: segment.end - seek.start,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let posterAt: number | undefined;
|
||||||
|
for (const step of steps) {
|
||||||
|
switch (step.op) {
|
||||||
|
case "trim":
|
||||||
|
case "keep":
|
||||||
|
keepSegments(
|
||||||
|
graph,
|
||||||
|
timeline,
|
||||||
|
normalizeSegments(step.op === "trim" ? [step] : step.segments, timeline.seconds, step.op),
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
case "concat":
|
||||||
|
appendClips(
|
||||||
|
graph,
|
||||||
|
timeline,
|
||||||
|
step.artifact_ids.map((id) => {
|
||||||
|
const clip = request.clips?.get(id);
|
||||||
|
if (!clip) throw new Error(`Video ${id} is not available.`);
|
||||||
|
return { clip, index: addInput(clip) };
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
case "reframe": {
|
||||||
|
const preset = MEDIA_PRESETS[step.preset];
|
||||||
|
if (timeline.video) {
|
||||||
|
timeline.video = fitFrame(
|
||||||
|
graph,
|
||||||
|
timeline.video,
|
||||||
|
timeline,
|
||||||
|
{ width: preset.width, height: preset.height },
|
||||||
|
step.fit,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
timeline.width = preset.width;
|
||||||
|
timeline.height = preset.height;
|
||||||
|
timeline.fit = step.fit;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "speed":
|
||||||
|
timeline.video?.filters.push(
|
||||||
|
`setpts=PTS/${number(step.factor)}`,
|
||||||
|
...(timeline.fps ? [`fps=${number(timeline.fps)}`] : []),
|
||||||
|
);
|
||||||
|
timeline.audio?.filters.push(...atempoFilters(step.factor));
|
||||||
|
timeline.seconds /= step.factor;
|
||||||
|
break;
|
||||||
|
case "volume":
|
||||||
|
timeline.audio?.filters.push(`volume=${number(step.gain)}`);
|
||||||
|
break;
|
||||||
|
case "mute":
|
||||||
|
timeline.audio = null;
|
||||||
|
break;
|
||||||
|
case "subtitles": {
|
||||||
|
const file = request.subtitleFiles?.get(step.artifact_id);
|
||||||
|
if (!file) throw new Error(`Subtitles ${step.artifact_id} are not available.`);
|
||||||
|
// Only generated names reach the filter graph, so nothing here needs escaping.
|
||||||
|
if (!SAFE_FILE_NAME.test(file)) throw new Error("Unsafe subtitle file name.");
|
||||||
|
timeline.video?.filters.push(
|
||||||
|
`subtitles=filename=${file}:force_style='${subtitleStyle(timeline.width, timeline.height)}'`,
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "extract_audio":
|
||||||
|
break;
|
||||||
|
case "poster":
|
||||||
|
posterAt = step.at;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (posterAt !== undefined && posterAt >= timeline.seconds) {
|
||||||
|
throw new Error(
|
||||||
|
`poster at ${seconds(posterAt)} s is past the end of the ${seconds(timeline.seconds)} s result.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const durationMs = Math.round(timeline.seconds * 1000);
|
||||||
|
const output = `file:${request.outputPath}`;
|
||||||
|
if (audioOnly || !timeline.video) {
|
||||||
|
if (!timeline.audio) throw new Error(`${source.name} has no audio.`);
|
||||||
|
timeline.audio.filters.push(AUDIO_FORMAT);
|
||||||
|
const audio = graph.flush(timeline.audio, "a");
|
||||||
|
return {
|
||||||
|
args: [
|
||||||
|
...inputs.flat(),
|
||||||
|
"-filter_complex",
|
||||||
|
graph.toString(),
|
||||||
|
"-map",
|
||||||
|
`[${audio}]`,
|
||||||
|
...AUDIO_ONLY_ENCODE_ARGS,
|
||||||
|
...STRIP_METADATA_ARGS,
|
||||||
|
"-f",
|
||||||
|
"mp4",
|
||||||
|
output,
|
||||||
|
],
|
||||||
|
kind: "audio",
|
||||||
|
durationMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
timeline.video.filters.push("format=yuv420p");
|
||||||
|
const videoOut = graph.flush(timeline.video, "v");
|
||||||
|
let audioOut: string | undefined;
|
||||||
|
if (timeline.audio) {
|
||||||
|
timeline.audio.filters.push(AUDIO_FORMAT);
|
||||||
|
audioOut = graph.flush(timeline.audio, "a");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
args: [
|
||||||
|
...inputs.flat(),
|
||||||
|
"-filter_complex",
|
||||||
|
graph.toString(),
|
||||||
|
"-map",
|
||||||
|
`[${videoOut}]`,
|
||||||
|
...(audioOut ? ["-map", `[${audioOut}]`, ...AUDIO_ENCODE_ARGS] : []),
|
||||||
|
...VIDEO_ENCODE_ARGS,
|
||||||
|
...STRIP_METADATA_ARGS,
|
||||||
|
"-f",
|
||||||
|
"mp4",
|
||||||
|
output,
|
||||||
|
],
|
||||||
|
kind: "video",
|
||||||
|
durationMs,
|
||||||
|
width: timeline.width,
|
||||||
|
height: timeline.height,
|
||||||
|
...(posterAt !== undefined ? { posterAtMs: Math.round(posterAt * 1000) } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Segment {
|
||||||
|
start: number;
|
||||||
|
end: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Validates ranges against the timeline, clamps ends to it, sorts and merges overlaps. */
|
||||||
|
function normalizeSegments(
|
||||||
|
ranges: readonly Segment[],
|
||||||
|
duration: number,
|
||||||
|
op: "trim" | "keep",
|
||||||
|
): Segment[] {
|
||||||
|
const clamped = ranges.map((range) => {
|
||||||
|
if (range.start >= duration) {
|
||||||
|
throw new Error(
|
||||||
|
`${op} ${seconds(range.start)}–${seconds(range.end)} s is outside the ${seconds(duration)} s video.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const end = Math.min(range.end, duration);
|
||||||
|
if (end - range.start < MIN_SEGMENT_SECONDS) {
|
||||||
|
throw new Error(
|
||||||
|
`${op} ${seconds(range.start)}–${seconds(end)} s is shorter than ${MIN_SEGMENT_SECONDS} s.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return { start: range.start, end };
|
||||||
|
});
|
||||||
|
clamped.sort((a, b) => a.start - b.start);
|
||||||
|
const merged: Segment[] = [];
|
||||||
|
for (const segment of clamped) {
|
||||||
|
const last = merged.at(-1);
|
||||||
|
if (last && segment.start <= last.end) last.end = Math.max(last.end, segment.end);
|
||||||
|
else merged.push({ ...segment });
|
||||||
|
}
|
||||||
|
return merged;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Keeps sorted, disjoint segments of the timeline: one trim, or split + trims + concat. */
|
||||||
|
function keepSegments(graph: FilterGraph, timeline: Timeline, segments: readonly Segment[]) {
|
||||||
|
const [only] = segments;
|
||||||
|
if (segments.length === 1 && only) {
|
||||||
|
const range = `start=${ffmpegSeconds(only.start)}:end=${ffmpegSeconds(only.end)}`;
|
||||||
|
timeline.video?.filters.push(`trim=${range}`, "setpts=PTS-STARTPTS");
|
||||||
|
timeline.audio?.filters.push(`atrim=${range}`, "asetpts=PTS-STARTPTS");
|
||||||
|
timeline.seconds = only.end - only.start;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const count = segments.length;
|
||||||
|
const branch = (track: Track | null, kind: "v" | "a"): string[] => {
|
||||||
|
if (!track) return [];
|
||||||
|
const input = graph.flush(track, kind);
|
||||||
|
const splits = segments.map(() => graph.label(kind));
|
||||||
|
graph.add([input], [`${kind === "v" ? "split" : "asplit"}=${count}`], splits);
|
||||||
|
return segments.map((segment, index) => {
|
||||||
|
const output = graph.label(kind);
|
||||||
|
const range = `start=${ffmpegSeconds(segment.start)}:end=${ffmpegSeconds(segment.end)}`;
|
||||||
|
graph.add(
|
||||||
|
[splits[index]!],
|
||||||
|
kind === "v"
|
||||||
|
? [`trim=${range}`, "setpts=PTS-STARTPTS"]
|
||||||
|
: [`atrim=${range}`, "asetpts=PTS-STARTPTS"],
|
||||||
|
[output],
|
||||||
|
);
|
||||||
|
return output;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
const videoParts = branch(timeline.video, "v");
|
||||||
|
const audioParts = branch(timeline.audio, "a");
|
||||||
|
concatParts(graph, timeline, videoParts, audioParts, count);
|
||||||
|
timeline.seconds = segments.reduce((total, segment) => total + segment.end - segment.start, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Appends clips normalized to the timeline: frame size, fit, fps and 48 kHz stereo audio. */
|
||||||
|
function appendClips(
|
||||||
|
graph: FilterGraph,
|
||||||
|
timeline: Timeline,
|
||||||
|
clips: readonly { clip: PlanInput; index: number }[],
|
||||||
|
) {
|
||||||
|
const fps = timeline.fps || FALLBACK_FPS;
|
||||||
|
const needsAudio = timeline.audio !== null || clips.some(({ clip }) => clip.probe.hasAudio);
|
||||||
|
const videoParts: string[] = [];
|
||||||
|
const audioParts: string[] = [];
|
||||||
|
if (timeline.video) {
|
||||||
|
timeline.video.filters.push("setsar=1", "format=yuv420p");
|
||||||
|
videoParts.push(graph.flush(timeline.video, "v"));
|
||||||
|
}
|
||||||
|
if (needsAudio) audioParts.push(audioPart(graph, timeline.audio, timeline.seconds));
|
||||||
|
let seconds = timeline.seconds;
|
||||||
|
for (const { clip, index } of clips) {
|
||||||
|
const video = clip.probe.video;
|
||||||
|
if (!video) throw new Error(`${clip.name} has no video.`);
|
||||||
|
const clipSeconds = clip.probe.durationMs / 1000;
|
||||||
|
if (clipSeconds <= 0) throw new Error(`Could not read the length of ${clip.name}.`);
|
||||||
|
if (timeline.video) {
|
||||||
|
const track = fitFrame(
|
||||||
|
graph,
|
||||||
|
{ label: `${index}:v:0`, filters: sourceFilters(video) },
|
||||||
|
displaySize(video),
|
||||||
|
timeline,
|
||||||
|
timeline.fit,
|
||||||
|
);
|
||||||
|
track.filters.push(`fps=${number(fps)}`, "format=yuv420p");
|
||||||
|
videoParts.push(graph.flush(track, "v"));
|
||||||
|
}
|
||||||
|
if (needsAudio) {
|
||||||
|
const track = clip.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null;
|
||||||
|
audioParts.push(audioPart(graph, track, clipSeconds));
|
||||||
|
}
|
||||||
|
seconds += clipSeconds;
|
||||||
|
}
|
||||||
|
concatParts(graph, timeline, videoParts, audioParts, clips.length + 1);
|
||||||
|
timeline.seconds = seconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A clip's audio in the common format, or silence as long as the clip when it has none. */
|
||||||
|
function audioPart(graph: FilterGraph, track: Track | null, seconds: number): string {
|
||||||
|
if (track) {
|
||||||
|
track.filters.push(AUDIO_FORMAT);
|
||||||
|
return graph.flush(track, "a");
|
||||||
|
}
|
||||||
|
const output = graph.label("a");
|
||||||
|
graph.add(
|
||||||
|
[],
|
||||||
|
[
|
||||||
|
"anullsrc=channel_layout=stereo:sample_rate=48000",
|
||||||
|
`atrim=end=${ffmpegSeconds(seconds)}`,
|
||||||
|
AUDIO_FORMAT,
|
||||||
|
],
|
||||||
|
[output],
|
||||||
|
);
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
function concatParts(
|
||||||
|
graph: FilterGraph,
|
||||||
|
timeline: Timeline,
|
||||||
|
videoParts: readonly string[],
|
||||||
|
audioParts: readonly string[],
|
||||||
|
count: number,
|
||||||
|
) {
|
||||||
|
const inputs = Array.from({ length: count }, (_, index) => [
|
||||||
|
...(videoParts[index] ? [videoParts[index]] : []),
|
||||||
|
...(audioParts[index] ? [audioParts[index]] : []),
|
||||||
|
]).flat();
|
||||||
|
const video = videoParts.length > 0 ? graph.label("v") : undefined;
|
||||||
|
const audio = audioParts.length > 0 ? graph.label("a") : undefined;
|
||||||
|
graph.add(
|
||||||
|
inputs,
|
||||||
|
[`concat=n=${count}:v=${video ? 1 : 0}:a=${audio ? 1 : 0}`],
|
||||||
|
[...(video ? [video] : []), ...(audio ? [audio] : [])],
|
||||||
|
);
|
||||||
|
timeline.video = video ? { label: video, filters: [] } : null;
|
||||||
|
timeline.audio = audio ? { label: audio, filters: [] } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Size {
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Scales a frame of size `from` into `to`: center crop, bars, or a blurred copy behind it. */
|
||||||
|
function fitFrame(graph: FilterGraph, track: Track, from: Size, to: Size, fit: MediaFit): Track {
|
||||||
|
const { width, height } = to;
|
||||||
|
if (Math.abs(from.width / from.height - width / height) < 0.01) {
|
||||||
|
track.filters.push(`scale=${width}:${height}`, "setsar=1");
|
||||||
|
return track;
|
||||||
|
}
|
||||||
|
const contain = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`;
|
||||||
|
if (fit === "crop") {
|
||||||
|
track.filters.push(
|
||||||
|
`scale=${width}:${height}:force_original_aspect_ratio=increase`,
|
||||||
|
`crop=${width}:${height}`,
|
||||||
|
"setsar=1",
|
||||||
|
);
|
||||||
|
return track;
|
||||||
|
}
|
||||||
|
if (fit === "pad") {
|
||||||
|
track.filters.push(
|
||||||
|
contain,
|
||||||
|
`pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`,
|
||||||
|
"setsar=1",
|
||||||
|
);
|
||||||
|
return track;
|
||||||
|
}
|
||||||
|
// Blur a small copy and scale it up: as smooth as a large blur at a fraction of the cost.
|
||||||
|
const input = graph.flush(track, "v");
|
||||||
|
const [background, foreground, blurred, contained, output] = [
|
||||||
|
graph.label("v"),
|
||||||
|
graph.label("v"),
|
||||||
|
graph.label("v"),
|
||||||
|
graph.label("v"),
|
||||||
|
graph.label("v"),
|
||||||
|
];
|
||||||
|
const smallWidth = even(width / 4);
|
||||||
|
const smallHeight = even(height / 4);
|
||||||
|
const radius = Math.max(2, Math.floor(Math.min(smallWidth, smallHeight) / 10));
|
||||||
|
graph.add([input], ["split=2"], [background, foreground]);
|
||||||
|
graph.add(
|
||||||
|
[background],
|
||||||
|
[
|
||||||
|
`scale=${smallWidth}:${smallHeight}:force_original_aspect_ratio=increase`,
|
||||||
|
`crop=${smallWidth}:${smallHeight}`,
|
||||||
|
`boxblur=${radius}:2`,
|
||||||
|
`scale=${width}:${height}`,
|
||||||
|
"setsar=1",
|
||||||
|
],
|
||||||
|
[blurred],
|
||||||
|
);
|
||||||
|
graph.add([foreground], [contain, "setsar=1"], [contained]);
|
||||||
|
graph.add([blurred, contained], ["overlay=(W-w)/2:(H-h)/2"], [output]);
|
||||||
|
return { label: output, filters: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Square pixels and even dimensions, which yuv420p H.264 requires. */
|
||||||
|
function sourceFilters(video: NonNullable<MediaProbe["video"]>): string[] {
|
||||||
|
const size = displaySize(video);
|
||||||
|
if (Math.abs(video.sampleAspectRatio - 1) > 0.01) {
|
||||||
|
return [`scale=${size.width}:${size.height}`, "setsar=1"];
|
||||||
|
}
|
||||||
|
if (size.width !== video.width || size.height !== video.height) {
|
||||||
|
return [`crop=${size.width}:${size.height}:0:0`];
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
function displaySize(video: NonNullable<MediaProbe["video"]>): Size {
|
||||||
|
if (Math.abs(video.sampleAspectRatio - 1) > 0.01) {
|
||||||
|
return {
|
||||||
|
width: even(Math.round(video.width * video.sampleAspectRatio)),
|
||||||
|
height: even(video.height),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { width: even(video.width), height: even(video.height) };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** atempo takes 0.5–2 per instance on older ffmpeg builds, so larger changes chain it. */
|
||||||
|
export function atempoFilters(factor: number): string[] {
|
||||||
|
const filters: string[] = [];
|
||||||
|
let rest = factor;
|
||||||
|
while (rest > 2) {
|
||||||
|
filters.push("atempo=2");
|
||||||
|
rest /= 2;
|
||||||
|
}
|
||||||
|
while (rest < 0.5) {
|
||||||
|
filters.push("atempo=0.5");
|
||||||
|
rest /= 0.5;
|
||||||
|
}
|
||||||
|
filters.push(`atempo=${number(rest)}`);
|
||||||
|
return filters;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Readable burned-in subtitles: bold white text with a dark outline, sized to the frame, and
|
||||||
|
* lifted above the bottom fifth of tall frames where 9:16 apps place their own controls.
|
||||||
|
*/
|
||||||
|
export function subtitleStyle(width: number, height: number): string {
|
||||||
|
const tall = height >= width * 1.5;
|
||||||
|
const fontSize = Math.round(Math.min(width, height) * 0.06);
|
||||||
|
const marginX = Math.round(width * (tall ? 0.1 : 0.05));
|
||||||
|
const marginY = Math.round(height * (tall ? 0.2 : 0.06));
|
||||||
|
return [
|
||||||
|
`PlayResX=${width}`,
|
||||||
|
`PlayResY=${height}`,
|
||||||
|
"FontName=DejaVu Sans",
|
||||||
|
`FontSize=${fontSize}`,
|
||||||
|
"Bold=1",
|
||||||
|
"PrimaryColour=&H00FFFFFF",
|
||||||
|
"OutlineColour=&H00000000",
|
||||||
|
"BackColour=&H80000000",
|
||||||
|
"BorderStyle=1",
|
||||||
|
`Outline=${Math.max(2, Math.round(fontSize / 16))}`,
|
||||||
|
"Shadow=1",
|
||||||
|
"Alignment=2",
|
||||||
|
`MarginL=${marginX}`,
|
||||||
|
`MarginR=${marginX}`,
|
||||||
|
`MarginV=${marginY}`,
|
||||||
|
].join(",");
|
||||||
|
}
|
||||||
|
|
||||||
|
function even(value: number): number {
|
||||||
|
return Math.max(2, 2 * Math.floor(value / 2));
|
||||||
|
}
|
||||||
|
|
||||||
|
function number(value: number): string {
|
||||||
|
return String(Number(value.toFixed(6)));
|
||||||
|
}
|
||||||
|
|
||||||
|
function seconds(value: number): string {
|
||||||
|
return String(Number(value.toFixed(2)));
|
||||||
|
}
|
||||||
331
packages/adapters/src/media/video-operations.ts
Normal file
331
packages/adapters/src/media/video-operations.ts
Normal file
@ -0,0 +1,331 @@
|
|||||||
|
import { copyFile, mkdtemp, open, stat } from "node:fs/promises";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import type {
|
||||||
|
MediaJobContext,
|
||||||
|
MediaJobInputFile,
|
||||||
|
MediaOperationHandler,
|
||||||
|
MediaOperationResult,
|
||||||
|
} from "@rakazo/adapter-kit";
|
||||||
|
import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts";
|
||||||
|
import {
|
||||||
|
ARTIFACT_MAX_BYTES_BY_KIND,
|
||||||
|
EditVideoInputSchema,
|
||||||
|
ExportVideoInputSchema,
|
||||||
|
MEDIA_PRESETS,
|
||||||
|
} from "@rakazo/contracts";
|
||||||
|
import type { FfmpegInfo } from "./ffmpeg.js";
|
||||||
|
import {
|
||||||
|
detectFfmpeg,
|
||||||
|
outputTooLargeMessage,
|
||||||
|
probeMedia,
|
||||||
|
renderPoster,
|
||||||
|
runFfmpeg,
|
||||||
|
} from "./ffmpeg.js";
|
||||||
|
import type { PlanInput, VideoEditPlan } from "./video-edit-plan.js";
|
||||||
|
import { planVideoEdit } from "./video-edit-plan.js";
|
||||||
|
|
||||||
|
export interface VideoOperationsDeps {
|
||||||
|
/** Finds ffmpeg; defaults to PATH lookup. */
|
||||||
|
detect?: () => Promise<FfmpegInfo | null>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const PROGRESS_INTERVAL_MS = 1_000;
|
||||||
|
/** Share of each output's progress spent encoding; the rest is probing and the poster. */
|
||||||
|
const ENCODE_SHARE = 0.95;
|
||||||
|
const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024;
|
||||||
|
|
||||||
|
/** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */
|
||||||
|
export function createVideoOperations(deps: VideoOperationsDeps = {}) {
|
||||||
|
const detect = deps.detect ?? detectFfmpeg;
|
||||||
|
const available = async () => (await detect()) !== null;
|
||||||
|
const tools = async () => {
|
||||||
|
const info = await detect();
|
||||||
|
if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server.");
|
||||||
|
return info;
|
||||||
|
};
|
||||||
|
const editVideo: MediaOperationHandler = {
|
||||||
|
operation: "edit_video",
|
||||||
|
run: async (context) => runEditVideo(context, await tools()),
|
||||||
|
};
|
||||||
|
const exportVideo: MediaOperationHandler = {
|
||||||
|
operation: "export_video",
|
||||||
|
run: async (context) => runExportVideo(context, await tools()),
|
||||||
|
};
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
operation: "edit_video" as const,
|
||||||
|
inputSchema: EditVideoInputSchema,
|
||||||
|
title: (input: EditVideoInput) => input.title,
|
||||||
|
available,
|
||||||
|
handler: editVideo,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
operation: "export_video" as const,
|
||||||
|
inputSchema: ExportVideoInputSchema,
|
||||||
|
title: (input: ExportVideoInput) =>
|
||||||
|
input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "),
|
||||||
|
available,
|
||||||
|
handler: exportVideo,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runEditVideo(
|
||||||
|
context: MediaJobContext,
|
||||||
|
tools: FfmpegInfo,
|
||||||
|
): Promise<MediaOperationResult> {
|
||||||
|
const input = EditVideoInputSchema.parse(context.input);
|
||||||
|
const workDir = await mkdtemp(join(context.tempDir, "edit-"));
|
||||||
|
const progress = createProgressReporter(context);
|
||||||
|
const source = await loadVideo(context, tools, input.artifact_id);
|
||||||
|
const name = cleanFileName(input.title, cleanFileName(withoutExtension(source.name), "video"));
|
||||||
|
|
||||||
|
if (input.steps.every((step) => step.op === "poster")) {
|
||||||
|
const at = input.steps[0]?.op === "poster" ? input.steps[0].at : 0;
|
||||||
|
if (at * 1000 >= source.probe.durationMs) {
|
||||||
|
throw new Error(
|
||||||
|
`poster at ${at} s is past the end of the ${source.probe.durationMs / 1000} s video.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const posterPath = join(workDir, "poster.jpg");
|
||||||
|
await renderPoster(source.path, posterPath, {
|
||||||
|
signal: context.signal,
|
||||||
|
mimeType: source.mimeType,
|
||||||
|
atMs: at * 1000,
|
||||||
|
});
|
||||||
|
const output = await context.addOutput({
|
||||||
|
path: posterPath,
|
||||||
|
name: `${name}.jpg`,
|
||||||
|
mimeType: "image/jpeg",
|
||||||
|
width: source.probe.video?.width,
|
||||||
|
height: source.probe.video?.height,
|
||||||
|
parentArtifactId: source.artifactId,
|
||||||
|
});
|
||||||
|
await progress.done();
|
||||||
|
return { summary: `Saved the frame at ${at} s of ${source.name} as ${describe(output)}.` };
|
||||||
|
}
|
||||||
|
|
||||||
|
const clips = new Map<string, PlanInput>();
|
||||||
|
const subtitleFiles = new Map<string, string>();
|
||||||
|
for (const step of input.steps) {
|
||||||
|
if (step.op === "concat") {
|
||||||
|
for (const id of step.artifact_ids) {
|
||||||
|
if (!clips.has(id)) clips.set(id, await loadVideo(context, tools, id));
|
||||||
|
}
|
||||||
|
} else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) {
|
||||||
|
subtitleFiles.set(
|
||||||
|
step.artifact_id,
|
||||||
|
await prepareSubtitles(context, step.artifact_id, workDir, subtitleFiles.size),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const audioOnly = input.steps.at(-1)?.op === "extract_audio";
|
||||||
|
const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4");
|
||||||
|
const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath });
|
||||||
|
const output = await encode(context, tools, plan, outputPath, workDir, progress.range(0, 1), {
|
||||||
|
name: `${name}${audioOnly ? ".m4a" : ".mp4"}`,
|
||||||
|
parentArtifactId: source.artifactId,
|
||||||
|
});
|
||||||
|
await progress.done();
|
||||||
|
return {
|
||||||
|
summary: `Edited ${source.name} into ${describe(output)}.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runExportVideo(
|
||||||
|
context: MediaJobContext,
|
||||||
|
tools: FfmpegInfo,
|
||||||
|
): Promise<MediaOperationResult> {
|
||||||
|
const input = ExportVideoInputSchema.parse(context.input);
|
||||||
|
const presets = [...new Set(input.presets)];
|
||||||
|
const workDir = await mkdtemp(join(context.tempDir, "export-"));
|
||||||
|
const progress = createProgressReporter(context);
|
||||||
|
const source = await loadVideo(context, tools, input.artifact_id);
|
||||||
|
const base = cleanFileName(withoutExtension(source.name), "video");
|
||||||
|
const outputs: MediaOutput[] = [];
|
||||||
|
for (const [index, preset] of presets.entries()) {
|
||||||
|
const outputPath = join(workDir, `${preset}.mp4`);
|
||||||
|
const plan = planVideoEdit({
|
||||||
|
source,
|
||||||
|
steps: [{ op: "reframe", preset, fit: input.fit }],
|
||||||
|
outputPath,
|
||||||
|
});
|
||||||
|
const share = 1 / presets.length;
|
||||||
|
outputs.push(
|
||||||
|
await encode(
|
||||||
|
context,
|
||||||
|
tools,
|
||||||
|
plan,
|
||||||
|
outputPath,
|
||||||
|
workDir,
|
||||||
|
progress.range(index * share, (index + 1) * share),
|
||||||
|
{
|
||||||
|
name: `${base}-${MEDIA_PRESETS[preset].aspect.replace(":", "x")}.mp4`,
|
||||||
|
parentArtifactId: source.artifactId,
|
||||||
|
preset,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await progress.done();
|
||||||
|
return { summary: `Exported ${source.name} as ${outputs.map(describe).join(", ")}.` };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Runs a plan, checks the result, renders its poster and stores it as an output. */
|
||||||
|
async function encode(
|
||||||
|
context: MediaJobContext,
|
||||||
|
tools: FfmpegInfo,
|
||||||
|
plan: VideoEditPlan,
|
||||||
|
outputPath: string,
|
||||||
|
workDir: string,
|
||||||
|
report: (fraction: number) => void,
|
||||||
|
file: Pick<Parameters<MediaJobContext["addOutput"]>[0], "name" | "parentArtifactId" | "preset">,
|
||||||
|
): Promise<MediaOutput> {
|
||||||
|
const maxBytes = ARTIFACT_MAX_BYTES_BY_KIND[plan.kind];
|
||||||
|
await runFfmpeg(plan.args, {
|
||||||
|
signal: context.signal,
|
||||||
|
cwd: workDir,
|
||||||
|
durationMs: plan.durationMs,
|
||||||
|
onProgress: (fraction) => report(fraction * ENCODE_SHARE),
|
||||||
|
maxOutputBytes: maxBytes,
|
||||||
|
outputs: [outputPath],
|
||||||
|
ffmpegPath: tools.ffmpegPath,
|
||||||
|
});
|
||||||
|
if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes));
|
||||||
|
const probe = await probeMedia(outputPath, {
|
||||||
|
signal: context.signal,
|
||||||
|
mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4",
|
||||||
|
ffprobePath: tools.ffprobePath,
|
||||||
|
});
|
||||||
|
if (plan.kind === "audio") {
|
||||||
|
const output = await context.addOutput({
|
||||||
|
...file,
|
||||||
|
path: outputPath,
|
||||||
|
mimeType: "audio/mp4",
|
||||||
|
durationMs: probe.durationMs,
|
||||||
|
});
|
||||||
|
report(1);
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
const posterPath = outputPath.replace(/\.mp4$/, ".jpg");
|
||||||
|
await renderPoster(outputPath, posterPath, {
|
||||||
|
signal: context.signal,
|
||||||
|
mimeType: "video/mp4",
|
||||||
|
atMs: plan.posterAtMs ?? Math.min(1_000, probe.durationMs / 4),
|
||||||
|
});
|
||||||
|
const output = await context.addOutput({
|
||||||
|
...file,
|
||||||
|
path: outputPath,
|
||||||
|
mimeType: "video/mp4",
|
||||||
|
width: probe.video?.width ?? plan.width,
|
||||||
|
height: probe.video?.height ?? plan.height,
|
||||||
|
durationMs: probe.durationMs,
|
||||||
|
posterPath,
|
||||||
|
});
|
||||||
|
report(1);
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadVideo(
|
||||||
|
context: MediaJobContext,
|
||||||
|
tools: FfmpegInfo,
|
||||||
|
artifactId: string,
|
||||||
|
): Promise<PlanInput & MediaJobInputFile> {
|
||||||
|
const file = await context.fetchArtifact(artifactId);
|
||||||
|
if (file.kind !== "video") throw new Error(`${file.name} is not a video.`);
|
||||||
|
const probe = await probeMedia(file.path, {
|
||||||
|
signal: context.signal,
|
||||||
|
mimeType: file.mimeType,
|
||||||
|
ffprobePath: tools.ffprobePath,
|
||||||
|
});
|
||||||
|
if (!probe.video) throw new Error(`${file.name} has no video.`);
|
||||||
|
if (probe.durationMs <= 0) throw new Error(`Could not read the length of ${file.name}.`);
|
||||||
|
return { ...file, probe };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Copies an SRT/VTT artifact to a generated name, so no user text reaches the filter graph. */
|
||||||
|
async function prepareSubtitles(
|
||||||
|
context: MediaJobContext,
|
||||||
|
artifactId: string,
|
||||||
|
workDir: string,
|
||||||
|
index: number,
|
||||||
|
): Promise<string> {
|
||||||
|
const file = await context.fetchArtifact(artifactId);
|
||||||
|
const format = file.size <= SUBTITLE_MAX_BYTES ? subtitleFormat(await readHead(file.path)) : null;
|
||||||
|
if (!format) throw new Error(`${file.name} is not an SRT or VTT file.`);
|
||||||
|
const name = `subtitles-${index}.${format}`;
|
||||||
|
await copyFile(file.path, join(workDir, name));
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readHead(path: string): Promise<string> {
|
||||||
|
const handle = await open(path, "r");
|
||||||
|
try {
|
||||||
|
const buffer = Buffer.alloc(1024);
|
||||||
|
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
|
||||||
|
return buffer.subarray(0, bytesRead).toString("utf8");
|
||||||
|
} finally {
|
||||||
|
await handle.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recognizes WebVTT and SubRip by content. Anything else is refused, so ffmpeg never probes
|
||||||
|
* an arbitrary file (a playlist, say) handed in as subtitles.
|
||||||
|
*/
|
||||||
|
export function subtitleFormat(head: string): "srt" | "vtt" | null {
|
||||||
|
const text = head.replace(/^/, "").trimStart();
|
||||||
|
if (/^WEBVTT(?:[ \t\r\n]|$)/.test(text)) return "vtt";
|
||||||
|
if (/^(?:\d+[ \t]*\r?\n)?\d{1,2}:\d{2}:\d{2}[,.]\d{1,3}[ \t]*-->/.test(text)) return "srt";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A name for produced files: no path separators or control characters, at most 80 chars. */
|
||||||
|
export function cleanFileName(text: string, fallback: string): string {
|
||||||
|
const name = text
|
||||||
|
.replaceAll(/[\p{Cc}\\/:*?"<>|]+/gu, " ")
|
||||||
|
.replaceAll(/\s+/g, " ")
|
||||||
|
.replace(/^[\s.]+/, "")
|
||||||
|
.slice(0, 80)
|
||||||
|
.trim();
|
||||||
|
return name || fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function withoutExtension(name: string): string {
|
||||||
|
return name.replace(/\.[A-Za-z0-9]{1,5}$/, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function describe(output: MediaOutput): string {
|
||||||
|
const size = output.width && output.height ? ` ${output.width}×${output.height}` : "";
|
||||||
|
const length =
|
||||||
|
output.durationMs !== undefined ? ` ${Number((output.durationMs / 1000).toFixed(1))} s` : "";
|
||||||
|
return `${output.name}${size}${length} (artifact ${output.artifactId})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maps ffmpeg progress onto the card: monotonic, at most once a second, written in order and
|
||||||
|
* never failing the job when a write fails.
|
||||||
|
*/
|
||||||
|
function createProgressReporter(context: MediaJobContext) {
|
||||||
|
let reported = 0;
|
||||||
|
let reportedAt = 0;
|
||||||
|
let writes = Promise.resolve();
|
||||||
|
const write = (fraction: number) => {
|
||||||
|
reported = fraction;
|
||||||
|
reportedAt = Date.now();
|
||||||
|
writes = writes.then(() => context.progress(fraction)).catch(() => undefined);
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
range: (from: number, to: number) => (fraction: number) => {
|
||||||
|
const value = Math.min(1, from + (to - from) * Math.min(1, Math.max(0, fraction)));
|
||||||
|
const boundary = fraction >= 1;
|
||||||
|
if (value <= reported) return;
|
||||||
|
if (!boundary && Date.now() - reportedAt < PROGRESS_INTERVAL_MS) return;
|
||||||
|
write(value);
|
||||||
|
},
|
||||||
|
done: async () => {
|
||||||
|
if (reported < 1) write(1);
|
||||||
|
await writes;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@ -10,6 +10,7 @@ export * from "./integration-settings.js";
|
|||||||
export * from "./local-settings.js";
|
export * from "./local-settings.js";
|
||||||
export * from "./mcp.js";
|
export * from "./mcp.js";
|
||||||
export * from "./media.js";
|
export * from "./media.js";
|
||||||
|
export * from "./media-video.js";
|
||||||
export * from "./model-tiers.js";
|
export * from "./model-tiers.js";
|
||||||
export * from "./openai-compatible-ui.js";
|
export * from "./openai-compatible-ui.js";
|
||||||
export * from "./project-goals.js";
|
export * from "./project-goals.js";
|
||||||
|
|||||||
80
packages/contracts/src/media-video.ts
Normal file
80
packages/contracts/src/media-video.ts
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
import * as z from "zod";
|
||||||
|
import { Id } from "./ids.js";
|
||||||
|
import { MediaFitSchema, MediaPresetIdSchema } from "./media.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Inputs of the `edit_video` and `export_video` media operations, which are also the arguments
|
||||||
|
* of the tools with those names. Times are seconds on the timeline left by the previous steps.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const Seconds = z.number().min(0).max(86_400);
|
||||||
|
|
||||||
|
const TimeRangeSchema = z
|
||||||
|
.object({ start: Seconds, end: Seconds })
|
||||||
|
.refine((range) => range.end > range.start, "end must be after start");
|
||||||
|
|
||||||
|
/** Written by the bot in the user's language; shown on the media card and used as file name. */
|
||||||
|
const MediaTitleSchema = z.string().trim().min(1).max(120);
|
||||||
|
|
||||||
|
export const EditVideoStepSchema = z.discriminatedUnion("op", [
|
||||||
|
z
|
||||||
|
.object({ op: z.literal("trim"), start: Seconds, end: Seconds })
|
||||||
|
.refine((step) => step.end > step.start, "end must be after start"),
|
||||||
|
/** Keeps these parts and cuts everything between them. */
|
||||||
|
z.object({ op: z.literal("keep"), segments: z.array(TimeRangeSchema).min(1).max(50) }),
|
||||||
|
/** Appends these videos, fitted to the current frame. */
|
||||||
|
z.object({ op: z.literal("concat"), artifact_ids: z.array(Id).min(1).max(20) }),
|
||||||
|
z.object({
|
||||||
|
op: z.literal("reframe"),
|
||||||
|
preset: MediaPresetIdSchema,
|
||||||
|
fit: MediaFitSchema.default("crop"),
|
||||||
|
}),
|
||||||
|
z.object({ op: z.literal("speed"), factor: z.number().min(0.25).max(4) }),
|
||||||
|
/** Linear gain: 0.5 halves the loudness, 2 doubles it. */
|
||||||
|
z.object({ op: z.literal("volume"), gain: z.number().min(0).max(8) }),
|
||||||
|
z.object({ op: z.literal("mute") }),
|
||||||
|
/** Burns in an SRT or VTT artifact. */
|
||||||
|
z.object({ op: z.literal("subtitles"), artifact_id: Id }),
|
||||||
|
/** Last step only: the result is the audio as M4A. */
|
||||||
|
z.object({ op: z.literal("extract_audio") }),
|
||||||
|
/** Cover frame at this second of the result; as the only kind of step, the result is that image. */
|
||||||
|
z.object({ op: z.literal("poster"), at: Seconds }),
|
||||||
|
]);
|
||||||
|
export type EditVideoStep = z.infer<typeof EditVideoStepSchema>;
|
||||||
|
|
||||||
|
export const EditVideoInputSchema = z
|
||||||
|
.object({
|
||||||
|
artifact_id: Id,
|
||||||
|
steps: z.array(EditVideoStepSchema).min(1).max(20),
|
||||||
|
title: MediaTitleSchema,
|
||||||
|
})
|
||||||
|
.superRefine((input, context) => {
|
||||||
|
const ops = input.steps.map((step) => step.op);
|
||||||
|
const extract = ops.indexOf("extract_audio");
|
||||||
|
if (extract >= 0 && extract !== ops.length - 1) {
|
||||||
|
context.addIssue({
|
||||||
|
code: "custom",
|
||||||
|
path: ["steps", extract],
|
||||||
|
message: "extract_audio must be the last step",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (extract >= 0 && ops.includes("poster")) {
|
||||||
|
context.addIssue({
|
||||||
|
code: "custom",
|
||||||
|
path: ["steps"],
|
||||||
|
message: "poster cannot be combined with extract_audio",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (ops.filter((op) => op === "poster").length > 1) {
|
||||||
|
context.addIssue({ code: "custom", path: ["steps"], message: "use at most one poster step" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
export type EditVideoInput = z.infer<typeof EditVideoInputSchema>;
|
||||||
|
|
||||||
|
export const ExportVideoInputSchema = z.object({
|
||||||
|
artifact_id: Id,
|
||||||
|
presets: z.array(MediaPresetIdSchema).min(1).max(6),
|
||||||
|
fit: MediaFitSchema.default("crop"),
|
||||||
|
title: MediaTitleSchema.optional(),
|
||||||
|
});
|
||||||
|
export type ExportVideoInput = z.infer<typeof ExportVideoInputSchema>;
|
||||||
Loading…
x
Reference in New Issue
Block a user