Brand videos with a logo overlay and an intro before the cut
edit_video gains an overlay step that lays a PNG, JPEG or WebP image over the video at a corner or the center, sized to a share of the frame width, with optional opacity and time range; tall frames keep the 9:16 safe areas clear. concat takes at: "start" so an intro can go before the video. The image is authorized like other inputs, read with its own demuxer and refused above 8192 pixels on a side. New Studio specialists are told to brand with the user's logo and intro. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
parent
2993013046
commit
3eb1fdd68f
@ -569,12 +569,12 @@ export const builtinAgentTools: ConnectorTool[] = [
|
||||
},
|
||||
{
|
||||
name: "edit_video",
|
||||
description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image; remove_silence {threshold_db -35, min_silence 0.6, padding 0.12} cuts pauses quieter than threshold_db dB and at least min_silence s long, keeping padding s on each side. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`,
|
||||
description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids, at "end" or "start"} adds videos at the end, or at the start for an intro, fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image; remove_silence {threshold_db -35, min_silence 0.6, padding 0.12} cuts pauses quieter than threshold_db dB and at least min_silence s long, keeping padding s on each side; overlay {artifact_id, position top_left|top_right|bottom_left|bottom_right|center, width 0.15, opacity 1, start, end} lays a PNG, JPEG or WebP image such as a logo or watermark over the video, width as a share of the frame, from start to end s or the whole time; top_right stays clear of 9:16 app buttons. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`,
|
||||
inputSchema: mediaParameters(EditVideoInputSchema),
|
||||
},
|
||||
{
|
||||
name: "export_video",
|
||||
description: `Export a video artifact as one H.264/AAC MP4 per platform and per preset, 6 at most. When the user names where it will be posted, use platforms: each file is fitted to that platform's frame, frame rate, bitrate and loudness, then checked. Platforms (longest video): ${VIDEO_PLATFORMS}. A platform the video is too long or too short for is skipped: trim with edit_video first. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`,
|
||||
description: `Export a video artifact as one H.264/AAC MP4 per platform and per preset, 6 at most. When the user names where it will be posted, use platforms: each file is fitted to that platform's frame, frame rate, bitrate and loudness, then checked. Platforms (longest video): ${VIDEO_PLATFORMS}. A platform the video is too long or too short for is skipped: trim with edit_video first. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio, subtitles or a logo use edit_video.`,
|
||||
inputSchema: mediaParameters(ExportVideoInputSchema),
|
||||
},
|
||||
{
|
||||
|
||||
@ -79,6 +79,9 @@ const DEMUXERS: Record<string, string> = {
|
||||
"audio/mpeg": "mp3",
|
||||
"audio/wav": "wav",
|
||||
"audio/ogg": "ogg",
|
||||
"image/png": "png_pipe",
|
||||
"image/jpeg": "jpeg_pipe",
|
||||
"image/webp": "webp_pipe",
|
||||
};
|
||||
|
||||
let detection: Promise<FfmpegInfo | null> | undefined;
|
||||
|
||||
@ -14,6 +14,8 @@ export interface TestClips {
|
||||
silentPortrait: string;
|
||||
srt: string;
|
||||
vtt: string;
|
||||
/** 64×32 PNG, opaque blue. */
|
||||
logo: string;
|
||||
}
|
||||
|
||||
/** Tiny generated clips; nothing binary is committed. */
|
||||
@ -40,7 +42,17 @@ export async function createTestClips(): Promise<TestClips> {
|
||||
);
|
||||
const vtt = join(dir, "captions.vtt");
|
||||
await writeFile(vtt, "WEBVTT\n\n00:00.100 --> 00:01.500\nHello from WebVTT\n");
|
||||
return { dir, landscape, silentPortrait, srt, vtt };
|
||||
const logo = join(dir, "logo.png");
|
||||
await runFfmpeg([
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=blue:size=64x32",
|
||||
"-frames:v",
|
||||
"1",
|
||||
`file:${logo}`,
|
||||
]);
|
||||
return { dir, landscape, silentPortrait, srt, vtt, logo };
|
||||
}
|
||||
|
||||
export interface TestArtifact {
|
||||
|
||||
@ -118,7 +118,7 @@ describe("planVideoEdit", () => {
|
||||
name: "clip.webm",
|
||||
probe: probe({ width: 1080, height: 1920, fps: 30, durationMs: 2000, hasAudio: false }),
|
||||
};
|
||||
const result = plan([{ op: "concat", artifact_ids: ["clip"] }], {
|
||||
const result = plan([{ op: "concat", artifact_ids: ["clip"], at: "end" }], {
|
||||
clips: new Map([["clip", clip]]),
|
||||
});
|
||||
expect(result.args).toContain("matroska");
|
||||
@ -134,11 +134,25 @@ describe("planVideoEdit", () => {
|
||||
expect(result.durationMs).toBe(12_000);
|
||||
});
|
||||
|
||||
it("puts an intro before the video", () => {
|
||||
const result = plan(
|
||||
[
|
||||
{ op: "trim", start: 0, end: 4 },
|
||||
{ op: "concat", artifact_ids: ["clip"], at: "start" },
|
||||
],
|
||||
{ clips: new Map([["clip", { ...source, probe: probe({ durationMs: 1500 }) }]]) },
|
||||
);
|
||||
const graph = graphOf(result.args);
|
||||
expect(graph).toContain("[0:v:0]setsar=1,format=yuv420p[v1]");
|
||||
expect(graph).toContain("[v3][a4][v1][a2]concat=n=2:v=1:a=1");
|
||||
expect(result.durationMs).toBe(5500);
|
||||
});
|
||||
|
||||
it("adds silence to a silent source when an appended clip has audio", () => {
|
||||
const silent = { ...source, probe: probe({ hasAudio: false }) };
|
||||
const result = planVideoEdit({
|
||||
source: silent,
|
||||
steps: [{ op: "concat", artifact_ids: ["clip"] }],
|
||||
steps: [{ op: "concat", artifact_ids: ["clip"], at: "end" }],
|
||||
clips: new Map([["clip", source]]),
|
||||
outputPath: "/out.mp4",
|
||||
});
|
||||
@ -152,7 +166,7 @@ describe("planVideoEdit", () => {
|
||||
const result = plan(
|
||||
[
|
||||
{ op: "reframe", preset: "vertical", fit: "crop" },
|
||||
{ op: "concat", artifact_ids: ["clip"] },
|
||||
{ op: "concat", artifact_ids: ["clip"], at: "end" },
|
||||
],
|
||||
{ clips: new Map([["clip", source]]) },
|
||||
);
|
||||
@ -161,6 +175,94 @@ describe("planVideoEdit", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("lays a logo over a corner with margins, its own demuxer, opacity and timing", () => {
|
||||
const logo: PlanInput = {
|
||||
path: "/jobs/1/input-logo",
|
||||
mimeType: "image/png",
|
||||
name: "logo.png",
|
||||
probe: probe({ codec: "png", width: 600, height: 200, durationMs: 40, hasAudio: false }),
|
||||
};
|
||||
const result = plan(
|
||||
[
|
||||
{
|
||||
op: "overlay",
|
||||
artifact_id: "logo",
|
||||
position: "top_right",
|
||||
width: 0.2,
|
||||
opacity: 0.5,
|
||||
start: 1,
|
||||
},
|
||||
],
|
||||
{ images: new Map([["logo", logo]]) },
|
||||
);
|
||||
expect(result.args).toEqual(
|
||||
expect.arrayContaining(["-f", "png_pipe", "-i", "file:/jobs/1/input-logo"]),
|
||||
);
|
||||
expect(graphOf(result.args)).toBe(
|
||||
[
|
||||
"[1:v:0]scale=w=384:h=994:force_original_aspect_ratio=decrease:force_divisible_by=2,format=rgba,colorchannelmixer=aa=0.5[v1]",
|
||||
"[0:v:0][v1]overlay=x=W-w-77:y=43:enable='between(t,1,10)'[v2]",
|
||||
"[v2]format=yuv420p[v3]",
|
||||
"[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a4]",
|
||||
].join(";"),
|
||||
);
|
||||
expect(result.durationMs).toBe(10_000);
|
||||
});
|
||||
|
||||
it("keeps a logo clear of 9:16 app buttons and centers one when asked", () => {
|
||||
const logo: PlanInput = { ...source, mimeType: "image/png", name: "logo.png" };
|
||||
const images = new Map([["logo", logo]]);
|
||||
const tall = graphOf(
|
||||
plan(
|
||||
[
|
||||
{ op: "reframe", preset: "vertical", fit: "crop" },
|
||||
{ op: "overlay", artifact_id: "logo", position: "bottom_left", width: 0.15, opacity: 1 },
|
||||
],
|
||||
{ images },
|
||||
).args,
|
||||
);
|
||||
expect(tall).toContain("scale=w=162:h=1248:");
|
||||
expect(tall).toContain("overlay=x=65:y=H-h-384[");
|
||||
expect(tall).not.toContain("colorchannelmixer");
|
||||
const center = graphOf(
|
||||
plan([{ op: "overlay", artifact_id: "logo", position: "center", width: 0.5, opacity: 1 }], {
|
||||
images,
|
||||
}).args,
|
||||
);
|
||||
expect(center).toContain("overlay=x=(W-w)/2:y=(H-h)/2[");
|
||||
});
|
||||
|
||||
it("ignores overlays in audio results and refuses a missing image or a late start", () => {
|
||||
const steps: EditVideoStep[] = [
|
||||
{ op: "overlay", artifact_id: "logo", position: "top_right", width: 0.15, opacity: 1 },
|
||||
{ op: "extract_audio" },
|
||||
];
|
||||
const audio = plan(steps);
|
||||
expect(audio.kind).toBe("audio");
|
||||
expect(audio.args).not.toContain("png_pipe");
|
||||
expect(() =>
|
||||
plan([
|
||||
{ op: "overlay", artifact_id: "logo", position: "top_right", width: 0.15, opacity: 1 },
|
||||
]),
|
||||
).toThrow("Image logo is not available.");
|
||||
expect(() =>
|
||||
plan(
|
||||
[
|
||||
{ op: "trim", start: 0, end: 2 },
|
||||
{
|
||||
op: "overlay",
|
||||
artifact_id: "logo",
|
||||
position: "top_right",
|
||||
width: 0.15,
|
||||
opacity: 1,
|
||||
start: 2,
|
||||
},
|
||||
],
|
||||
{ images: new Map([["logo", source]]) },
|
||||
),
|
||||
).toThrow("overlay from 2 s starts after the end of the 2 s video.");
|
||||
});
|
||||
|
||||
it("reframes by crop, pad or blur, and only scales when the aspect already matches", () => {
|
||||
expect(graphOf(plan([{ op: "reframe", preset: "square", fit: "crop" }]).args)).toContain(
|
||||
"scale=1080:1080:force_original_aspect_ratio=increase,crop=1080:1080,setsar=1",
|
||||
|
||||
@ -20,6 +20,8 @@ export interface VideoEditPlanRequest {
|
||||
steps: readonly EditVideoStep[];
|
||||
/** Videos named by concat steps, by artifact id. */
|
||||
clips?: ReadonlyMap<string, PlanInput>;
|
||||
/** Images named by overlay steps, by artifact id. */
|
||||
images?: ReadonlyMap<string, PlanInput>;
|
||||
/** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */
|
||||
subtitleFiles?: ReadonlyMap<string, string>;
|
||||
/**
|
||||
@ -191,7 +193,7 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan {
|
||||
);
|
||||
break;
|
||||
case "concat":
|
||||
appendClips(
|
||||
addClips(
|
||||
graph,
|
||||
timeline,
|
||||
step.artifact_ids.map((id) => {
|
||||
@ -199,6 +201,7 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan {
|
||||
if (!clip) throw new Error(`Video ${id} is not available.`);
|
||||
return { clip, index: addInput(clip) };
|
||||
}),
|
||||
step.at,
|
||||
);
|
||||
break;
|
||||
case "reframe": {
|
||||
@ -252,6 +255,18 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan {
|
||||
if (speech) keepSegments(graph, timeline, speech);
|
||||
break;
|
||||
}
|
||||
case "overlay": {
|
||||
if (!timeline.video) break;
|
||||
const image = request.images?.get(step.artifact_id);
|
||||
if (!image) throw new Error(`Image ${step.artifact_id} is not available.`);
|
||||
if (step.start !== undefined && step.start >= timeline.seconds) {
|
||||
throw new Error(
|
||||
`overlay from ${seconds(step.start)} s starts after the end of the ${seconds(timeline.seconds)} s video.`,
|
||||
);
|
||||
}
|
||||
overlayImage(graph, timeline, timeline.video, step, addInput(image));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -461,21 +476,27 @@ function keepSegments(graph: FilterGraph, timeline: Timeline, segments: readonly
|
||||
timeline.seconds = segments.reduce((total, segment) => total + segment.end - segment.start, 0);
|
||||
}
|
||||
|
||||
/** Appends clips normalized to the timeline: frame size, fit, fps and 48 kHz stereo audio. */
|
||||
function appendClips(
|
||||
/**
|
||||
* Adds clips at the end or the start of the timeline, normalized to it: frame size, fit, fps
|
||||
* and 48 kHz stereo audio.
|
||||
*/
|
||||
function addClips(
|
||||
graph: FilterGraph,
|
||||
timeline: Timeline,
|
||||
clips: readonly { clip: PlanInput; index: number }[],
|
||||
at: "end" | "start",
|
||||
) {
|
||||
const fps = timeline.fps || FALLBACK_FPS;
|
||||
const needsAudio = timeline.audio !== null || clips.some(({ clip }) => clip.probe.hasAudio);
|
||||
const videoParts: string[] = [];
|
||||
const audioParts: string[] = [];
|
||||
let mainVideo: string | undefined;
|
||||
let mainAudio: string | undefined;
|
||||
if (timeline.video) {
|
||||
timeline.video.filters.push("setsar=1", "format=yuv420p");
|
||||
videoParts.push(graph.flush(timeline.video, "v"));
|
||||
mainVideo = graph.flush(timeline.video, "v");
|
||||
}
|
||||
if (needsAudio) audioParts.push(audioPart(graph, timeline.audio, timeline.seconds));
|
||||
if (needsAudio) mainAudio = audioPart(graph, timeline.audio, timeline.seconds);
|
||||
let seconds = timeline.seconds;
|
||||
for (const { clip, index } of clips) {
|
||||
const video = clip.probe.video;
|
||||
@ -499,10 +520,60 @@ function appendClips(
|
||||
}
|
||||
seconds += clipSeconds;
|
||||
}
|
||||
const add = at === "start" ? "push" : "unshift";
|
||||
if (mainVideo) videoParts[add](mainVideo);
|
||||
if (mainAudio) audioParts[add](mainAudio);
|
||||
concatParts(graph, timeline, videoParts, audioParts, clips.length + 1);
|
||||
timeline.seconds = seconds;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scales an image to its share of the frame width, within the margins, and lays it over the
|
||||
* video at a corner or the center. Margins are wider on tall frames, where 9:16 apps put their
|
||||
* own buttons and caption, so the top corners stay visible there.
|
||||
*/
|
||||
function overlayImage(
|
||||
graph: FilterGraph,
|
||||
timeline: Timeline,
|
||||
video: Track,
|
||||
step: Extract<EditVideoStep, { op: "overlay" }>,
|
||||
index: number,
|
||||
) {
|
||||
const { width, height } = timeline;
|
||||
const tall = height >= width * 1.5;
|
||||
const marginX = Math.round(width * (tall ? 0.06 : 0.04));
|
||||
const top = Math.round(height * (tall ? 0.15 : 0.04));
|
||||
const bottom = Math.round(height * (tall ? 0.2 : 0.04));
|
||||
const image = graph.label("v");
|
||||
graph.add(
|
||||
[`${index}:v:0`],
|
||||
[
|
||||
`scale=w=${even(width * step.width)}:h=${even(height - top - bottom)}:force_original_aspect_ratio=decrease:force_divisible_by=2`,
|
||||
"format=rgba",
|
||||
...(step.opacity < 1 ? [`colorchannelmixer=aa=${number(step.opacity)}`] : []),
|
||||
],
|
||||
[image],
|
||||
);
|
||||
const x = step.position.endsWith("left")
|
||||
? `${marginX}`
|
||||
: step.position.endsWith("right")
|
||||
? `W-w-${marginX}`
|
||||
: "(W-w)/2";
|
||||
const y = step.position.startsWith("top")
|
||||
? `${top}`
|
||||
: step.position.startsWith("bottom")
|
||||
? `H-h-${bottom}`
|
||||
: "(H-h)/2";
|
||||
// A still image ends after its one frame; overlay repeats it for as long as the video runs.
|
||||
const timed = step.start !== undefined || step.end !== undefined;
|
||||
const enable = timed
|
||||
? `:enable='between(t,${ffmpegSeconds(step.start ?? 0)},${ffmpegSeconds(step.end ?? timeline.seconds)})'`
|
||||
: "";
|
||||
const output = graph.label("v");
|
||||
graph.add([graph.flush(video, "v"), image], [`overlay=x=${x}:y=${y}${enable}`], [output]);
|
||||
timeline.video = { label: output, filters: [] };
|
||||
}
|
||||
|
||||
/** A clip's audio in the common format, or silence as long as the clip when it has none. */
|
||||
function audioPart(graph: FilterGraph, track: Track | null, seconds: number): string {
|
||||
if (track) {
|
||||
|
||||
@ -159,6 +159,30 @@ describePostgres("video operations in the media runtime (PostgreSQL + ffmpeg)",
|
||||
});
|
||||
});
|
||||
|
||||
it("brands a stored video with the user's intro and logo", async () => {
|
||||
const { runtime, scope, store } = await setup();
|
||||
const source = await store(clips.landscape, "talk.mp4", "video/mp4");
|
||||
const intro = await store(clips.silentPortrait, "intro.mp4", "video/mp4");
|
||||
const logo = await store(clips.logo, "logo.png", "image/png");
|
||||
const started = await runtime.start({
|
||||
...scope,
|
||||
operation: "edit_video",
|
||||
input: {
|
||||
artifact_id: source,
|
||||
title: "Talk",
|
||||
steps: [
|
||||
{ op: "overlay", artifact_id: logo, opacity: 0.8 },
|
||||
{ op: "concat", artifact_ids: [intro], at: "start" },
|
||||
],
|
||||
},
|
||||
});
|
||||
await runtime.runJob(started.jobId);
|
||||
const job = await runtime.get(started.jobId);
|
||||
expect(job).toMatchObject({ status: "done" });
|
||||
expect(job!.outputs[0]).toMatchObject({ width: 320, height: 180, parentArtifactId: source });
|
||||
expect(Math.abs((job!.outputs[0]!.durationMs ?? 0) - 5000)).toBeLessThan(200);
|
||||
});
|
||||
|
||||
it("exports one stored version per preset", async () => {
|
||||
const { runtime, scope, store } = await setup();
|
||||
const source = await store(clips.landscape, "holiday.mp4", "video/mp4");
|
||||
@ -235,15 +259,17 @@ describePostgres("video operations in the media runtime (PostgreSQL + ffmpeg)",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses appended clips and subtitles the user cannot read", async () => {
|
||||
it("refuses appended clips, subtitles and overlays the user cannot read", async () => {
|
||||
const { runtime, scope, store } = await setup();
|
||||
const other = await setup();
|
||||
const source = await store(clips.landscape, "mine.mp4", "video/mp4");
|
||||
const foreignClip = await other.store(clips.silentPortrait, "theirs.mp4", "video/mp4");
|
||||
const foreignSubtitles = await other.store(clips.srt, "theirs.srt", "application/x-subrip");
|
||||
const foreignLogo = await other.store(clips.logo, "theirs.png", "image/png");
|
||||
for (const step of [
|
||||
{ op: "concat", artifact_ids: [foreignClip] },
|
||||
{ op: "subtitles", artifact_id: foreignSubtitles },
|
||||
{ op: "overlay", artifact_id: foreignLogo },
|
||||
]) {
|
||||
await expect(
|
||||
runtime.start({
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { rm } from "node:fs/promises";
|
||||
import { readFile, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { MEDIA_PLATFORM_PROFILES, MediaPlatformIdSchema } from "@rakazo/contracts";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
@ -42,9 +42,10 @@ describe("video operation registrations", () => {
|
||||
steps: [
|
||||
{ op: "concat", artifact_ids: ["clip-1", "clip-2"] },
|
||||
{ op: "subtitles", artifact_id: "subs" },
|
||||
{ op: "overlay", artifact_id: "logo" },
|
||||
],
|
||||
});
|
||||
expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs"]);
|
||||
expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs", "logo"]);
|
||||
const exported = operation("export_video");
|
||||
const exportInput = exported.inputSchema.parse({ artifact_id: "src", presets: ["vertical"] });
|
||||
expect(exported.artifactIds?.(exportInput)).toEqual(["src"]);
|
||||
@ -124,6 +125,7 @@ describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => {
|
||||
srt: { path: clips.srt, name: "captions.srt", mimeType: "application/x-subrip" },
|
||||
vtt: { path: clips.vtt, name: "captions.vtt", mimeType: "text/vtt" },
|
||||
notes: { path: clips.landscape, name: "notes.txt", mimeType: "text/plain" },
|
||||
logo: { path: clips.logo, name: "logo.png", mimeType: "image/png" },
|
||||
};
|
||||
});
|
||||
|
||||
@ -225,6 +227,68 @@ describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => {
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("puts an intro first and a translucent logo on the main part", async () => {
|
||||
const red = join(clips.dir, "red.mp4");
|
||||
const green = join(clips.dir, "green.mp4");
|
||||
for (const [path, color] of [
|
||||
[red, "red"],
|
||||
[green, "green"],
|
||||
] as const) {
|
||||
await runFfmpeg([
|
||||
...["-f", "lavfi", "-i", `color=c=${color}:size=320x180:rate=25:duration=1`],
|
||||
...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", `file:${path}`],
|
||||
]);
|
||||
}
|
||||
const registration = operation("edit_video");
|
||||
const job = await fakeMediaJobContext(
|
||||
registration.inputSchema.parse({
|
||||
artifact_id: "red",
|
||||
title: "Branded",
|
||||
steps: [
|
||||
{ op: "overlay", artifact_id: "logo", position: "top_left", width: 0.25, opacity: 0.5 },
|
||||
{ op: "concat", artifact_ids: ["green"], at: "start" },
|
||||
],
|
||||
}),
|
||||
{
|
||||
...artifacts,
|
||||
red: { path: red, name: "red.mp4", mimeType: "video/mp4" },
|
||||
green: { path: green, name: "green.mp4", mimeType: "video/mp4" },
|
||||
},
|
||||
);
|
||||
try {
|
||||
await registration.handler.run(job.context);
|
||||
const output = job.outputs[0]!.path;
|
||||
expect((await probeMedia(output, { mimeType: "video/mp4" })).durationMs).toBeCloseTo(
|
||||
2000,
|
||||
-2,
|
||||
);
|
||||
// The logo is 80×40 at 13, 7: half blue over red there, plain red elsewhere, none on the intro.
|
||||
const [introR, introG] = await pixel(output, 0.5, 30, 20);
|
||||
expect(introG).toBeGreaterThan(100);
|
||||
expect(introR).toBeLessThan(40);
|
||||
const [r, g, b] = await pixel(output, 1.5, 50, 25);
|
||||
expect(r).toBeGreaterThan(90);
|
||||
expect(r).toBeLessThan(170);
|
||||
expect(b).toBeGreaterThan(90);
|
||||
expect(b).toBeLessThan(170);
|
||||
expect(g).toBeLessThan(40);
|
||||
const [plainR, , plainB] = await pixel(output, 1.5, 200, 120);
|
||||
expect(plainR).toBeGreaterThan(200);
|
||||
expect(plainB).toBeLessThan(40);
|
||||
} finally {
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
async function pixel(path: string, at: number, x: number, y: number): Promise<number[]> {
|
||||
const out = join(clips.dir, `pixel-${at}-${x}-${y}.rgb`);
|
||||
await runFfmpeg([
|
||||
...["-ss", String(at), "-i", `file:${path}`, "-frames:v", "1"],
|
||||
...["-vf", `format=rgb24,crop=1:1:${x}:${y}`, "-f", "rawvideo", `file:${out}`],
|
||||
]);
|
||||
return [...(await readFile(out))];
|
||||
}
|
||||
|
||||
it("extracts the audio as M4A", async () => {
|
||||
const { outputs } = await run("edit_video", {
|
||||
artifact_id: "land",
|
||||
@ -357,6 +421,13 @@ describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => {
|
||||
steps: [{ op: "subtitles", artifact_id: "land" }],
|
||||
}),
|
||||
).rejects.toThrow("Landscape clip.mp4 is not an SRT or VTT file.");
|
||||
await expect(
|
||||
run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "x",
|
||||
steps: [{ op: "overlay", artifact_id: "srt" }],
|
||||
}),
|
||||
).rejects.toThrow("captions.srt is not a PNG, JPEG or WebP image.");
|
||||
await expect(
|
||||
run("export_video", { artifact_id: "notes", presets: ["square"] }),
|
||||
).rejects.toThrow("notes.txt is not a video.");
|
||||
|
||||
@ -8,6 +8,7 @@ import type {
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type {
|
||||
EditVideoInput,
|
||||
EditVideoStep,
|
||||
ExportVideoInput,
|
||||
MediaFit,
|
||||
MediaOutput,
|
||||
@ -54,6 +55,9 @@ export interface VideoOperationsDeps {
|
||||
/** Share of each output's progress spent encoding; the rest is probing and the poster. */
|
||||
const ENCODE_SHARE = 0.95;
|
||||
const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024;
|
||||
const OVERLAY_MIME_TYPES = new Set(["image/png", "image/jpeg", "image/webp"]);
|
||||
/** Larger images would take gigabytes to decode for something drawn a few hundred pixels wide. */
|
||||
const OVERLAY_MAX_SIDE = 8192;
|
||||
|
||||
/** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */
|
||||
export function createVideoOperations(
|
||||
@ -70,16 +74,7 @@ export function createVideoOperations(
|
||||
inputSchema: EditVideoInputSchema,
|
||||
title: (input) => input.title,
|
||||
available,
|
||||
artifactIds: (input) => [
|
||||
input.artifact_id,
|
||||
...input.steps.flatMap((step) =>
|
||||
step.op === "concat"
|
||||
? step.artifact_ids
|
||||
: step.op === "subtitles"
|
||||
? [step.artifact_id]
|
||||
: [],
|
||||
),
|
||||
],
|
||||
artifactIds: (input) => [input.artifact_id, ...input.steps.flatMap(stepArtifactIds)],
|
||||
handler: {
|
||||
operation: "edit_video",
|
||||
run: async (context) => {
|
||||
@ -144,12 +139,15 @@ async function runEditVideo(context: MediaJobContext): Promise<MediaOperationRes
|
||||
}
|
||||
|
||||
const clips = new Map<string, PlanInput>();
|
||||
const images = new Map<string, PlanInput>();
|
||||
const subtitleFiles = new Map<string, string>();
|
||||
for (const step of input.steps) {
|
||||
if (step.op === "concat") {
|
||||
for (const id of step.artifact_ids) {
|
||||
if (!clips.has(id)) clips.set(id, await loadVideo(context, id));
|
||||
}
|
||||
} else if (step.op === "overlay" && !images.has(step.artifact_id)) {
|
||||
images.set(step.artifact_id, await loadImage(context, step.artifact_id));
|
||||
} else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) {
|
||||
subtitleFiles.set(
|
||||
step.artifact_id,
|
||||
@ -160,13 +158,14 @@ async function runEditVideo(context: MediaJobContext): Promise<MediaOperationRes
|
||||
const audioOnly = input.steps.at(-1)?.op === "extract_audio";
|
||||
const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4");
|
||||
const speech = await detectSpeech(
|
||||
{ source, steps: input.steps, clips, subtitleFiles },
|
||||
{ source, steps: input.steps, clips, images, subtitleFiles },
|
||||
{ workDir, signal: context.signal },
|
||||
);
|
||||
const plan = planVideoEdit({
|
||||
source,
|
||||
steps: input.steps,
|
||||
clips,
|
||||
images,
|
||||
subtitleFiles,
|
||||
speech,
|
||||
outputPath,
|
||||
@ -427,6 +426,26 @@ async function loadVideo(
|
||||
return { ...file, probe };
|
||||
}
|
||||
|
||||
function stepArtifactIds(step: EditVideoStep): string[] {
|
||||
if (step.op === "concat") return step.artifact_ids;
|
||||
if (step.op === "subtitles" || step.op === "overlay") return [step.artifact_id];
|
||||
return [];
|
||||
}
|
||||
|
||||
/** A PNG, JPEG or WebP artifact for an overlay, read with its format's demuxer only. */
|
||||
async function loadImage(context: MediaJobContext, artifactId: string): Promise<PlanInput> {
|
||||
const file = await context.fetchArtifact(artifactId);
|
||||
if (file.kind !== "image" || !OVERLAY_MIME_TYPES.has(file.mimeType)) {
|
||||
throw new Error(`${file.name} is not a PNG, JPEG or WebP image.`);
|
||||
}
|
||||
const probe = await probeMedia(file.path, { signal: context.signal, mimeType: file.mimeType });
|
||||
if (!probe.video) throw new Error(`${file.name} is not a PNG, JPEG or WebP image.`);
|
||||
if (Math.max(probe.video.width, probe.video.height) > OVERLAY_MAX_SIDE) {
|
||||
throw new Error(`${file.name} is larger than ${OVERLAY_MAX_SIDE} pixels on a side.`);
|
||||
}
|
||||
return { ...file, probe };
|
||||
}
|
||||
|
||||
/** Copies an SRT/VTT artifact to a generated name, so no user text reaches the filter graph. */
|
||||
async function prepareSubtitles(
|
||||
context: MediaJobContext,
|
||||
|
||||
@ -35,12 +35,39 @@ describe("edit_video input", () => {
|
||||
]).success,
|
||||
).toBe(false);
|
||||
expect(edit([{ op: "blur" }]).success).toBe(false);
|
||||
expect(edit([{ op: "overlay", artifact_id: "logo", start: 4, end: 2 }]).success).toBe(false);
|
||||
expect(edit([{ op: "overlay", artifact_id: "logo", width: 1.5 }]).success).toBe(false);
|
||||
expect(edit([{ op: "overlay", artifact_id: "logo", opacity: 0 }]).success).toBe(false);
|
||||
expect(edit([{ op: "overlay", artifact_id: "logo", position: "middle" }]).success).toBe(false);
|
||||
expect(edit([{ op: "concat", artifact_ids: ["intro"], at: "middle" }]).success).toBe(false);
|
||||
expect(
|
||||
EditVideoInputSchema.safeParse({ artifact_id: "a", steps: [{ op: "mute" }] }).success,
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("edit_video branding steps", () => {
|
||||
it("defaults a logo to the top right at 15% of the width and clips to the end", () => {
|
||||
const parsed = EditVideoInputSchema.parse({
|
||||
artifact_id: "a",
|
||||
title: "Reel",
|
||||
steps: [
|
||||
{ op: "concat", artifact_ids: ["intro"], at: "start" },
|
||||
{ op: "concat", artifact_ids: ["outro"] },
|
||||
{ op: "overlay", artifact_id: "logo" },
|
||||
{ op: "overlay", artifact_id: "mark", position: "center", opacity: 0.4, end: 3 },
|
||||
],
|
||||
});
|
||||
expect(parsed.steps).toEqual([
|
||||
{ op: "concat", artifact_ids: ["intro"], at: "start" },
|
||||
{ op: "concat", artifact_ids: ["outro"], at: "end" },
|
||||
{ op: "overlay", artifact_id: "logo", position: "top_right", width: 0.15, opacity: 1 },
|
||||
{ op: "overlay", artifact_id: "mark", position: "center", width: 0.15, opacity: 0.4, end: 3 },
|
||||
]);
|
||||
expect(EditVideoInputSchema.parse(parsed)).toEqual(parsed);
|
||||
});
|
||||
});
|
||||
|
||||
describe("export_video input", () => {
|
||||
it("needs at least one preset or platform and defaults to cropping", () => {
|
||||
expect(ExportVideoInputSchema.parse({ artifact_id: "a", presets: ["square"] })).toEqual({
|
||||
|
||||
@ -18,14 +18,37 @@ const TimeRangeSchema = z
|
||||
/** Written by the bot in the user's language; shown on the media card and used as file name. */
|
||||
export const MediaTitleSchema = z.string().trim().min(1).max(120);
|
||||
|
||||
/** Lays an image, a logo or watermark say, over the video: the whole time or from start to end. */
|
||||
const OverlayStepSchema = z
|
||||
.object({
|
||||
op: z.literal("overlay"),
|
||||
artifact_id: Id,
|
||||
position: z
|
||||
.enum(["top_left", "top_right", "bottom_left", "bottom_right", "center"])
|
||||
.default("top_right"),
|
||||
/** Share of the frame width. */
|
||||
width: z.number().min(0.02).max(1).default(0.15),
|
||||
opacity: z.number().min(0.05).max(1).default(1),
|
||||
start: Seconds.optional(),
|
||||
end: Seconds.optional(),
|
||||
})
|
||||
.refine(
|
||||
(step) => step.start === undefined || step.end === undefined || step.end > step.start,
|
||||
"end must be after start",
|
||||
);
|
||||
|
||||
export const EditVideoStepSchema = z.discriminatedUnion("op", [
|
||||
z
|
||||
.object({ op: z.literal("trim"), start: Seconds, end: Seconds })
|
||||
.refine((step) => step.end > step.start, "end must be after start"),
|
||||
/** Keeps these parts and cuts everything between them. */
|
||||
z.object({ op: z.literal("keep"), segments: z.array(TimeRangeSchema).min(1).max(50) }),
|
||||
/** Appends these videos, fitted to the current frame. */
|
||||
z.object({ op: z.literal("concat"), artifact_ids: z.array(Id).min(1).max(20) }),
|
||||
/** Adds these videos at the end, or at the start for an intro, fitted to the current frame. */
|
||||
z.object({
|
||||
op: z.literal("concat"),
|
||||
artifact_ids: z.array(Id).min(1).max(20),
|
||||
at: z.enum(["end", "start"]).default("end"),
|
||||
}),
|
||||
z.object({
|
||||
op: z.literal("reframe"),
|
||||
preset: MediaPresetIdSchema,
|
||||
@ -42,6 +65,7 @@ export const EditVideoStepSchema = z.discriminatedUnion("op", [
|
||||
/** Cover frame at this second of the result; as the only kind of step, the result is that image. */
|
||||
z.object({ op: z.literal("poster"), at: Seconds }),
|
||||
RemoveSilenceStepSchema,
|
||||
OverlayStepSchema,
|
||||
]);
|
||||
export type EditVideoStep = z.infer<typeof EditVideoStepSchema>;
|
||||
|
||||
|
||||
@ -41,6 +41,7 @@ export const STUDIO_BOT_DEFAULTS = {
|
||||
"- Keep the pace tight: cut pauses, filler and repeats; short-form wants a cut every few seconds while the story stays easy to follow.",
|
||||
"- Respect safe areas in 9:16: keep faces, text and subtitles out of the top 15% and bottom 20%, where the platform's buttons sit. When reframing, keep the subject centred.",
|
||||
"- Most social video plays muted: transcribe speech, then burn in short subtitles.",
|
||||
"- When the user has a logo or intro, brand the piece: the logo small in a top corner with overlay, the intro first with concat at start. Remember which files they are.",
|
||||
"- Keep the source: every edit makes a new file, so try a cut and refine it.",
|
||||
"",
|
||||
"Media jobs run in the background and the user sees each one's card. Set continue_after when you need a result for your next step. Ask only when the purpose or format is unclear and a wrong guess would waste a long render.",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user