diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index 4a8aaae..3c140df 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -569,12 +569,12 @@ export const builtinAgentTools: ConnectorTool[] = [ }, { name: "edit_video", - description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image; remove_silence {threshold_db -35, min_silence 0.6, padding 0.12} cuts pauses quieter than threshold_db dB and at least min_silence s long, keeping padding s on each side. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`, + description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids, at "end" or "start"} adds videos at the end, or at the start for an intro, fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image; remove_silence {threshold_db -35, min_silence 0.6, padding 0.12} cuts pauses quieter than threshold_db dB and at least min_silence s long, keeping padding s on each side; overlay {artifact_id, position top_left|top_right|bottom_left|bottom_right|center, width 0.15, opacity 1, start, end} lays a PNG, JPEG or WebP image such as a logo or watermark over the video, width as a share of the frame, from start to end s or the whole time; top_right stays clear of 9:16 app buttons. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`, inputSchema: mediaParameters(EditVideoInputSchema), }, { name: "export_video", - description: `Export a video artifact as one H.264/AAC MP4 per platform and per preset, 6 at most. When the user names where it will be posted, use platforms: each file is fitted to that platform's frame, frame rate, bitrate and loudness, then checked. Platforms (longest video): ${VIDEO_PLATFORMS}. A platform the video is too long or too short for is skipped: trim with edit_video first. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`, + description: `Export a video artifact as one H.264/AAC MP4 per platform and per preset, 6 at most. When the user names where it will be posted, use platforms: each file is fitted to that platform's frame, frame rate, bitrate and loudness, then checked. Platforms (longest video): ${VIDEO_PLATFORMS}. A platform the video is too long or too short for is skipped: trim with edit_video first. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio, subtitles or a logo use edit_video.`, inputSchema: mediaParameters(ExportVideoInputSchema), }, { diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts index bf22a3e..c1db41f 100644 --- a/packages/adapters/src/media/ffmpeg.ts +++ b/packages/adapters/src/media/ffmpeg.ts @@ -81,6 +81,9 @@ const DEMUXERS: Record = { "audio/mpeg": "mp3", "audio/wav": "wav", "audio/ogg": "ogg", + "image/png": "png_pipe", + "image/jpeg": "jpeg_pipe", + "image/webp": "webp_pipe", }; let detection: Promise | undefined; diff --git a/packages/adapters/src/media/media-test-support.ts b/packages/adapters/src/media/media-test-support.ts index 63d9f56..edb950a 100644 --- a/packages/adapters/src/media/media-test-support.ts +++ b/packages/adapters/src/media/media-test-support.ts @@ -14,6 +14,8 @@ export interface TestClips { silentPortrait: string; srt: string; vtt: string; + /** 64×32 PNG, opaque blue. */ + logo: string; } /** Tiny generated clips; nothing binary is committed. */ @@ -40,7 +42,17 @@ export async function createTestClips(): Promise { ); const vtt = join(dir, "captions.vtt"); await writeFile(vtt, "WEBVTT\n\n00:00.100 --> 00:01.500\nHello from WebVTT\n"); - return { dir, landscape, silentPortrait, srt, vtt }; + const logo = join(dir, "logo.png"); + await runFfmpeg([ + "-f", + "lavfi", + "-i", + "color=c=blue:size=64x32", + "-frames:v", + "1", + `file:${logo}`, + ]); + return { dir, landscape, silentPortrait, srt, vtt, logo }; } export interface TestArtifact { diff --git a/packages/adapters/src/media/video-edit-plan.test.ts b/packages/adapters/src/media/video-edit-plan.test.ts index ce64277..01c54e3 100644 --- a/packages/adapters/src/media/video-edit-plan.test.ts +++ b/packages/adapters/src/media/video-edit-plan.test.ts @@ -118,7 +118,7 @@ describe("planVideoEdit", () => { name: "clip.webm", probe: probe({ width: 1080, height: 1920, fps: 30, durationMs: 2000, hasAudio: false }), }; - const result = plan([{ op: "concat", artifact_ids: ["clip"] }], { + const result = plan([{ op: "concat", artifact_ids: ["clip"], at: "end" }], { clips: new Map([["clip", clip]]), }); expect(result.args).toContain("matroska"); @@ -134,11 +134,25 @@ describe("planVideoEdit", () => { expect(result.durationMs).toBe(12_000); }); + it("puts an intro before the video", () => { + const result = plan( + [ + { op: "trim", start: 0, end: 4 }, + { op: "concat", artifact_ids: ["clip"], at: "start" }, + ], + { clips: new Map([["clip", { ...source, probe: probe({ durationMs: 1500 }) }]]) }, + ); + const graph = graphOf(result.args); + expect(graph).toContain("[0:v:0]setsar=1,format=yuv420p[v1]"); + expect(graph).toContain("[v3][a4][v1][a2]concat=n=2:v=1:a=1"); + expect(result.durationMs).toBe(5500); + }); + it("adds silence to a silent source when an appended clip has audio", () => { const silent = { ...source, probe: probe({ hasAudio: false }) }; const result = planVideoEdit({ source: silent, - steps: [{ op: "concat", artifact_ids: ["clip"] }], + steps: [{ op: "concat", artifact_ids: ["clip"], at: "end" }], clips: new Map([["clip", source]]), outputPath: "/out.mp4", }); @@ -152,7 +166,7 @@ describe("planVideoEdit", () => { const result = plan( [ { op: "reframe", preset: "vertical", fit: "crop" }, - { op: "concat", artifact_ids: ["clip"] }, + { op: "concat", artifact_ids: ["clip"], at: "end" }, ], { clips: new Map([["clip", source]]) }, ); @@ -161,6 +175,94 @@ describe("planVideoEdit", () => { ); }); + it("lays a logo over a corner with margins, its own demuxer, opacity and timing", () => { + const logo: PlanInput = { + path: "/jobs/1/input-logo", + mimeType: "image/png", + name: "logo.png", + probe: probe({ codec: "png", width: 600, height: 200, durationMs: 40, hasAudio: false }), + }; + const result = plan( + [ + { + op: "overlay", + artifact_id: "logo", + position: "top_right", + width: 0.2, + opacity: 0.5, + start: 1, + }, + ], + { images: new Map([["logo", logo]]) }, + ); + expect(result.args).toEqual( + expect.arrayContaining(["-f", "png_pipe", "-i", "file:/jobs/1/input-logo"]), + ); + expect(graphOf(result.args)).toBe( + [ + "[1:v:0]scale=w=384:h=994:force_original_aspect_ratio=decrease:force_divisible_by=2,format=rgba,colorchannelmixer=aa=0.5[v1]", + "[0:v:0][v1]overlay=x=W-w-77:y=43:enable='between(t,1,10)'[v2]", + "[v2]format=yuv420p[v3]", + "[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a4]", + ].join(";"), + ); + expect(result.durationMs).toBe(10_000); + }); + + it("keeps a logo clear of 9:16 app buttons and centers one when asked", () => { + const logo: PlanInput = { ...source, mimeType: "image/png", name: "logo.png" }; + const images = new Map([["logo", logo]]); + const tall = graphOf( + plan( + [ + { op: "reframe", preset: "vertical", fit: "crop" }, + { op: "overlay", artifact_id: "logo", position: "bottom_left", width: 0.15, opacity: 1 }, + ], + { images }, + ).args, + ); + expect(tall).toContain("scale=w=162:h=1248:"); + expect(tall).toContain("overlay=x=65:y=H-h-384["); + expect(tall).not.toContain("colorchannelmixer"); + const center = graphOf( + plan([{ op: "overlay", artifact_id: "logo", position: "center", width: 0.5, opacity: 1 }], { + images, + }).args, + ); + expect(center).toContain("overlay=x=(W-w)/2:y=(H-h)/2["); + }); + + it("ignores overlays in audio results and refuses a missing image or a late start", () => { + const steps: EditVideoStep[] = [ + { op: "overlay", artifact_id: "logo", position: "top_right", width: 0.15, opacity: 1 }, + { op: "extract_audio" }, + ]; + const audio = plan(steps); + expect(audio.kind).toBe("audio"); + expect(audio.args).not.toContain("png_pipe"); + expect(() => + plan([ + { op: "overlay", artifact_id: "logo", position: "top_right", width: 0.15, opacity: 1 }, + ]), + ).toThrow("Image logo is not available."); + expect(() => + plan( + [ + { op: "trim", start: 0, end: 2 }, + { + op: "overlay", + artifact_id: "logo", + position: "top_right", + width: 0.15, + opacity: 1, + start: 2, + }, + ], + { images: new Map([["logo", source]]) }, + ), + ).toThrow("overlay from 2 s starts after the end of the 2 s video."); + }); + it("reframes by crop, pad or blur, and only scales when the aspect already matches", () => { expect(graphOf(plan([{ op: "reframe", preset: "square", fit: "crop" }]).args)).toContain( "scale=1080:1080:force_original_aspect_ratio=increase,crop=1080:1080,setsar=1", diff --git a/packages/adapters/src/media/video-edit-plan.ts b/packages/adapters/src/media/video-edit-plan.ts index 05b9162..7375435 100644 --- a/packages/adapters/src/media/video-edit-plan.ts +++ b/packages/adapters/src/media/video-edit-plan.ts @@ -20,6 +20,8 @@ export interface VideoEditPlanRequest { steps: readonly EditVideoStep[]; /** Videos named by concat steps, by artifact id. */ clips?: ReadonlyMap; + /** Images named by overlay steps, by artifact id. */ + images?: ReadonlyMap; /** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */ subtitleFiles?: ReadonlyMap; /** @@ -191,7 +193,7 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { ); break; case "concat": - appendClips( + addClips( graph, timeline, step.artifact_ids.map((id) => { @@ -199,6 +201,7 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { if (!clip) throw new Error(`Video ${id} is not available.`); return { clip, index: addInput(clip) }; }), + step.at, ); break; case "reframe": { @@ -252,6 +255,18 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { if (speech) keepSegments(graph, timeline, speech); break; } + case "overlay": { + if (!timeline.video) break; + const image = request.images?.get(step.artifact_id); + if (!image) throw new Error(`Image ${step.artifact_id} is not available.`); + if (step.start !== undefined && step.start >= timeline.seconds) { + throw new Error( + `overlay from ${seconds(step.start)} s starts after the end of the ${seconds(timeline.seconds)} s video.`, + ); + } + overlayImage(graph, timeline, timeline.video, step, addInput(image)); + break; + } } } @@ -461,21 +476,27 @@ function keepSegments(graph: FilterGraph, timeline: Timeline, segments: readonly timeline.seconds = segments.reduce((total, segment) => total + segment.end - segment.start, 0); } -/** Appends clips normalized to the timeline: frame size, fit, fps and 48 kHz stereo audio. */ -function appendClips( +/** + * Adds clips at the end or the start of the timeline, normalized to it: frame size, fit, fps + * and 48 kHz stereo audio. + */ +function addClips( graph: FilterGraph, timeline: Timeline, clips: readonly { clip: PlanInput; index: number }[], + at: "end" | "start", ) { const fps = timeline.fps || FALLBACK_FPS; const needsAudio = timeline.audio !== null || clips.some(({ clip }) => clip.probe.hasAudio); const videoParts: string[] = []; const audioParts: string[] = []; + let mainVideo: string | undefined; + let mainAudio: string | undefined; if (timeline.video) { timeline.video.filters.push("setsar=1", "format=yuv420p"); - videoParts.push(graph.flush(timeline.video, "v")); + mainVideo = graph.flush(timeline.video, "v"); } - if (needsAudio) audioParts.push(audioPart(graph, timeline.audio, timeline.seconds)); + if (needsAudio) mainAudio = audioPart(graph, timeline.audio, timeline.seconds); let seconds = timeline.seconds; for (const { clip, index } of clips) { const video = clip.probe.video; @@ -499,10 +520,60 @@ function appendClips( } seconds += clipSeconds; } + const add = at === "start" ? "push" : "unshift"; + if (mainVideo) videoParts[add](mainVideo); + if (mainAudio) audioParts[add](mainAudio); concatParts(graph, timeline, videoParts, audioParts, clips.length + 1); timeline.seconds = seconds; } +/** + * Scales an image to its share of the frame width, within the margins, and lays it over the + * video at a corner or the center. Margins are wider on tall frames, where 9:16 apps put their + * own buttons and caption, so the top corners stay visible there. + */ +function overlayImage( + graph: FilterGraph, + timeline: Timeline, + video: Track, + step: Extract, + index: number, +) { + const { width, height } = timeline; + const tall = height >= width * 1.5; + const marginX = Math.round(width * (tall ? 0.06 : 0.04)); + const top = Math.round(height * (tall ? 0.15 : 0.04)); + const bottom = Math.round(height * (tall ? 0.2 : 0.04)); + const image = graph.label("v"); + graph.add( + [`${index}:v:0`], + [ + `scale=w=${even(width * step.width)}:h=${even(height - top - bottom)}:force_original_aspect_ratio=decrease:force_divisible_by=2`, + "format=rgba", + ...(step.opacity < 1 ? [`colorchannelmixer=aa=${number(step.opacity)}`] : []), + ], + [image], + ); + const x = step.position.endsWith("left") + ? `${marginX}` + : step.position.endsWith("right") + ? `W-w-${marginX}` + : "(W-w)/2"; + const y = step.position.startsWith("top") + ? `${top}` + : step.position.startsWith("bottom") + ? `H-h-${bottom}` + : "(H-h)/2"; + // A still image ends after its one frame; overlay repeats it for as long as the video runs. + const timed = step.start !== undefined || step.end !== undefined; + const enable = timed + ? `:enable='between(t,${ffmpegSeconds(step.start ?? 0)},${ffmpegSeconds(step.end ?? timeline.seconds)})'` + : ""; + const output = graph.label("v"); + graph.add([graph.flush(video, "v"), image], [`overlay=x=${x}:y=${y}${enable}`], [output]); + timeline.video = { label: output, filters: [] }; +} + /** A clip's audio in the common format, or silence as long as the clip when it has none. */ function audioPart(graph: FilterGraph, track: Track | null, seconds: number): string { if (track) { diff --git a/packages/adapters/src/media/video-operations.postgres.test.ts b/packages/adapters/src/media/video-operations.postgres.test.ts index 614cc4f..1d18824 100644 --- a/packages/adapters/src/media/video-operations.postgres.test.ts +++ b/packages/adapters/src/media/video-operations.postgres.test.ts @@ -159,6 +159,30 @@ describePostgres("video operations in the media runtime (PostgreSQL + ffmpeg)", }); }); + it("brands a stored video with the user's intro and logo", async () => { + const { runtime, scope, store } = await setup(); + const source = await store(clips.landscape, "talk.mp4", "video/mp4"); + const intro = await store(clips.silentPortrait, "intro.mp4", "video/mp4"); + const logo = await store(clips.logo, "logo.png", "image/png"); + const started = await runtime.start({ + ...scope, + operation: "edit_video", + input: { + artifact_id: source, + title: "Talk", + steps: [ + { op: "overlay", artifact_id: logo, opacity: 0.8 }, + { op: "concat", artifact_ids: [intro], at: "start" }, + ], + }, + }); + await runtime.runJob(started.jobId); + const job = await runtime.get(started.jobId); + expect(job).toMatchObject({ status: "done" }); + expect(job!.outputs[0]).toMatchObject({ width: 320, height: 180, parentArtifactId: source }); + expect(Math.abs((job!.outputs[0]!.durationMs ?? 0) - 5000)).toBeLessThan(200); + }); + it("exports one stored version per preset", async () => { const { runtime, scope, store } = await setup(); const source = await store(clips.landscape, "holiday.mp4", "video/mp4"); @@ -235,15 +259,17 @@ describePostgres("video operations in the media runtime (PostgreSQL + ffmpeg)", }); }); - it("refuses appended clips and subtitles the user cannot read", async () => { + it("refuses appended clips, subtitles and overlays the user cannot read", async () => { const { runtime, scope, store } = await setup(); const other = await setup(); const source = await store(clips.landscape, "mine.mp4", "video/mp4"); const foreignClip = await other.store(clips.silentPortrait, "theirs.mp4", "video/mp4"); const foreignSubtitles = await other.store(clips.srt, "theirs.srt", "application/x-subrip"); + const foreignLogo = await other.store(clips.logo, "theirs.png", "image/png"); for (const step of [ { op: "concat", artifact_ids: [foreignClip] }, { op: "subtitles", artifact_id: foreignSubtitles }, + { op: "overlay", artifact_id: foreignLogo }, ]) { await expect( runtime.start({ diff --git a/packages/adapters/src/media/video-operations.test.ts b/packages/adapters/src/media/video-operations.test.ts index ddd3ad0..18ee851 100644 --- a/packages/adapters/src/media/video-operations.test.ts +++ b/packages/adapters/src/media/video-operations.test.ts @@ -1,4 +1,4 @@ -import { rm } from "node:fs/promises"; +import { readFile, rm } from "node:fs/promises"; import { join } from "node:path"; import { MEDIA_PLATFORM_PROFILES, MediaPlatformIdSchema } from "@rakazo/contracts"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -42,9 +42,10 @@ describe("video operation registrations", () => { steps: [ { op: "concat", artifact_ids: ["clip-1", "clip-2"] }, { op: "subtitles", artifact_id: "subs" }, + { op: "overlay", artifact_id: "logo" }, ], }); - expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs"]); + expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs", "logo"]); const exported = operation("export_video"); const exportInput = exported.inputSchema.parse({ artifact_id: "src", presets: ["vertical"] }); expect(exported.artifactIds?.(exportInput)).toEqual(["src"]); @@ -124,6 +125,7 @@ describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => { srt: { path: clips.srt, name: "captions.srt", mimeType: "application/x-subrip" }, vtt: { path: clips.vtt, name: "captions.vtt", mimeType: "text/vtt" }, notes: { path: clips.landscape, name: "notes.txt", mimeType: "text/plain" }, + logo: { path: clips.logo, name: "logo.png", mimeType: "image/png" }, }; }); @@ -225,6 +227,68 @@ describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => { await rm(job.tempDir, { recursive: true, force: true }); }); + it("puts an intro first and a translucent logo on the main part", async () => { + const red = join(clips.dir, "red.mp4"); + const green = join(clips.dir, "green.mp4"); + for (const [path, color] of [ + [red, "red"], + [green, "green"], + ] as const) { + await runFfmpeg([ + ...["-f", "lavfi", "-i", `color=c=${color}:size=320x180:rate=25:duration=1`], + ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", `file:${path}`], + ]); + } + const registration = operation("edit_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ + artifact_id: "red", + title: "Branded", + steps: [ + { op: "overlay", artifact_id: "logo", position: "top_left", width: 0.25, opacity: 0.5 }, + { op: "concat", artifact_ids: ["green"], at: "start" }, + ], + }), + { + ...artifacts, + red: { path: red, name: "red.mp4", mimeType: "video/mp4" }, + green: { path: green, name: "green.mp4", mimeType: "video/mp4" }, + }, + ); + try { + await registration.handler.run(job.context); + const output = job.outputs[0]!.path; + expect((await probeMedia(output, { mimeType: "video/mp4" })).durationMs).toBeCloseTo( + 2000, + -2, + ); + // The logo is 80×40 at 13, 7: half blue over red there, plain red elsewhere, none on the intro. + const [introR, introG] = await pixel(output, 0.5, 30, 20); + expect(introG).toBeGreaterThan(100); + expect(introR).toBeLessThan(40); + const [r, g, b] = await pixel(output, 1.5, 50, 25); + expect(r).toBeGreaterThan(90); + expect(r).toBeLessThan(170); + expect(b).toBeGreaterThan(90); + expect(b).toBeLessThan(170); + expect(g).toBeLessThan(40); + const [plainR, , plainB] = await pixel(output, 1.5, 200, 120); + expect(plainR).toBeGreaterThan(200); + expect(plainB).toBeLessThan(40); + } finally { + await rm(job.tempDir, { recursive: true, force: true }); + } + }); + + async function pixel(path: string, at: number, x: number, y: number): Promise { + const out = join(clips.dir, `pixel-${at}-${x}-${y}.rgb`); + await runFfmpeg([ + ...["-ss", String(at), "-i", `file:${path}`, "-frames:v", "1"], + ...["-vf", `format=rgb24,crop=1:1:${x}:${y}`, "-f", "rawvideo", `file:${out}`], + ]); + return [...(await readFile(out))]; + } + it("extracts the audio as M4A", async () => { const { outputs } = await run("edit_video", { artifact_id: "land", @@ -357,6 +421,13 @@ describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => { steps: [{ op: "subtitles", artifact_id: "land" }], }), ).rejects.toThrow("Landscape clip.mp4 is not an SRT or VTT file."); + await expect( + run("edit_video", { + artifact_id: "land", + title: "x", + steps: [{ op: "overlay", artifact_id: "srt" }], + }), + ).rejects.toThrow("captions.srt is not a PNG, JPEG or WebP image."); await expect( run("export_video", { artifact_id: "notes", presets: ["square"] }), ).rejects.toThrow("notes.txt is not a video."); diff --git a/packages/adapters/src/media/video-operations.ts b/packages/adapters/src/media/video-operations.ts index 1c0629b..ef79250 100644 --- a/packages/adapters/src/media/video-operations.ts +++ b/packages/adapters/src/media/video-operations.ts @@ -8,6 +8,7 @@ import type { } from "@rakazo/adapter-kit"; import type { EditVideoInput, + EditVideoStep, ExportVideoInput, MediaFit, MediaOutput, @@ -54,6 +55,9 @@ export interface VideoOperationsDeps { /** Share of each output's progress spent encoding; the rest is probing and the poster. */ const ENCODE_SHARE = 0.95; const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024; +const OVERLAY_MIME_TYPES = new Set(["image/png", "image/jpeg", "image/webp"]); +/** Larger images would take gigabytes to decode for something drawn a few hundred pixels wide. */ +const OVERLAY_MAX_SIDE = 8192; /** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */ export function createVideoOperations( @@ -70,16 +74,7 @@ export function createVideoOperations( inputSchema: EditVideoInputSchema, title: (input) => input.title, available, - artifactIds: (input) => [ - input.artifact_id, - ...input.steps.flatMap((step) => - step.op === "concat" - ? step.artifact_ids - : step.op === "subtitles" - ? [step.artifact_id] - : [], - ), - ], + artifactIds: (input) => [input.artifact_id, ...input.steps.flatMap(stepArtifactIds)], handler: { operation: "edit_video", run: async (context) => { @@ -144,12 +139,15 @@ async function runEditVideo(context: MediaJobContext): Promise(); + const images = new Map(); const subtitleFiles = new Map(); for (const step of input.steps) { if (step.op === "concat") { for (const id of step.artifact_ids) { if (!clips.has(id)) clips.set(id, await loadVideo(context, id)); } + } else if (step.op === "overlay" && !images.has(step.artifact_id)) { + images.set(step.artifact_id, await loadImage(context, step.artifact_id)); } else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) { subtitleFiles.set( step.artifact_id, @@ -160,13 +158,14 @@ async function runEditVideo(context: MediaJobContext): Promise { + const file = await context.fetchArtifact(artifactId); + if (file.kind !== "image" || !OVERLAY_MIME_TYPES.has(file.mimeType)) { + throw new Error(`${file.name} is not a PNG, JPEG or WebP image.`); + } + const probe = await probeMedia(file.path, { signal: context.signal, mimeType: file.mimeType }); + if (!probe.video) throw new Error(`${file.name} is not a PNG, JPEG or WebP image.`); + if (Math.max(probe.video.width, probe.video.height) > OVERLAY_MAX_SIDE) { + throw new Error(`${file.name} is larger than ${OVERLAY_MAX_SIDE} pixels on a side.`); + } + return { ...file, probe }; +} + /** Copies an SRT/VTT artifact to a generated name, so no user text reaches the filter graph. */ async function prepareSubtitles( context: MediaJobContext, diff --git a/packages/contracts/src/media-video.test.ts b/packages/contracts/src/media-video.test.ts index 0499442..d03a7ef 100644 --- a/packages/contracts/src/media-video.test.ts +++ b/packages/contracts/src/media-video.test.ts @@ -35,12 +35,39 @@ describe("edit_video input", () => { ]).success, ).toBe(false); expect(edit([{ op: "blur" }]).success).toBe(false); + expect(edit([{ op: "overlay", artifact_id: "logo", start: 4, end: 2 }]).success).toBe(false); + expect(edit([{ op: "overlay", artifact_id: "logo", width: 1.5 }]).success).toBe(false); + expect(edit([{ op: "overlay", artifact_id: "logo", opacity: 0 }]).success).toBe(false); + expect(edit([{ op: "overlay", artifact_id: "logo", position: "middle" }]).success).toBe(false); + expect(edit([{ op: "concat", artifact_ids: ["intro"], at: "middle" }]).success).toBe(false); expect( EditVideoInputSchema.safeParse({ artifact_id: "a", steps: [{ op: "mute" }] }).success, ).toBe(false); }); }); +describe("edit_video branding steps", () => { + it("defaults a logo to the top right at 15% of the width and clips to the end", () => { + const parsed = EditVideoInputSchema.parse({ + artifact_id: "a", + title: "Reel", + steps: [ + { op: "concat", artifact_ids: ["intro"], at: "start" }, + { op: "concat", artifact_ids: ["outro"] }, + { op: "overlay", artifact_id: "logo" }, + { op: "overlay", artifact_id: "mark", position: "center", opacity: 0.4, end: 3 }, + ], + }); + expect(parsed.steps).toEqual([ + { op: "concat", artifact_ids: ["intro"], at: "start" }, + { op: "concat", artifact_ids: ["outro"], at: "end" }, + { op: "overlay", artifact_id: "logo", position: "top_right", width: 0.15, opacity: 1 }, + { op: "overlay", artifact_id: "mark", position: "center", width: 0.15, opacity: 0.4, end: 3 }, + ]); + expect(EditVideoInputSchema.parse(parsed)).toEqual(parsed); + }); +}); + describe("export_video input", () => { it("needs at least one preset or platform and defaults to cropping", () => { expect(ExportVideoInputSchema.parse({ artifact_id: "a", presets: ["square"] })).toEqual({ diff --git a/packages/contracts/src/media-video.ts b/packages/contracts/src/media-video.ts index 2505e3b..3b2ff6c 100644 --- a/packages/contracts/src/media-video.ts +++ b/packages/contracts/src/media-video.ts @@ -18,14 +18,37 @@ const TimeRangeSchema = z /** Written by the bot in the user's language; shown on the media card and used as file name. */ export const MediaTitleSchema = z.string().trim().min(1).max(120); +/** Lays an image, a logo or watermark say, over the video: the whole time or from start to end. */ +const OverlayStepSchema = z + .object({ + op: z.literal("overlay"), + artifact_id: Id, + position: z + .enum(["top_left", "top_right", "bottom_left", "bottom_right", "center"]) + .default("top_right"), + /** Share of the frame width. */ + width: z.number().min(0.02).max(1).default(0.15), + opacity: z.number().min(0.05).max(1).default(1), + start: Seconds.optional(), + end: Seconds.optional(), + }) + .refine( + (step) => step.start === undefined || step.end === undefined || step.end > step.start, + "end must be after start", + ); + export const EditVideoStepSchema = z.discriminatedUnion("op", [ z .object({ op: z.literal("trim"), start: Seconds, end: Seconds }) .refine((step) => step.end > step.start, "end must be after start"), /** Keeps these parts and cuts everything between them. */ z.object({ op: z.literal("keep"), segments: z.array(TimeRangeSchema).min(1).max(50) }), - /** Appends these videos, fitted to the current frame. */ - z.object({ op: z.literal("concat"), artifact_ids: z.array(Id).min(1).max(20) }), + /** Adds these videos at the end, or at the start for an intro, fitted to the current frame. */ + z.object({ + op: z.literal("concat"), + artifact_ids: z.array(Id).min(1).max(20), + at: z.enum(["end", "start"]).default("end"), + }), z.object({ op: z.literal("reframe"), preset: MediaPresetIdSchema, @@ -42,6 +65,7 @@ export const EditVideoStepSchema = z.discriminatedUnion("op", [ /** Cover frame at this second of the result; as the only kind of step, the result is that image. */ z.object({ op: z.literal("poster"), at: Seconds }), RemoveSilenceStepSchema, + OverlayStepSchema, ]); export type EditVideoStep = z.infer; diff --git a/packages/db/src/studio-agent.ts b/packages/db/src/studio-agent.ts index 5e24154..f130b19 100644 --- a/packages/db/src/studio-agent.ts +++ b/packages/db/src/studio-agent.ts @@ -41,6 +41,7 @@ export const STUDIO_BOT_DEFAULTS = { "- Keep the pace tight: cut pauses, filler and repeats; short-form wants a cut every few seconds while the story stays easy to follow.", "- Respect safe areas in 9:16: keep faces, text and subtitles out of the top 15% and bottom 20%, where the platform's buttons sit. When reframing, keep the subject centred.", "- Most social video plays muted: transcribe speech, then burn in short subtitles.", + "- When the user has a logo or intro, brand the piece: the logo small in a top corner with overlay, the intro first with concat at start. Remember which files they are.", "- Keep the source: every edit makes a new file, so try a cut and refine it.", "", "Media jobs run in the background and the user sees each one's card. Set continue_after when you need a result for your next step. Ask only when the purpose or format is unclear and a wrong guess would waste a long render.",