evOo-local/packages/adapters/src/media/video-operations.test.ts

import { readFile, rm } from "node:fs/promises";
import { join } from "node:path";
import { MEDIA_PLATFORM_PROFILES, MediaPlatformIdSchema } from "@rakazo/contracts";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { builtinAgentTools } from "../builtin-tools.js";
import { createMediaOperations } from "../media-operations.js";
import { MEDIA_CONTINUE_AFTER_PROPERTY } from "../media-tools.js";
import { parametersFor } from "../pi-runtime.js";
import { detectFfmpeg, measureLoudness, probeMedia, runFfmpeg } from "./ffmpeg.js";
import type { TestArtifact, TestClips } from "./media-test-support.js";
import { createTestClips, fakeMediaJobContext } from "./media-test-support.js";
import { LOUDNESS_TOLERANCE_LU } from "./platform-export.js";
import { cleanFileName, createVideoOperations, subtitleFormat } from "./video-operations.js";

const ffmpeg = await detectFfmpeg();

function operation(name: "edit_video" | "export_video", deps = {}) {
  const registration = createVideoOperations(deps).find((entry) => entry.operation === name);
  if (!registration) throw new Error(`missing ${name}`);
  return registration;
}

describe("video operation registrations", () => {
  it("are registered for the runtime and unavailable without ffmpeg", async () => {
    const names = createMediaOperations({ loadSettings: async () => ({}) as never }).map(
      (entry) => entry.operation,
    );
    expect(names).toEqual(expect.arrayContaining(["edit_video", "export_video"]));
    const edit = operation("edit_video", { detect: async () => null });
    expect(await edit.available({ spaceId: "space-1" })).toBe(false);
    const { context } = await fakeMediaJobContext({}, {});
    await expect(edit.handler.run(context)).rejects.toThrow(
      "ffmpeg and ffprobe are not installed on this server.",
    );
  });

  it("names every artifact the input reads so start() can authorize them", () => {
    const edit = operation("edit_video");
    const input = edit.inputSchema.parse({
      artifact_id: "src",
      title: "Reel",
      steps: [
        { op: "concat", artifact_ids: ["clip-1", "clip-2"] },
        { op: "subtitles", artifact_id: "subs" },
        { op: "overlay", artifact_id: "logo" },
      ],
    });
    expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs", "logo"]);
    const exported = operation("export_video");
    const exportInput = exported.inputSchema.parse({ artifact_id: "src", presets: ["vertical"] });
    expect(exported.artifactIds?.(exportInput)).toEqual(["src"]);
  });

  it("titles exports by platform and aspect ratio unless the bot gave a title", () => {
    const exported = operation("export_video");
    expect(
      exported.title(
        exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical", "square"] }),
      ),
    ).toBe("9:16 · 1:1");
    expect(
      exported.title(
        exported.inputSchema.parse({
          artifact_id: "a",
          platforms: ["tiktok", "youtube-shorts"],
          presets: ["square"],
        }),
      ),
    ).toBe("TikTok · YouTube Shorts · 1:1");
    expect(
      exported.title(
        exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical"], title: "Reel" }),
      ),
    ).toBe("Reel");
  });

  it("defines tools named like the operations, with continue_after, that the runtime can encode", () => {
    for (const name of ["edit_video", "export_video"]) {
      const tool = builtinAgentTools.find((entry) => entry.name === name);
      expect(tool).toBeDefined();
      expect(tool?.inputSchema.properties).toMatchObject(MEDIA_CONTINUE_AFTER_PROPERTY);
      expect(() => parametersFor(tool!)).not.toThrow();
    }
  });

  it("tells the model every platform id with its longest video", () => {
    const tool = builtinAgentTools.find((entry) => entry.name === "export_video")!;
    const properties = tool.inputSchema.properties as Record<
      string,
      { items?: { enum?: string[] } }
    >;
    expect(properties.platforms?.items?.enum).toEqual(MediaPlatformIdSchema.options);
    for (const profile of Object.values(MEDIA_PLATFORM_PROFILES)) {
      expect(tool.description).toContain(`${profile.id} ${profile.duration.hardMaxMs / 1000} s`);
    }
  });
});

describe("subtitle and file name helpers", () => {
  it("recognizes SRT and VTT by content only", () => {
    expect(subtitleFormat("WEBVTT\n\n00:00.000 --> 00:01.000\nHi")).toBe("vtt");
    expect(subtitleFormat("1\r\n00:00:01,000 --> 00:00:02,000\r\nHi")).toBe("srt");
    expect(subtitleFormat("00:00:01.000 --> 00:00:02.000\nHi")).toBe("srt");
    expect(subtitleFormat("#EXTM3U\n#EXTINF:1,\nfile:/etc/passwd")).toBeNull();
    expect(subtitleFormat("WEBVTTX")).toBeNull();
  });

  it("cleans file names without losing the user's language", () => {
    expect(cleanFileName("Reel: Sommer/Urlaub 2026", "video")).toBe("Reel Sommer Urlaub 2026");
    expect(cleanFileName("../..\u0000", "video")).toBe("video");
    expect(cleanFileName("x".repeat(200), "video")).toHaveLength(80);
  });
});

// CI runners may not have ffmpeg; these run wherever it is installed.
describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => {
  let clips: TestClips;
  let artifacts: Record<string, TestArtifact>;

  beforeAll(async () => {
    clips = await createTestClips();
    artifacts = {
      land: { path: clips.landscape, name: "Landscape clip.mp4", mimeType: "video/mp4" },
      silent: { path: clips.silentPortrait, name: "silent.mp4", mimeType: "video/mp4" },
      srt: { path: clips.srt, name: "captions.srt", mimeType: "application/x-subrip" },
      vtt: { path: clips.vtt, name: "captions.vtt", mimeType: "text/vtt" },
      notes: { path: clips.landscape, name: "notes.txt", mimeType: "text/plain" },
      logo: { path: clips.logo, name: "logo.png", mimeType: "image/png" },
    };
  });

  afterAll(async () => {
    if (clips) await rm(clips.dir, { recursive: true, force: true });
  });

  async function run(name: "edit_video" | "export_video", input: Record<string, unknown>) {
    const registration = operation(name);
    const job = await fakeMediaJobContext(registration.inputSchema.parse(input), artifacts);
    try {
      const result = await registration.handler.run(job.context);
      return { ...job, result };
    } finally {
      await rm(job.tempDir, { recursive: true, force: true });
    }
  }

  it("cuts, reframes to 9:16 with a blurred fill, burns in subtitles and speeds up", async () => {
    const { outputs, progress, result } = await run("edit_video", {
      artifact_id: "land",
      title: "Sommer-Reel",
      steps: [
        { op: "trim", start: 0.5, end: 2.5 },
        { op: "reframe", preset: "vertical", fit: "blur" },
        { op: "subtitles", artifact_id: "srt" },
        { op: "speed", factor: 2 },
        { op: "poster", at: 0.5 },
      ],
    });
    expect(outputs).toHaveLength(1);
    const [output] = outputs;
    expect(output).toMatchObject({
      name: "Sommer-Reel.mp4",
      mimeType: "video/mp4",
      width: 1080,
      height: 1920,
      parentArtifactId: "land",
    });
    expect(output!.posterSize).toBeGreaterThan(0);
    expect(Math.abs(output!.durationMs! - 1000)).toBeLessThan(150);
    expect(progress.at(-1)).toBe(1);
    expect(progress).toEqual([...progress].sort((a, b) => a - b));
    expect(result.summary).toContain("Sommer-Reel.mp4 (1080×1920, ");
  });

  it("keeps segments, appends a silent clip with generated silence, and changes volume", async () => {
    const registration = operation("edit_video");
    const job = await fakeMediaJobContext(
      registration.inputSchema.parse({
        artifact_id: "land",
        title: "Cut",
        steps: [
          {
            op: "keep",
            segments: [
              { start: 0, end: 0.5 },
              { start: 2, end: 2.8 },
            ],
          },
          { op: "concat", artifact_ids: ["silent"] },
          { op: "volume", gain: 0.5 },
          { op: "subtitles", artifact_id: "vtt" },
        ],
      }),
      artifacts,
    );
    await registration.handler.run(job.context);
    const [output] = job.outputs;
    const probe = await probeMedia(output!.path, { mimeType: "video/mp4" });
    expect(probe.hasAudio).toBe(true);
    expect(probe.video).toMatchObject({ width: 320, height: 180, rotation: 0 });
    expect(Math.abs(probe.durationMs - 3300)).toBeLessThan(200);
    await rm(job.tempDir, { recursive: true, force: true });
  });

  it("plays rotated phone video upright without rotating it twice", async (context) => {
    const rotated = join(clips.dir, "rotated.mp4");
    try {
      await runFfmpeg([
        ...["-display_rotation", "90", "-i", `file:${clips.landscape}`],
        ...["-c", "copy", `file:${rotated}`],
      ]);
    } catch {
      context.skip("this ffmpeg cannot write rotation metadata");
    }
    const registration = operation("edit_video");
    const job = await fakeMediaJobContext(
      registration.inputSchema.parse({
        artifact_id: "rotated",
        title: "Upright",
        steps: [{ op: "trim", start: 0, end: 1 }],
      }),
      { rotated: { path: rotated, name: "rotated.mp4", mimeType: "video/mp4" } },
    );
    await registration.handler.run(job.context);
    const probe = await probeMedia(job.outputs[0]!.path, { mimeType: "video/mp4" });
    expect(probe.video).toMatchObject({ width: 180, height: 320, rotation: 0 });
    await rm(job.tempDir, { recursive: true, force: true });
  });

  it("puts an intro first and a translucent logo on the main part", async () => {
    const red = join(clips.dir, "red.mp4");
    const green = join(clips.dir, "green.mp4");
    for (const [path, color] of [
      [red, "red"],
      [green, "green"],
    ] as const) {
      await runFfmpeg([
        ...["-f", "lavfi", "-i", `color=c=${color}:size=320x180:rate=25:duration=1`],
        ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", `file:${path}`],
      ]);
    }
    const registration = operation("edit_video");
    const job = await fakeMediaJobContext(
      registration.inputSchema.parse({
        artifact_id: "red",
        title: "Branded",
        steps: [
          { op: "overlay", artifact_id: "logo", position: "top_left", width: 0.25, opacity: 0.5 },
          { op: "concat", artifact_ids: ["green"], at: "start" },
        ],
      }),
      {
        ...artifacts,
        red: { path: red, name: "red.mp4", mimeType: "video/mp4" },
        green: { path: green, name: "green.mp4", mimeType: "video/mp4" },
      },
    );
    try {
      await registration.handler.run(job.context);
      const output = job.outputs[0]!.path;
      expect((await probeMedia(output, { mimeType: "video/mp4" })).durationMs).toBeCloseTo(
        2000,
        -2,
      );
      // The logo is 80×40 at 13, 7: half blue over red there, plain red elsewhere, none on the intro.
      const [introR, introG] = await pixel(output, 0.5, 30, 20);
      expect(introG).toBeGreaterThan(100);
      expect(introR).toBeLessThan(40);
      const [r, g, b] = await pixel(output, 1.5, 50, 25);
      expect(r).toBeGreaterThan(90);
      expect(r).toBeLessThan(170);
      expect(b).toBeGreaterThan(90);
      expect(b).toBeLessThan(170);
      expect(g).toBeLessThan(40);
      const [plainR, , plainB] = await pixel(output, 1.5, 200, 120);
      expect(plainR).toBeGreaterThan(200);
      expect(plainB).toBeLessThan(40);
    } finally {
      await rm(job.tempDir, { recursive: true, force: true });
    }
  });

  async function pixel(path: string, at: number, x: number, y: number): Promise<number[]> {
    const out = join(clips.dir, `pixel-${at}-${x}-${y}.rgb`);
    await runFfmpeg([
      ...["-ss", String(at), "-i", `file:${path}`, "-frames:v", "1"],
      ...["-vf", `format=rgb24,crop=1:1:${x}:${y}`, "-f", "rawvideo", `file:${out}`],
    ]);
    return [...(await readFile(out))];
  }

  it("extracts the audio as M4A", async () => {
    const { outputs } = await run("edit_video", {
      artifact_id: "land",
      title: "Ton",
      steps: [{ op: "trim", start: 1, end: 2 }, { op: "extract_audio" }],
    });
    expect(outputs).toEqual([
      expect.objectContaining({ name: "Ton.m4a", mimeType: "audio/mp4", parentArtifactId: "land" }),
    ]);
    expect(outputs[0]?.posterPath).toBeUndefined();
    expect(Math.abs(outputs[0]!.durationMs! - 1000)).toBeLessThan(100);
  });

  it("saves a single frame when poster is the only step", async () => {
    const { outputs } = await run("edit_video", {
      artifact_id: "land",
      title: "Cover",
      steps: [{ op: "poster", at: 1 }],
    });
    expect(outputs).toEqual([
      expect.objectContaining({
        name: "Cover.jpg",
        mimeType: "image/jpeg",
        width: 320,
        height: 180,
      }),
    ]);
  });

  it("exports one output per distinct preset with posters and batch progress", async () => {
    const { outputs, progress } = await run("export_video", {
      artifact_id: "land",
      presets: ["square", "portrait", "square"],
      fit: "pad",
    });
    expect(
      outputs.map((output) => [output.name, output.preset, output.width, output.height]),
    ).toEqual([
      ["Landscape clip-1x1.mp4", "square", 1080, 1080],
      ["Landscape clip-4x5.mp4", "portrait", 1080, 1350],
    ]);
    for (const output of outputs) {
      expect(output.parentArtifactId).toBe("land");
      expect(output.posterSize).toBeGreaterThan(0);
    }
    expect(progress).toContain(0.5);
    expect(progress.at(-1)).toBe(1);
  });

  it("exports for a platform at its frame, frame rate and loudness, and skips one it is too short for", async () => {
    const fast = join(clips.dir, "fast.mp4");
    await runFfmpeg([
      ...["-f", "lavfi", "-i", "testsrc=size=320x180:rate=60:duration=4"],
      ...["-f", "lavfi", "-i", "sine=frequency=330:sample_rate=48000:duration=4"],
      ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"],
      ...["-c:a", "aac", "-shortest", `file:${fast}`],
    ]);
    const registration = operation("export_video");
    const job = await fakeMediaJobContext(
      registration.inputSchema.parse({
        artifact_id: "fast",
        platforms: ["tiktok", "snapchat-spotlight"],
      }),
      { fast: { path: fast, name: "Fast clip.mp4", mimeType: "video/mp4" } },
    );
    try {
      const result = await registration.handler.run(job.context);
      expect(job.skipped).toEqual([
        {
          platform: "snapchat-spotlight",
          issues: [{ code: "too_short", minMs: 5000, actualMs: expect.any(Number) }],
        },
      ]);
      expect(job.outputs).toHaveLength(1);
      const [output] = job.outputs;
      expect(output).toMatchObject({
        name: "Fast clip-tiktok.mp4",
        platform: "tiktok",
        width: 1080,
        height: 1920,
        parentArtifactId: "fast",
        check: { status: "pass", issues: [] },
      });
      expect(output!.posterSize).toBeGreaterThan(0);
      const probe = await probeMedia(output!.path, { mimeType: "video/mp4" });
      expect(probe.video).toMatchObject({ codec: "h264", width: 1080, height: 1920, fps: 30 });
      expect(Math.abs(probe.durationMs - 4000)).toBeLessThan(100);
      const loudness = await measureLoudness(
        output!.path,
        { integratedLufs: -14, truePeakDb: -1 },
        { mimeType: "video/mp4" },
      );
      expect(Math.abs(loudness.integratedLufs + 14)).toBeLessThanOrEqual(LOUDNESS_TOLERANCE_LU);
      expect(result.summary).toContain("Fast clip-tiktok.mp4 for TikTok (1080×1920, 4 s)");
      expect(result.summary).toContain(
        "Not exported for Snapchat Spotlight: 4 s is shorter than its 5 s minimum; use a longer video.",
      );
      expect(job.progress.at(-1)).toBe(1);
    } finally {
      await rm(job.tempDir, { recursive: true, force: true });
    }
  });

  it("fails without rendering when the video fits none of the platforms", async () => {
    const registration = operation("export_video");
    const job = await fakeMediaJobContext(
      registration.inputSchema.parse({ artifact_id: "land", platforms: ["snapchat-spotlight"] }),
      artifacts,
    );
    await expect(registration.handler.run(job.context)).rejects.toThrow(
      "Not exported for Snapchat Spotlight: 3 s is shorter than its 5 s minimum; use a longer video.",
    );
    expect(job.skipped.map((target) => target.platform)).toEqual(["snapchat-spotlight"]);
    expect(job.outputs).toEqual([]);
    await rm(job.tempDir, { recursive: true, force: true });
  });

  it("fails clearly on ranges past the end, non-subtitle files and non-videos", async () => {
    await expect(
      run("edit_video", {
        artifact_id: "land",
        title: "x",
        steps: [{ op: "trim", start: 5, end: 6 }],
      }),
    ).rejects.toThrow("trim 5–6 s is outside the 3 s video.");
    await expect(
      run("edit_video", {
        artifact_id: "land",
        title: "x",
        steps: [{ op: "subtitles", artifact_id: "land" }],
      }),
    ).rejects.toThrow("Landscape clip.mp4 is not an SRT or VTT file.");
    await expect(
      run("edit_video", {
        artifact_id: "land",
        title: "x",
        steps: [{ op: "overlay", artifact_id: "srt" }],
      }),
    ).rejects.toThrow("captions.srt is not a PNG, JPEG or WebP image.");
    await expect(
      run("export_video", { artifact_id: "notes", presets: ["square"] }),
    ).rejects.toThrow("notes.txt is not a video.");
  });

  it("stops ffmpeg and stores nothing when the job is cancelled", async () => {
    const registration = operation("export_video");
    const job = await fakeMediaJobContext(
      registration.inputSchema.parse({ artifact_id: "land", presets: ["cinema"], fit: "blur" }),
      artifacts,
    );
    const progress = job.context.progress;
    job.context.progress = async (fraction) => {
      await progress(fraction);
      job.controller.abort(new Error("cancelled"));
    };
    await expect(registration.handler.run(job.context)).rejects.toThrow("cancelled");
    expect(job.outputs).toEqual([]);
    await rm(job.tempDir, { recursive: true, force: true });
  });
});