evOo-local/packages/adapters/src/comfyui-provider.test.ts

import sharp from "sharp";
import { describe, expect, it } from "vitest";
import type { ComfyUiProviderConfig } from "./comfyui-provider.js";
import {
  createComfyUiProvider,
  describeExecutionError,
  parseGpuStatus,
} from "./comfyui-provider.js";
import { ComfyUiEmulator } from "./testing/comfyui-emulator.js";

const BASE_URL = "http://127.0.0.1:8188";
const IMAGE = { prompt: "a lighthouse", width: 512, height: 512 };

function setup(config: Partial<ComfyUiProviderConfig> = {}, emulator = new ComfyUiEmulator()) {
  const provider = createComfyUiProvider({
    baseUrl: BASE_URL,
    fetch: emulator.fetch,
    webSocket: emulator.openWebSocket,
    pollIntervalMs: 1,
    ...config,
  });
  return { provider, emulator };
}

/** Progress rounded so float sums compare exactly. */
function rounded(progress: number[]): number[] {
  return progress.map((fraction) => Number(fraction.toFixed(4)));
}

function submitted(emulator: ComfyUiEmulator, index = 0) {
  const graph = emulator.prompts[index]?.graph ?? {};
  const byClass = (classType: string) =>
    Object.values(graph).find((node) => node.class_type === classType)?.inputs ?? {};
  return { graph, byClass };
}

describe("createComfyUiProvider", () => {
  it("refuses metadata hosts, public hosts and keys over public http", () => {
    expect(() => createComfyUiProvider({ baseUrl: "http://169.254.169.254" })).toThrow(
      "blocked metadata or link-local host",
    );
    expect(() => createComfyUiProvider({ baseUrl: "http://comfy.example.com" })).toThrow(
      "Public model endpoints are blocked",
    );
    expect(() => createComfyUiProvider({ baseUrl: "ftp://127.0.0.1" })).toThrow("http or https");
  });

  it("validates custom workflows up front", () => {
    expect(() => createComfyUiProvider({ baseUrl: BASE_URL, imageWorkflow: "{" })).toThrow(
      "Workflow is not valid JSON",
    );
    expect(() =>
      createComfyUiProvider({ baseUrl: BASE_URL, videoWorkflow: '{"nodes": [], "links": []}' }),
    ).toThrow("API format");
  });

  it("keeps a reverse-proxy path prefix and sends the API key", async () => {
    const emulator = new ComfyUiEmulator({ apiKey: "comfy-key", basePath: "/comfy" });
    const { provider } = setup({ baseUrl: `${BASE_URL}/comfy/`, apiKey: "comfy-key" }, emulator);
    await provider.generateImage(IMAGE);
    expect(emulator.requests.map((request) => request.path)).toContain("prompt");
    expect(emulator.requests.every((request) => request.authorization === "Bearer comfy-key")).toBe(
      true,
    );
    const { provider: keyless } = setup({ baseUrl: `${BASE_URL}/comfy` }, emulator);
    await expect(keyless.generateImage(IMAGE)).rejects.toThrow("ComfyUI returned 401");
  });

  it("fills the default text-to-image workflow", async () => {
    const { provider, emulator } = setup();
    await provider.generateImage({ ...IMAGE, width: 1001, negativePrompt: "fog", seed: 42 });
    const { byClass } = submitted(emulator);
    expect(byClass("CheckpointLoaderSimple").ckpt_name).toBe("sd_xl_base_1.0.safetensors");
    expect(byClass("EmptyLatentImage")).toMatchObject({ width: 1000, height: 512 });
    expect(byClass("KSampler")).toMatchObject({ seed: 42, denoise: 1 });
    const texts = emulator.prompts[0]!.graph;
    expect(Object.values(texts).map((node) => node.inputs.text)).toEqual(
      expect.arrayContaining(["a lighthouse", "fog"]),
    );
  });

  it("uses the configured model, else the first checkpoint the video workflows do not use", async () => {
    const emulator = new ComfyUiEmulator();
    emulator.checkpoints = ["ltx-video-2b-v0.9.5.safetensors", "v1-5-pruned.safetensors"];
    emulator.checkpointListFormat = "legacy";
    await setup({}, emulator).provider.generateImage(IMAGE);
    expect(submitted(emulator, 0).byClass("CheckpointLoaderSimple").ckpt_name).toBe(
      "v1-5-pruned.safetensors",
    );
    emulator.checkpoints.push("dreamshaper.safetensors");
    await setup({ imageModel: "dreamshaper.safetensors" }, emulator).provider.generateImage(IMAGE);
    expect(submitted(emulator, 1).byClass("CheckpointLoaderSimple").ckpt_name).toBe(
      "dreamshaper.safetensors",
    );
    emulator.checkpoints = ["ltx-video-2b-v0.9.5.safetensors"];
    await expect(setup({}, emulator).provider.generateImage(IMAGE)).rejects.toThrow(
      "ComfyUI has no image checkpoint installed",
    );
  });

  it("uploads the source image for image-to-image", async () => {
    const { provider, emulator } = setup();
    const bytes = new Uint8Array(Buffer.from((await sourcePng()).bytes));
    await provider.generateImage({
      ...IMAGE,
      source: { bytes, mimeType: "image/png" },
      strength: 0.3,
    });
    const { byClass } = submitted(emulator);
    const uploaded = String(byClass("LoadImage").image);
    expect(uploaded).toMatch(/^evoo-.+\.png$/);
    expect(Buffer.from(emulator.uploads.get(uploaded)!).equals(Buffer.from(bytes))).toBe(true);
    expect(byClass("KSampler").denoise).toBe(0.3);
  });

  it("lets a custom workflow replace only the default it matches", async () => {
    const custom = {
      "1": {
        class_type: "CustomTextToImage",
        inputs: { text: "style: {{prompt}}", seed: "{{seed}}" },
      },
      "2": { class_type: "SaveImage", inputs: { images: ["1", 0], filename_prefix: "x" } },
    };
    const { provider, emulator } = setup({ imageWorkflow: JSON.stringify(custom) });
    await provider.generateImage(IMAGE);
    expect(submitted(emulator, 0).byClass("CustomTextToImage").text).toBe("style: a lighthouse");
    await provider.generateImage({ ...IMAGE, source: await sourcePng() });
    expect(submitted(emulator, 1).byClass("LoadImage").image).toBeDefined();
  });

  it("fills the LTX-Video workflow with an 8n+1 frame count and reports the clip", async () => {
    const { provider, emulator } = setup();
    const [clip] = await provider.generateVideo({
      prompt: "a paper boat",
      width: 470,
      height: 836,
      durationSeconds: 5,
    });
    const { byClass } = submitted(emulator);
    expect(byClass("EmptyLTXVLatentVideo")).toMatchObject({ width: 480, height: 832, length: 121 });
    expect(byClass("CreateVideo").fps).toBe(24);
    expect(clip).toMatchObject({
      mimeType: "video/mp4",
      width: 480,
      height: 832,
      durationMs: 5042,
    });
  });

  it("reads video from a custom workflow's gifs output and skips frame previews", async () => {
    const custom = {
      "1": { class_type: "TextToFrames", inputs: { text: "{{prompt}}" } },
      "2": { class_type: "PreviewImage", inputs: { images: ["1", 0] } },
      "3": { class_type: "VHS_VideoCombine", inputs: { images: ["1", 0], frame_rate: 16 } },
    };
    const { provider } = setup({ videoWorkflow: JSON.stringify(custom) });
    const clips = await provider.generateVideo({ ...IMAGE, durationSeconds: 2 });
    expect(clips.map((clip) => [clip.mimeType, clip.durationMs])).toEqual([
      ["video/mp4", undefined],
    ]);
  });

  it("explains rejected workflows and failed runs", async () => {
    const emulator = new ComfyUiEmulator();
    const { provider } = setup({ imageModel: "missing.safetensors" }, emulator);
    await expect(provider.generateImage(IMAGE)).rejects.toThrow(
      "ComfyUI rejected the workflow: CheckpointLoaderSimple: Value not in list: ckpt_name: 'missing.safetensors' not in list",
    );
    const working = setup({}, emulator).provider;
    emulator.failNextExecution = "CUDA out of memory. Tried to allocate 2.00 GiB\nTraceback...";
    await expect(working.generateImage(IMAGE)).rejects.toThrow(
      /^ComfyUI failed in KSampler: CUDA out of memory\. Tried to allocate 2\.00 GiB$/,
    );
    emulator.failNextStatus = 500;
    await expect(working.generateImage(IMAGE)).rejects.toThrow("ComfyUI returned 500");
  });

  it("reports an unreachable server", async () => {
    const provider = createComfyUiProvider({
      baseUrl: BASE_URL,
      fetch: async () => {
        throw new TypeError("fetch failed");
      },
    });
    await expect(provider.generateImage(IMAGE)).rejects.toThrow("ComfyUI is unreachable");
    await expect(provider.gpuStatus()).rejects.toThrow("ComfyUI is unreachable");
  });

  it("interrupts its own running prompt when the caller cancels", async () => {
    const emulator = new ComfyUiEmulator();
    emulator.pendingPolls = 1_000_000;
    const { provider } = setup({ pollIntervalMs: 5 }, emulator);
    const controller = new AbortController();
    const run = provider.generateImage({ ...IMAGE, signal: controller.signal });
    while (!emulator.requests.some((request) => request.path.startsWith("history/"))) {
      await new Promise((resolve) => setTimeout(resolve, 1));
    }
    controller.abort(new Error("cancelled by user"));
    await expect(run).rejects.toThrow("cancelled by user");
    expect(emulator.deleted).toEqual(["prompt-1"]);
    expect(emulator.interrupted).toEqual(["prompt-1"]);
  });

  it("reports live step progress in order, short of done until the outputs are in", async () => {
    const { provider, emulator } = setup();
    emulator.pendingPolls = 4;
    const progress: number[] = [];
    await provider.generateImage({ ...IMAGE, onProgress: (fraction) => progress.push(fraction) });
    expect(progress).toEqual([0.25, 0.5, 0.75, 0.99, 1]);
    const [socket] = emulator.sockets;
    expect(socket?.url.href).toBe(
      `ws://127.0.0.1:8188/ws?clientId=${emulator.prompts[0]?.clientId}`,
    );
    expect(socket?.closedByClient).toBe(true);
  });

  it("maps progress across the prompts of a job", async () => {
    const { provider, emulator } = setup();
    emulator.pendingPolls = 2;
    const progress: number[] = [];
    await provider.generateImage({
      ...IMAGE,
      count: 2,
      onProgress: (fraction) => progress.push(fraction),
    });
    expect(rounded(progress)).toEqual([0.25, 0.495, 0.5, 0.75, 0.995, 1]);
    expect(emulator.sockets).toHaveLength(1);
  });

  it("sends the key and keeps the path prefix on the websocket", async () => {
    const emulator = new ComfyUiEmulator({ apiKey: "comfy-key", basePath: "/comfy" });
    const { provider } = setup(
      { baseUrl: "https://127.0.0.1:8188/comfy", apiKey: "comfy-key" },
      emulator,
    );
    const progress: number[] = [];
    await provider.generateImage({ ...IMAGE, onProgress: (fraction) => progress.push(fraction) });
    const socket = emulator.sockets[0]!;
    expect(`${socket.url.origin}${socket.url.pathname}`).toBe("wss://127.0.0.1:8188/comfy/ws");
    expect(socket.authorization).toBe("Bearer comfy-key");
    expect(progress).toEqual([0.99, 1]);
  });

  it("generates as before when the websocket is unavailable", async () => {
    const emulator = new ComfyUiEmulator();
    emulator.webSocketAvailable = false;
    const progress: number[] = [];
    const onProgress = (fraction: number) => progress.push(fraction);
    await setup({}, emulator).provider.generateImage({ ...IMAGE, onProgress });
    const throwing = setup({
      webSocket: () => {
        throw new Error("no websocket");
      },
    });
    await throwing.provider.generateImage({ ...IMAGE, onProgress });
    // A substituted fetch without a matching socket opens none, rather than a real connection.
    const fetchOnly = new ComfyUiEmulator();
    await setup({ webSocket: undefined }, fetchOnly).provider.generateImage({
      ...IMAGE,
      onProgress,
    });
    expect(progress).toEqual([1, 1, 1]);
    expect(emulator.sockets.map((socket) => socket.readyState)).toEqual([3]);
    expect(fetchOnly.sockets).toEqual([]);
  });

  it("ignores stray messages, a socket that drops mid-run and a failing listener", async () => {
    const { provider, emulator } = setup();
    emulator.pendingPolls = 3;
    const progress: number[] = [];
    await provider.generateImage({
      ...IMAGE,
      onProgress: (fraction) => {
        progress.push(fraction);
        const socket = emulator.sockets[0]!;
        socket.push("not json");
        socket.push({ type: "progress", data: { value: "2", max: 3, prompt_id: "prompt-1" } });
        socket.push({ type: "progress", data: { value: 2, max: 0, prompt_id: "prompt-1" } });
        socket.push({ type: "progress", data: { value: 3, max: 3, prompt_id: "prompt-9" } });
        socket.drop();
      },
    });
    expect(rounded(progress)).toEqual([0.3333, 1]);
    const failing = setup().provider;
    await expect(
      failing.generateImage({
        ...IMAGE,
        onProgress: () => {
          throw new Error("listener failed");
        },
      }),
    ).resolves.toHaveLength(1);
  });

  it("closes the websocket when the caller cancels or the run fails", async () => {
    const emulator = new ComfyUiEmulator();
    emulator.pendingPolls = 1_000_000;
    const { provider } = setup({ pollIntervalMs: 5 }, emulator);
    const controller = new AbortController();
    const onProgress = () => undefined;
    const run = provider.generateImage({ ...IMAGE, signal: controller.signal, onProgress });
    while (!emulator.requests.some((request) => request.path.startsWith("history/"))) {
      await new Promise((resolve) => setTimeout(resolve, 1));
    }
    expect(emulator.sockets[0]?.readyState).toBe(1);
    controller.abort(new Error("cancelled by user"));
    expect(emulator.sockets[0]?.closedByClient).toBe(true);
    await expect(run).rejects.toThrow("cancelled by user");
    emulator.pendingPolls = 1;
    emulator.failNextExecution = "CUDA out of memory";
    await expect(provider.generateImage({ ...IMAGE, onProgress })).rejects.toThrow(
      "CUDA out of memory",
    );
    expect(emulator.sockets.map((socket) => socket.readyState)).toEqual([3, 3]);
  });

  it("reports GPU memory other processes leave to ComfyUI", async () => {
    const emulator = new ComfyUiEmulator();
    emulator.gpu = { totalMb: 12_288, freeMb: 3_000, torchReservedMb: 7_000, torchFreeMb: 1_000 };
    await expect(setup({}, emulator).provider.gpuStatus()).resolves.toEqual({
      vramFreeMb: 9_000,
      vramTotalMb: 12_288,
    });
    emulator.gpu = null;
    await expect(setup({}, emulator).provider.gpuStatus()).resolves.toBeUndefined();
    expect(parseGpuStatus({ devices: [{ type: "cuda", vram_total: 0, vram_free: 0 }] })).toBe(
      undefined,
    );
  });

  it("names interrupted runs", () => {
    expect(describeExecutionError([["execution_interrupted", {}]])).toBe(
      "ComfyUI interrupted the run",
    );
    expect(describeExecutionError(undefined)).toBe("ComfyUI failed");
  });
});

async function sourcePng() {
  const png = await sharp({
    create: { width: 32, height: 32, channels: 3, background: { r: 1, g: 2, b: 3 } },
  })
    .png()
    .toBuffer();
  return { bytes: new Uint8Array(png), mimeType: "image/png" };
}