feat(media): add the ComfyUI provider, default workflows and emulator
Image and video generation run on the user's own GPU server. ComfyUI is the
common self-hosted backend, so this adds a provider for its HTTP API that the
generation operations can call.
- Core: API-format workflow templates with a small placeholder format
({{prompt}}, {{width}}, {{seed}}, ...) that is validated and filled without
evaluating anything. Defaults: text-to-image and image-to-image on core nodes
for any SD-family checkpoint, and text/image-to-video on ComfyUI's core
LTX-Video nodes; a README lists the placeholders and model files.
- Adapters: createComfyUiProvider submits a workflow, polls its history,
uploads input images and downloads outputs through the address-checked fetch
used for other self-hosted model servers, with timeouts, capped reads and
readable errors. A custom workflow from settings replaces the default it
matches. Cancelling dequeues or interrupts only its own prompt. GPU status
counts memory ComfyUI can free for itself as available.
- Contracts: input schemas for generate_image, generate_video and edit_image.
- Tests: an in-process ComfyUI emulator, a provider-neutral generation
conformance suite and provider tests, all offline.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
parent
7ff93f08bb
commit
fa0986bd2d
227
packages/adapters/src/comfyui-provider.test.ts
Normal file
227
packages/adapters/src/comfyui-provider.test.ts
Normal file
@ -0,0 +1,227 @@
|
||||
import sharp from "sharp";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ComfyUiProviderConfig } from "./comfyui-provider.js";
|
||||
import {
|
||||
createComfyUiProvider,
|
||||
describeExecutionError,
|
||||
parseGpuStatus,
|
||||
} from "./comfyui-provider.js";
|
||||
import { ComfyUiEmulator } from "./testing/comfyui-emulator.js";
|
||||
|
||||
const BASE_URL = "http://127.0.0.1:8188";
|
||||
const IMAGE = { prompt: "a lighthouse", width: 512, height: 512 };
|
||||
|
||||
function setup(config: Partial<ComfyUiProviderConfig> = {}, emulator = new ComfyUiEmulator()) {
|
||||
const provider = createComfyUiProvider({
|
||||
baseUrl: BASE_URL,
|
||||
fetch: emulator.fetch,
|
||||
pollIntervalMs: 1,
|
||||
...config,
|
||||
});
|
||||
return { provider, emulator };
|
||||
}
|
||||
|
||||
function submitted(emulator: ComfyUiEmulator, index = 0) {
|
||||
const graph = emulator.prompts[index]?.graph ?? {};
|
||||
const byClass = (classType: string) =>
|
||||
Object.values(graph).find((node) => node.class_type === classType)?.inputs ?? {};
|
||||
return { graph, byClass };
|
||||
}
|
||||
|
||||
describe("createComfyUiProvider", () => {
|
||||
it("refuses metadata hosts, public hosts and keys over public http", () => {
|
||||
expect(() => createComfyUiProvider({ baseUrl: "http://169.254.169.254" })).toThrow(
|
||||
"blocked metadata or link-local host",
|
||||
);
|
||||
expect(() => createComfyUiProvider({ baseUrl: "http://comfy.example.com" })).toThrow(
|
||||
"Public model endpoints are blocked",
|
||||
);
|
||||
expect(() => createComfyUiProvider({ baseUrl: "ftp://127.0.0.1" })).toThrow("http or https");
|
||||
});
|
||||
|
||||
it("validates custom workflows up front", () => {
|
||||
expect(() => createComfyUiProvider({ baseUrl: BASE_URL, imageWorkflow: "{" })).toThrow(
|
||||
"Workflow is not valid JSON",
|
||||
);
|
||||
expect(() =>
|
||||
createComfyUiProvider({ baseUrl: BASE_URL, videoWorkflow: '{"nodes": [], "links": []}' }),
|
||||
).toThrow("API format");
|
||||
});
|
||||
|
||||
it("keeps a reverse-proxy path prefix and sends the API key", async () => {
|
||||
const emulator = new ComfyUiEmulator({ apiKey: "comfy-key", basePath: "/comfy" });
|
||||
const { provider } = setup({ baseUrl: `${BASE_URL}/comfy/`, apiKey: "comfy-key" }, emulator);
|
||||
await provider.generateImage(IMAGE);
|
||||
expect(emulator.requests.map((request) => request.path)).toContain("prompt");
|
||||
expect(emulator.requests.every((request) => request.authorization === "Bearer comfy-key")).toBe(
|
||||
true,
|
||||
);
|
||||
const { provider: keyless } = setup({ baseUrl: `${BASE_URL}/comfy` }, emulator);
|
||||
await expect(keyless.generateImage(IMAGE)).rejects.toThrow("ComfyUI returned 401");
|
||||
});
|
||||
|
||||
it("fills the default text-to-image workflow", async () => {
|
||||
const { provider, emulator } = setup();
|
||||
await provider.generateImage({ ...IMAGE, width: 1001, negativePrompt: "fog", seed: 42 });
|
||||
const { byClass } = submitted(emulator);
|
||||
expect(byClass("CheckpointLoaderSimple").ckpt_name).toBe("sd_xl_base_1.0.safetensors");
|
||||
expect(byClass("EmptyLatentImage")).toMatchObject({ width: 1000, height: 512 });
|
||||
expect(byClass("KSampler")).toMatchObject({ seed: 42, denoise: 1 });
|
||||
const texts = emulator.prompts[0]!.graph;
|
||||
expect(Object.values(texts).map((node) => node.inputs.text)).toEqual(
|
||||
expect.arrayContaining(["a lighthouse", "fog"]),
|
||||
);
|
||||
});
|
||||
|
||||
it("uses the configured model, else the first checkpoint the video workflows do not use", async () => {
|
||||
const emulator = new ComfyUiEmulator();
|
||||
emulator.checkpoints = ["ltx-video-2b-v0.9.5.safetensors", "v1-5-pruned.safetensors"];
|
||||
emulator.checkpointListFormat = "legacy";
|
||||
await setup({}, emulator).provider.generateImage(IMAGE);
|
||||
expect(submitted(emulator, 0).byClass("CheckpointLoaderSimple").ckpt_name).toBe(
|
||||
"v1-5-pruned.safetensors",
|
||||
);
|
||||
emulator.checkpoints.push("dreamshaper.safetensors");
|
||||
await setup({ imageModel: "dreamshaper.safetensors" }, emulator).provider.generateImage(IMAGE);
|
||||
expect(submitted(emulator, 1).byClass("CheckpointLoaderSimple").ckpt_name).toBe(
|
||||
"dreamshaper.safetensors",
|
||||
);
|
||||
emulator.checkpoints = ["ltx-video-2b-v0.9.5.safetensors"];
|
||||
await expect(setup({}, emulator).provider.generateImage(IMAGE)).rejects.toThrow(
|
||||
"ComfyUI has no image checkpoint installed",
|
||||
);
|
||||
});
|
||||
|
||||
it("uploads the source image for image-to-image", async () => {
|
||||
const { provider, emulator } = setup();
|
||||
const bytes = new Uint8Array(Buffer.from((await sourcePng()).bytes));
|
||||
await provider.generateImage({
|
||||
...IMAGE,
|
||||
source: { bytes, mimeType: "image/png" },
|
||||
strength: 0.3,
|
||||
});
|
||||
const { byClass } = submitted(emulator);
|
||||
const uploaded = String(byClass("LoadImage").image);
|
||||
expect(uploaded).toMatch(/^evoo-.+\.png$/);
|
||||
expect(Buffer.from(emulator.uploads.get(uploaded)!).equals(Buffer.from(bytes))).toBe(true);
|
||||
expect(byClass("KSampler").denoise).toBe(0.3);
|
||||
});
|
||||
|
||||
it("lets a custom workflow replace only the default it matches", async () => {
|
||||
const custom = {
|
||||
"1": {
|
||||
class_type: "CustomTextToImage",
|
||||
inputs: { text: "style: {{prompt}}", seed: "{{seed}}" },
|
||||
},
|
||||
"2": { class_type: "SaveImage", inputs: { images: ["1", 0], filename_prefix: "x" } },
|
||||
};
|
||||
const { provider, emulator } = setup({ imageWorkflow: JSON.stringify(custom) });
|
||||
await provider.generateImage(IMAGE);
|
||||
expect(submitted(emulator, 0).byClass("CustomTextToImage").text).toBe("style: a lighthouse");
|
||||
await provider.generateImage({ ...IMAGE, source: await sourcePng() });
|
||||
expect(submitted(emulator, 1).byClass("LoadImage").image).toBeDefined();
|
||||
});
|
||||
|
||||
it("fills the LTX-Video workflow with an 8n+1 frame count and reports the clip", async () => {
|
||||
const { provider, emulator } = setup();
|
||||
const [clip] = await provider.generateVideo({
|
||||
prompt: "a paper boat",
|
||||
width: 470,
|
||||
height: 836,
|
||||
durationSeconds: 5,
|
||||
});
|
||||
const { byClass } = submitted(emulator);
|
||||
expect(byClass("EmptyLTXVLatentVideo")).toMatchObject({ width: 480, height: 832, length: 121 });
|
||||
expect(byClass("CreateVideo").fps).toBe(24);
|
||||
expect(clip).toMatchObject({
|
||||
mimeType: "video/mp4",
|
||||
width: 480,
|
||||
height: 832,
|
||||
durationMs: 5042,
|
||||
});
|
||||
});
|
||||
|
||||
it("reads video from a custom workflow's gifs output and skips frame previews", async () => {
|
||||
const custom = {
|
||||
"1": { class_type: "TextToFrames", inputs: { text: "{{prompt}}" } },
|
||||
"2": { class_type: "PreviewImage", inputs: { images: ["1", 0] } },
|
||||
"3": { class_type: "VHS_VideoCombine", inputs: { images: ["1", 0], frame_rate: 16 } },
|
||||
};
|
||||
const { provider } = setup({ videoWorkflow: JSON.stringify(custom) });
|
||||
const clips = await provider.generateVideo({ ...IMAGE, durationSeconds: 2 });
|
||||
expect(clips.map((clip) => [clip.mimeType, clip.durationMs])).toEqual([
|
||||
["video/mp4", undefined],
|
||||
]);
|
||||
});
|
||||
|
||||
it("explains rejected workflows and failed runs", async () => {
|
||||
const emulator = new ComfyUiEmulator();
|
||||
const { provider } = setup({ imageModel: "missing.safetensors" }, emulator);
|
||||
await expect(provider.generateImage(IMAGE)).rejects.toThrow(
|
||||
"ComfyUI rejected the workflow: CheckpointLoaderSimple: Value not in list: ckpt_name: 'missing.safetensors' not in list",
|
||||
);
|
||||
const working = setup({}, emulator).provider;
|
||||
emulator.failNextExecution = "CUDA out of memory. Tried to allocate 2.00 GiB\nTraceback...";
|
||||
await expect(working.generateImage(IMAGE)).rejects.toThrow(
|
||||
/^ComfyUI failed in KSampler: CUDA out of memory\. Tried to allocate 2\.00 GiB$/,
|
||||
);
|
||||
emulator.failNextStatus = 500;
|
||||
await expect(working.generateImage(IMAGE)).rejects.toThrow("ComfyUI returned 500");
|
||||
});
|
||||
|
||||
it("reports an unreachable server", async () => {
|
||||
const provider = createComfyUiProvider({
|
||||
baseUrl: BASE_URL,
|
||||
fetch: async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
},
|
||||
});
|
||||
await expect(provider.generateImage(IMAGE)).rejects.toThrow("ComfyUI is unreachable");
|
||||
await expect(provider.gpuStatus()).rejects.toThrow("ComfyUI is unreachable");
|
||||
});
|
||||
|
||||
it("interrupts its own running prompt when the caller cancels", async () => {
|
||||
const emulator = new ComfyUiEmulator();
|
||||
emulator.pendingPolls = 1_000_000;
|
||||
const { provider } = setup({ pollIntervalMs: 5 }, emulator);
|
||||
const controller = new AbortController();
|
||||
const run = provider.generateImage({ ...IMAGE, signal: controller.signal });
|
||||
while (!emulator.requests.some((request) => request.path.startsWith("history/"))) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
}
|
||||
controller.abort(new Error("cancelled by user"));
|
||||
await expect(run).rejects.toThrow("cancelled by user");
|
||||
expect(emulator.deleted).toEqual(["prompt-1"]);
|
||||
expect(emulator.interrupted).toEqual(["prompt-1"]);
|
||||
});
|
||||
|
||||
it("reports GPU memory other processes leave to ComfyUI", async () => {
|
||||
const emulator = new ComfyUiEmulator();
|
||||
emulator.gpu = { totalMb: 12_288, freeMb: 3_000, torchReservedMb: 7_000, torchFreeMb: 1_000 };
|
||||
await expect(setup({}, emulator).provider.gpuStatus()).resolves.toEqual({
|
||||
vramFreeMb: 9_000,
|
||||
vramTotalMb: 12_288,
|
||||
});
|
||||
emulator.gpu = null;
|
||||
await expect(setup({}, emulator).provider.gpuStatus()).resolves.toBeUndefined();
|
||||
expect(parseGpuStatus({ devices: [{ type: "cuda", vram_total: 0, vram_free: 0 }] })).toBe(
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("names interrupted runs", () => {
|
||||
expect(describeExecutionError([["execution_interrupted", {}]])).toBe(
|
||||
"ComfyUI interrupted the run",
|
||||
);
|
||||
expect(describeExecutionError(undefined)).toBe("ComfyUI failed");
|
||||
});
|
||||
});
|
||||
|
||||
async function sourcePng() {
|
||||
const png = await sharp({
|
||||
create: { width: 32, height: 32, channels: 3, background: { r: 1, g: 2, b: 3 } },
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
return { bytes: new Uint8Array(png), mimeType: "image/png" };
|
||||
}
|
||||
596
packages/adapters/src/comfyui-provider.ts
Normal file
596
packages/adapters/src/comfyui-provider.ts
Normal file
@ -0,0 +1,596 @@
|
||||
import { randomInt, randomUUID } from "node:crypto";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import type {
|
||||
AdapterDescriptor,
|
||||
GeneratedMedia,
|
||||
GpuStatus,
|
||||
ImageGenerationCapabilities,
|
||||
ImageGenerationProvider,
|
||||
ImageGenerationRequest,
|
||||
MediaFile,
|
||||
MediaProgressListener,
|
||||
VideoGenerationCapabilities,
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationRequest,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type { AttachmentKind } from "@rakazo/contracts";
|
||||
import {
|
||||
ARTIFACT_MAX_BYTES_BY_KIND,
|
||||
attachmentKindOf,
|
||||
MEDIA_VIDEO_MAX_SECONDS,
|
||||
} from "@rakazo/contracts";
|
||||
import { attachmentExtensionForMimeType, inferAttachmentMimeType } from "@rakazo/core";
|
||||
import type { ComfyWorkflow, WorkflowTemplate, WorkflowValues } from "@rakazo/core/media/workflows";
|
||||
import {
|
||||
DEFAULT_WORKFLOWS,
|
||||
fillWorkflow,
|
||||
parseWorkflowTemplate,
|
||||
} from "@rakazo/core/media/workflows";
|
||||
import {
|
||||
assertAllowedOpenAiCompatibleRequestUrl,
|
||||
assertHttpsForKeyedOpenAiCompatibleUrl,
|
||||
} from "./openai-compatible-url.js";
|
||||
import { createOpenAiCompatibleFetch } from "./pi-openai-compatible-provider.js";
|
||||
import { readBodyCapped } from "./web-ssrf.js";
|
||||
|
||||
export const COMFYUI_PROVIDER_ID = "comfyui";
|
||||
/** Frame rate of generated video; frame counts follow LTX-Video's 8n+1 rule. */
|
||||
export const COMFYUI_VIDEO_FPS = 24;
|
||||
const MAX_IMAGE_SIDE = 2048;
|
||||
const REQUEST_TIMEOUT_MS = 30_000;
|
||||
const TRANSFER_TIMEOUT_MS = 120_000;
|
||||
const CANCEL_TIMEOUT_MS = 5_000;
|
||||
const IMAGE_PROMPT_TIMEOUT_MS = 10 * 60_000;
|
||||
const VIDEO_PROMPT_TIMEOUT_MS = 30 * 60_000;
|
||||
const POLL_INTERVAL_MS = 1_000;
|
||||
const MAX_JSON_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_ERROR_CHARS = 400;
|
||||
const MIB = 1024 * 1024;
|
||||
const IMAGE_STEPS = 25;
|
||||
const VIDEO_STEPS = 30;
|
||||
const DEFAULT_STRENGTH = 0.6;
|
||||
const DEFAULT_NEGATIVE_PROMPTS = {
|
||||
image: "blurry, low quality, distorted, watermark, text",
|
||||
video: "worst quality, inconsistent motion, blurry, jittery, distorted",
|
||||
};
|
||||
const JSON_HEADERS = { Accept: "application/json", "Content-Type": "application/json" };
|
||||
|
||||
export interface ComfyUiProviderConfig {
|
||||
baseUrl: string;
|
||||
apiKey?: string;
|
||||
/** Checkpoint for `{{model}}`; the first installed image checkpoint when omitted. */
|
||||
imageModel?: string;
|
||||
/** API-format workflow JSON that replaces the matching default image workflow. */
|
||||
imageWorkflow?: string;
|
||||
/** API-format workflow JSON that replaces the matching default video workflow. */
|
||||
videoWorkflow?: string;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
/** How long to wait between checks for a finished prompt; tests shorten it. */
|
||||
pollIntervalMs?: number;
|
||||
}
|
||||
|
||||
export interface ComfyUiProvider extends ImageGenerationProvider, VideoGenerationProvider {
|
||||
describe(): AdapterDescriptor<ImageGenerationCapabilities & VideoGenerationCapabilities>;
|
||||
gpuStatus(signal?: AbortSignal): Promise<GpuStatus | undefined>;
|
||||
}
|
||||
|
||||
type MediaKind = "image" | "video";
|
||||
|
||||
interface OutputFile {
|
||||
filename: string;
|
||||
subfolder: string;
|
||||
type: string;
|
||||
mimeType: string;
|
||||
kind: AttachmentKind;
|
||||
}
|
||||
|
||||
interface GenerationJob {
|
||||
kind: MediaKind;
|
||||
template: WorkflowTemplate;
|
||||
prompt: string;
|
||||
negativePrompt?: string;
|
||||
width: number;
|
||||
height: number;
|
||||
seed?: number;
|
||||
count: number;
|
||||
source?: MediaFile;
|
||||
strength?: number;
|
||||
frames?: number;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: MediaProgressListener;
|
||||
}
|
||||
|
||||
/**
|
||||
* Image and video generation on a self-hosted ComfyUI server through its HTTP API: submit an
|
||||
* API-format workflow, poll its history, download the outputs. Requests go through the same
|
||||
* address-checked fetch as OpenAI-compatible model servers.
|
||||
*/
|
||||
export function createComfyUiProvider(config: ComfyUiProviderConfig): ComfyUiProvider {
|
||||
return new ComfyUiClient(config);
|
||||
}
|
||||
|
||||
class ComfyUiClient implements ComfyUiProvider {
|
||||
private readonly baseUrl: URL;
|
||||
private readonly apiKey: string | undefined;
|
||||
private readonly imageModel: string | undefined;
|
||||
private readonly imageWorkflow: WorkflowTemplate | undefined;
|
||||
private readonly videoWorkflow: WorkflowTemplate | undefined;
|
||||
private readonly fetch: typeof globalThis.fetch;
|
||||
private readonly pollIntervalMs: number;
|
||||
private readonly clientId = randomUUID();
|
||||
|
||||
constructor(config: ComfyUiProviderConfig) {
|
||||
this.baseUrl = comfyUiBaseUrl(config.baseUrl);
|
||||
this.apiKey = config.apiKey?.trim() || undefined;
|
||||
assertHttpsForKeyedOpenAiCompatibleUrl(this.baseUrl, this.apiKey);
|
||||
this.imageModel = config.imageModel?.trim() || undefined;
|
||||
this.imageWorkflow = config.imageWorkflow?.trim()
|
||||
? parseWorkflowTemplate(config.imageWorkflow)
|
||||
: undefined;
|
||||
this.videoWorkflow = config.videoWorkflow?.trim()
|
||||
? parseWorkflowTemplate(config.videoWorkflow)
|
||||
: undefined;
|
||||
this.fetch = createOpenAiCompatibleFetch(config.fetch);
|
||||
this.pollIntervalMs = config.pollIntervalMs ?? POLL_INTERVAL_MS;
|
||||
}
|
||||
|
||||
describe() {
|
||||
return {
|
||||
id: COMFYUI_PROVIDER_ID,
|
||||
contractVersion: "1",
|
||||
adapterVersion: "0.1.0",
|
||||
capabilities: {
|
||||
textToImage: true,
|
||||
imageToImage: true,
|
||||
inpaint: false,
|
||||
maxWidth: MAX_IMAGE_SIDE,
|
||||
maxHeight: MAX_IMAGE_SIDE,
|
||||
textToVideo: true,
|
||||
imageToVideo: true,
|
||||
maxDurationSeconds: MEDIA_VIDEO_MAX_SECONDS,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async gpuStatus(signal?: AbortSignal): Promise<GpuStatus | undefined> {
|
||||
return parseGpuStatus(await this.getJson("system_stats", signal));
|
||||
}
|
||||
|
||||
async generateImage(request: ImageGenerationRequest): Promise<GeneratedMedia[]> {
|
||||
if (request.mask) throw new Error("ComfyUI inpainting is not supported");
|
||||
return this.generate({
|
||||
...request,
|
||||
kind: "image",
|
||||
template: pickTemplate(
|
||||
this.imageWorkflow,
|
||||
Boolean(request.source),
|
||||
DEFAULT_WORKFLOWS.textToImage,
|
||||
DEFAULT_WORKFLOWS.imageToImage,
|
||||
),
|
||||
width: roundSide(request.width, 8),
|
||||
height: roundSide(request.height, 8),
|
||||
count: request.count ?? 1,
|
||||
});
|
||||
}
|
||||
|
||||
async generateVideo(request: VideoGenerationRequest): Promise<GeneratedMedia[]> {
|
||||
const seconds = Math.min(Math.max(request.durationSeconds, 1), MEDIA_VIDEO_MAX_SECONDS);
|
||||
return this.generate({
|
||||
...request,
|
||||
kind: "video",
|
||||
template: pickTemplate(
|
||||
this.videoWorkflow,
|
||||
Boolean(request.source),
|
||||
DEFAULT_WORKFLOWS.textToVideo,
|
||||
DEFAULT_WORKFLOWS.imageToVideo,
|
||||
),
|
||||
width: roundSide(request.width, 32),
|
||||
height: roundSide(request.height, 32),
|
||||
frames: 8 * Math.round((seconds * COMFYUI_VIDEO_FPS) / 8) + 1,
|
||||
count: 1,
|
||||
});
|
||||
}
|
||||
|
||||
private async generate(job: GenerationJob): Promise<GeneratedMedia[]> {
|
||||
const { template, signal } = job;
|
||||
const values: WorkflowValues = {
|
||||
prompt: job.prompt,
|
||||
negative_prompt: job.negativePrompt ?? DEFAULT_NEGATIVE_PROMPTS[job.kind],
|
||||
width: job.width,
|
||||
height: job.height,
|
||||
steps: job.kind === "image" ? IMAGE_STEPS : VIDEO_STEPS,
|
||||
strength: job.strength ?? DEFAULT_STRENGTH,
|
||||
frames: job.frames ?? 1,
|
||||
fps: COMFYUI_VIDEO_FPS,
|
||||
};
|
||||
if (template.placeholders.has("model")) values.model = await this.imageCheckpoint(signal);
|
||||
if (template.placeholders.has("image")) {
|
||||
if (!job.source) throw new Error("ComfyUI workflow needs an input image");
|
||||
values.image = await this.upload(job.source, signal);
|
||||
}
|
||||
const sized = template.placeholders.has("width") && template.placeholders.has("height");
|
||||
const timed = template.placeholders.has("frames") && template.placeholders.has("fps");
|
||||
const firstSeed = job.seed ?? randomInt(2 ** 48 - 1);
|
||||
const results: GeneratedMedia[] = [];
|
||||
for (let index = 0; index < job.count; index += 1) {
|
||||
const graph = fillWorkflow(template, { ...values, seed: firstSeed + index });
|
||||
const outputs = await this.runPrompt(graph, job.kind, signal);
|
||||
const files = outputFiles(outputs, job.kind);
|
||||
if (files.length === 0) throw new Error(`ComfyUI produced no ${job.kind}`);
|
||||
for (const file of files) {
|
||||
results.push({
|
||||
bytes: await this.download(file, signal),
|
||||
mimeType: file.mimeType,
|
||||
name: file.filename,
|
||||
...(sized && file.kind === job.kind ? { width: job.width, height: job.height } : {}),
|
||||
...(timed && file.kind === "video"
|
||||
? { durationMs: Math.round(((values.frames ?? 1) / COMFYUI_VIDEO_FPS) * 1000) }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
job.onProgress?.((index + 1) / job.count);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Submits a prompt and waits for its outputs; a cancelled or timed-out prompt is dequeued. */
|
||||
private async runPrompt(
|
||||
graph: ComfyWorkflow,
|
||||
kind: MediaKind,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const deadline = AbortSignal.timeout(
|
||||
kind === "image" ? IMAGE_PROMPT_TIMEOUT_MS : VIDEO_PROMPT_TIMEOUT_MS,
|
||||
);
|
||||
const runSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
|
||||
const promptId = await this.submit(graph, runSignal);
|
||||
try {
|
||||
for (;;) {
|
||||
const outputs = await this.finishedOutputs(promptId, runSignal);
|
||||
if (outputs) return outputs;
|
||||
await sleep(this.pollIntervalMs, undefined, { signal: runSignal });
|
||||
}
|
||||
} catch (error) {
|
||||
if (!runSignal.aborted) throw error;
|
||||
await this.cancelPrompt(promptId);
|
||||
if (signal?.aborted) throw signal.reason ?? error;
|
||||
throw new Error("ComfyUI timed out");
|
||||
}
|
||||
}
|
||||
|
||||
private async submit(graph: ComfyWorkflow, signal: AbortSignal): Promise<string> {
|
||||
const body = JSON.stringify({ prompt: graph, client_id: this.clientId });
|
||||
return this.send("prompt", { method: "POST", headers: JSON_HEADERS, body }, signal, {
|
||||
read: async (response, readSignal) => {
|
||||
if (response.status === 400) {
|
||||
const rejection = await readJson(response, readSignal).catch(() => undefined);
|
||||
throw new Error(describeWorkflowRejection(rejection));
|
||||
}
|
||||
const result = await readOkJson(response, readSignal);
|
||||
const promptId = isRecord(result) ? result.prompt_id : undefined;
|
||||
if (typeof promptId !== "string" || !promptId) {
|
||||
throw new Error("ComfyUI did not accept the workflow");
|
||||
}
|
||||
return promptId;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private async finishedOutputs(
|
||||
promptId: string,
|
||||
signal: AbortSignal,
|
||||
): Promise<Record<string, unknown> | undefined> {
|
||||
const history = await this.getJson(`history/${encodeURIComponent(promptId)}`, signal);
|
||||
const entry = isRecord(history) ? history[promptId] : undefined;
|
||||
if (!isRecord(entry)) return undefined;
|
||||
const status = isRecord(entry.status) ? entry.status : {};
|
||||
if (status.status_str === "error") throw new Error(describeExecutionError(status.messages));
|
||||
if (status.completed === false) return undefined;
|
||||
return isRecord(entry.outputs) ? entry.outputs : {};
|
||||
}
|
||||
|
||||
/** Best effort: drops the prompt from the queue, or interrupts it when it is ours and running. */
|
||||
private async cancelPrompt(promptId: string): Promise<void> {
|
||||
const options = { timeoutMs: CANCEL_TIMEOUT_MS, read: discardBody };
|
||||
try {
|
||||
const body = JSON.stringify({ delete: [promptId] });
|
||||
await this.send("queue", { method: "POST", headers: JSON_HEADERS, body }, undefined, options);
|
||||
const queue = await this.send(
|
||||
"queue",
|
||||
{ headers: { Accept: "application/json" } },
|
||||
undefined,
|
||||
{
|
||||
timeoutMs: CANCEL_TIMEOUT_MS,
|
||||
read: readOkJson,
|
||||
},
|
||||
);
|
||||
const running =
|
||||
isRecord(queue) && Array.isArray(queue.queue_running) ? queue.queue_running : [];
|
||||
if (running.some((entry) => Array.isArray(entry) && entry[1] === promptId)) {
|
||||
const interrupt = JSON.stringify({ prompt_id: promptId });
|
||||
await this.send(
|
||||
"interrupt",
|
||||
{ method: "POST", headers: JSON_HEADERS, body: interrupt },
|
||||
undefined,
|
||||
options,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// The prompt then finishes on its own; nothing waits for it.
|
||||
}
|
||||
}
|
||||
|
||||
private async imageCheckpoint(signal: AbortSignal | undefined): Promise<string> {
|
||||
if (this.imageModel) return this.imageModel;
|
||||
const info = await this.getJson("object_info/CheckpointLoaderSimple", signal);
|
||||
const videoCheckpoints = new Set(
|
||||
[DEFAULT_WORKFLOWS.textToVideo, DEFAULT_WORKFLOWS.imageToVideo, this.videoWorkflow].flatMap(
|
||||
(template) => (template ? checkpointNames(template.graph) : []),
|
||||
),
|
||||
);
|
||||
const checkpoint = parseCheckpointList(info).find((name) => !videoCheckpoints.has(name));
|
||||
if (!checkpoint) throw new Error("ComfyUI has no image checkpoint installed");
|
||||
return checkpoint;
|
||||
}
|
||||
|
||||
private async upload(file: MediaFile, signal: AbortSignal | undefined): Promise<string> {
|
||||
const form = new FormData();
|
||||
const extension = attachmentExtensionForMimeType(file.mimeType) || ".png";
|
||||
form.append(
|
||||
"image",
|
||||
new Blob([Buffer.from(file.bytes)], { type: file.mimeType }),
|
||||
`evoo-${randomUUID()}${extension}`,
|
||||
);
|
||||
form.append("type", "input");
|
||||
form.append("overwrite", "true");
|
||||
// Serialize here: the address-checked fetch runs on the undici package, whose FormData
|
||||
// class differs from the global one.
|
||||
const encoded = new Response(form);
|
||||
const headers = {
|
||||
Accept: "application/json",
|
||||
"Content-Type": encoded.headers.get("content-type") ?? "",
|
||||
};
|
||||
const body = new Uint8Array(await encoded.arrayBuffer());
|
||||
return this.send("upload/image", { method: "POST", headers, body }, signal, {
|
||||
timeoutMs: TRANSFER_TIMEOUT_MS,
|
||||
read: async (response, readSignal) => {
|
||||
const result = await readOkJson(response, readSignal);
|
||||
const name = isRecord(result) && typeof result.name === "string" ? result.name : "";
|
||||
if (!name) throw new Error("ComfyUI did not accept the image");
|
||||
const subfolder =
|
||||
isRecord(result) && typeof result.subfolder === "string" ? result.subfolder : "";
|
||||
return subfolder ? `${subfolder}/${name}` : name;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private download(file: OutputFile, signal: AbortSignal | undefined): Promise<Uint8Array> {
|
||||
const query = new URLSearchParams({
|
||||
filename: file.filename,
|
||||
subfolder: file.subfolder,
|
||||
type: file.type,
|
||||
});
|
||||
return this.send(`view?${query}`, {}, signal, {
|
||||
timeoutMs: TRANSFER_TIMEOUT_MS,
|
||||
read: async (response, readSignal) => {
|
||||
await assertOk(response);
|
||||
return readBodyCapped(response, ARTIFACT_MAX_BYTES_BY_KIND[file.kind], readSignal);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private getJson(path: string, signal: AbortSignal | undefined): Promise<unknown> {
|
||||
return this.send(path, { headers: { Accept: "application/json" } }, signal, {
|
||||
read: readOkJson,
|
||||
});
|
||||
}
|
||||
|
||||
/** One request, with the timeout covering the response body too. */
|
||||
private async send<T>(
|
||||
path: string,
|
||||
init: {
|
||||
method?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: string | Uint8Array<ArrayBuffer>;
|
||||
},
|
||||
signal: AbortSignal | undefined,
|
||||
options: {
|
||||
timeoutMs?: number;
|
||||
read: (response: Response, signal: AbortSignal) => Promise<T>;
|
||||
},
|
||||
): Promise<T> {
|
||||
const timeout = AbortSignal.timeout(options.timeoutMs ?? REQUEST_TIMEOUT_MS);
|
||||
const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
||||
const headers: Record<string, string> = { ...init.headers };
|
||||
if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`;
|
||||
try {
|
||||
const response = await this.fetch(new URL(path, this.baseUrl).href, {
|
||||
method: init.method ?? "GET",
|
||||
headers,
|
||||
body: init.body,
|
||||
redirect: "error",
|
||||
signal: requestSignal,
|
||||
});
|
||||
return await options.read(response, requestSignal);
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
if (timeout.aborted) throw new Error("ComfyUI timed out");
|
||||
// undici reports refused or reset connections as a bare "fetch failed".
|
||||
if (error instanceof TypeError && error.message === "fetch failed") {
|
||||
throw new Error("ComfyUI is unreachable");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function comfyUiBaseUrl(raw: string): URL {
|
||||
const url = assertAllowedOpenAiCompatibleRequestUrl(raw.trim());
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
url.pathname = `${url.pathname.replace(/\/+$/, "")}/`;
|
||||
return url;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function truncate(text: string): string {
|
||||
return text.length > MAX_ERROR_CHARS ? `${text.slice(0, MAX_ERROR_CHARS - 1)}…` : text;
|
||||
}
|
||||
|
||||
function roundSide(side: number, multiple: number): number {
|
||||
const rounded = Math.round(side / multiple) * multiple;
|
||||
return Math.min(Math.max(rounded, multiple * Math.ceil(64 / multiple)), MAX_IMAGE_SIDE);
|
||||
}
|
||||
|
||||
/** A custom workflow replaces the default that matches whether it takes an input image. */
|
||||
function pickTemplate(
|
||||
custom: WorkflowTemplate | undefined,
|
||||
withImage: boolean,
|
||||
textDefault: WorkflowTemplate,
|
||||
imageDefault: WorkflowTemplate,
|
||||
): WorkflowTemplate {
|
||||
if (custom && custom.placeholders.has("image") === withImage) return custom;
|
||||
return withImage ? imageDefault : textDefault;
|
||||
}
|
||||
|
||||
function checkpointNames(graph: ComfyWorkflow): string[] {
|
||||
return Object.values(graph).flatMap((node) =>
|
||||
typeof node.inputs.ckpt_name === "string" ? [node.inputs.ckpt_name] : [],
|
||||
);
|
||||
}
|
||||
|
||||
async function discardBody(response: Response): Promise<void> {
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
}
|
||||
|
||||
async function assertOk(response: Response): Promise<void> {
|
||||
if (response.ok) return;
|
||||
await discardBody(response);
|
||||
throw new Error(`ComfyUI returned ${response.status}`);
|
||||
}
|
||||
|
||||
async function readJson(response: Response, signal: AbortSignal): Promise<unknown> {
|
||||
const bytes = await readBodyCapped(response, MAX_JSON_BYTES, signal);
|
||||
try {
|
||||
return JSON.parse(new TextDecoder().decode(bytes));
|
||||
} catch {
|
||||
throw new Error("ComfyUI returned invalid JSON");
|
||||
}
|
||||
}
|
||||
|
||||
async function readOkJson(response: Response, signal: AbortSignal): Promise<unknown> {
|
||||
await assertOk(response);
|
||||
return readJson(response, signal);
|
||||
}
|
||||
|
||||
/**
|
||||
* ComfyUI can evict its own models, so memory its PyTorch process holds counts as available;
|
||||
* only memory other processes use (another model server on the same GPU) does not.
|
||||
*/
|
||||
export function parseGpuStatus(body: unknown): GpuStatus | undefined {
|
||||
const devices = isRecord(body) && Array.isArray(body.devices) ? body.devices : [];
|
||||
for (const device of devices) {
|
||||
if (!isRecord(device) || device.type === "cpu") continue;
|
||||
const total = finiteNumber(device.vram_total);
|
||||
const free = finiteNumber(device.vram_free);
|
||||
if (!total || free === undefined) continue;
|
||||
const reserved = finiteNumber(device.torch_vram_total) ?? 0;
|
||||
const reservedFree = finiteNumber(device.torch_vram_free) ?? 0;
|
||||
const available = Math.min(total, Math.max(0, free - reservedFree + reserved));
|
||||
return { vramFreeMb: Math.floor(available / MIB), vramTotalMb: Math.floor(total / MIB) };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function finiteNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
/** Older ComfyUI lists combo values as `[[...names], {...}]`, newer as `["COMBO", {options}]`. */
|
||||
export function parseCheckpointList(body: unknown): string[] {
|
||||
const loader = isRecord(body) ? body.CheckpointLoaderSimple : undefined;
|
||||
const input = isRecord(loader) ? loader.input : undefined;
|
||||
const required = isRecord(input) ? input.required : undefined;
|
||||
const spec = isRecord(required) ? required.ckpt_name : undefined;
|
||||
if (!Array.isArray(spec)) return [];
|
||||
const names = Array.isArray(spec[0])
|
||||
? spec[0]
|
||||
: isRecord(spec[1]) && Array.isArray(spec[1].options)
|
||||
? spec[1].options
|
||||
: [];
|
||||
return names.filter((name): name is string => typeof name === "string" && name.length > 0);
|
||||
}
|
||||
|
||||
/** Saved files win over previews; video runs return clips (or animations), not frames. */
|
||||
function outputFiles(outputs: Record<string, unknown>, kind: MediaKind): OutputFile[] {
|
||||
const files: OutputFile[] = [];
|
||||
for (const node of Object.values(outputs)) {
|
||||
if (!isRecord(node)) continue;
|
||||
for (const key of ["images", "gifs", "videos"]) {
|
||||
const entries = node[key];
|
||||
if (!Array.isArray(entries)) continue;
|
||||
for (const entry of entries) {
|
||||
if (!isRecord(entry) || typeof entry.filename !== "string" || !entry.filename) continue;
|
||||
const mimeType = inferAttachmentMimeType(entry.filename);
|
||||
if (!mimeType) continue;
|
||||
files.push({
|
||||
filename: entry.filename,
|
||||
subfolder: typeof entry.subfolder === "string" ? entry.subfolder : "",
|
||||
type: typeof entry.type === "string" ? entry.type : "output",
|
||||
mimeType,
|
||||
kind: attachmentKindOf(mimeType),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
const videos = files.filter((file) => file.kind === "video");
|
||||
const wanted =
|
||||
kind === "image"
|
||||
? files.filter((file) => file.kind === "image")
|
||||
: videos.length > 0
|
||||
? videos
|
||||
: files.filter((file) => file.mimeType === "image/gif" || file.mimeType === "image/webp");
|
||||
const saved = wanted.filter((file) => file.type === "output");
|
||||
return saved.length > 0 ? saved : wanted;
|
||||
}
|
||||
|
||||
export function describeWorkflowRejection(body: unknown): string {
|
||||
const problems: string[] = [];
|
||||
const nodeErrors = isRecord(body) && isRecord(body.node_errors) ? body.node_errors : {};
|
||||
for (const node of Object.values(nodeErrors)) {
|
||||
if (!isRecord(node) || !Array.isArray(node.errors)) continue;
|
||||
for (const error of node.errors) {
|
||||
if (!isRecord(error)) continue;
|
||||
const parts = [node.class_type, error.message, error.details];
|
||||
problems.push(parts.filter((part) => typeof part === "string" && part).join(": "));
|
||||
}
|
||||
}
|
||||
if (problems.length === 0 && isRecord(body) && isRecord(body.error)) {
|
||||
if (typeof body.error.message === "string") problems.push(body.error.message);
|
||||
}
|
||||
return truncate(
|
||||
problems.length > 0
|
||||
? `ComfyUI rejected the workflow: ${problems.join("; ")}`
|
||||
: "ComfyUI rejected the workflow",
|
||||
);
|
||||
}
|
||||
|
||||
export function describeExecutionError(messages: unknown): string {
|
||||
const events = Array.isArray(messages) ? messages : [];
|
||||
for (const event of events) {
|
||||
if (!Array.isArray(event)) continue;
|
||||
const [name, data] = event;
|
||||
if (name === "execution_interrupted") return "ComfyUI interrupted the run";
|
||||
if (name !== "execution_error" || !isRecord(data)) continue;
|
||||
const node = typeof data.node_type === "string" ? data.node_type : "a node";
|
||||
const message =
|
||||
typeof data.exception_message === "string"
|
||||
? data.exception_message.trim().split("\n")[0]
|
||||
: "";
|
||||
return truncate(`ComfyUI failed in ${node}${message ? `: ${message}` : ""}`);
|
||||
}
|
||||
return "ComfyUI failed";
|
||||
}
|
||||
111
packages/adapters/src/image-generation-conformance.test.ts
Normal file
111
packages/adapters/src/image-generation-conformance.test.ts
Normal file
@ -0,0 +1,111 @@
|
||||
import type {
|
||||
ImageGenerationProvider,
|
||||
MediaFile,
|
||||
VideoGenerationProvider,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import { attachmentKindOf } from "@rakazo/contracts";
|
||||
import sharp from "sharp";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createComfyUiProvider } from "./comfyui-provider.js";
|
||||
import { ComfyUiEmulator } from "./testing/comfyui-emulator.js";
|
||||
|
||||
type MediaGenerationProvider = ImageGenerationProvider & VideoGenerationProvider;
|
||||
|
||||
const factories: Record<string, () => MediaGenerationProvider> = {
|
||||
comfyui: () =>
|
||||
createComfyUiProvider({
|
||||
baseUrl: "http://127.0.0.1:8188",
|
||||
fetch: new ComfyUiEmulator().fetch,
|
||||
pollIntervalMs: 1,
|
||||
}),
|
||||
};
|
||||
|
||||
async function sourceImage(): Promise<MediaFile> {
|
||||
const png = await sharp({
|
||||
create: { width: 96, height: 64, channels: 3, background: { r: 10, g: 20, b: 30 } },
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
return { bytes: new Uint8Array(png), mimeType: "image/png", name: "source.png" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Offline conformance: every image/video generation provider describes its limits, returns
|
||||
* the requested number of decodable images, is reproducible by seed, takes a source image,
|
||||
* reports progress, returns video, and honors cancellation.
|
||||
*/
|
||||
for (const [name, create] of Object.entries(factories)) {
|
||||
describe(`${name} media generation conformance (offline)`, () => {
|
||||
it("describes the contract and its limits", () => {
|
||||
const image: ImageGenerationProvider = create();
|
||||
const video: VideoGenerationProvider = create();
|
||||
expect(image.describe().contractVersion).toBe("1");
|
||||
expect(image.describe().capabilities.maxWidth).toBeGreaterThanOrEqual(1024);
|
||||
expect(image.describe().capabilities.maxHeight).toBeGreaterThanOrEqual(1024);
|
||||
expect(video.describe().contractVersion).toBe("1");
|
||||
expect(video.describe().capabilities.maxDurationSeconds).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("returns the requested number of images, reproducible by seed", async () => {
|
||||
const provider = create();
|
||||
const progress: number[] = [];
|
||||
const request = { prompt: "a red kite", width: 256, height: 192, count: 2, seed: 7 };
|
||||
const images = await provider.generateImage({
|
||||
...request,
|
||||
onProgress: (fraction) => progress.push(fraction),
|
||||
});
|
||||
expect(images).toHaveLength(2);
|
||||
for (const image of images) {
|
||||
expect(attachmentKindOf(image.mimeType)).toBe("image");
|
||||
const metadata = await sharp(image.bytes).metadata();
|
||||
expect([metadata.width, metadata.height]).toEqual([256, 192]);
|
||||
}
|
||||
expect(Buffer.from(images[0]!.bytes).equals(Buffer.from(images[1]!.bytes))).toBe(false);
|
||||
expect(progress.at(-1)).toBe(1);
|
||||
const again = await provider.generateImage({ ...request, count: 1 });
|
||||
expect(Buffer.from(again[0]!.bytes).equals(Buffer.from(images[0]!.bytes))).toBe(true);
|
||||
});
|
||||
|
||||
it("generates from a source image", async () => {
|
||||
const images = await create().generateImage({
|
||||
prompt: "the same scene at night",
|
||||
width: 96,
|
||||
height: 64,
|
||||
source: await sourceImage(),
|
||||
strength: 0.5,
|
||||
});
|
||||
expect(images).toHaveLength(1);
|
||||
expect(attachmentKindOf(images[0]!.mimeType)).toBe("image");
|
||||
});
|
||||
|
||||
it("returns video from text and from a start frame", async () => {
|
||||
const provider = create();
|
||||
for (const source of [undefined, await sourceImage()]) {
|
||||
const [clip, ...rest] = await provider.generateVideo({
|
||||
prompt: "waves rolling onto a beach",
|
||||
width: 768,
|
||||
height: 512,
|
||||
durationSeconds: 2,
|
||||
source,
|
||||
});
|
||||
expect(rest).toHaveLength(0);
|
||||
expect(attachmentKindOf(clip!.mimeType)).toBe("video");
|
||||
expect(clip!.bytes.byteLength).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("reports GPU memory with free not above total", async () => {
|
||||
const status = await create().gpuStatus?.();
|
||||
if (!status) return;
|
||||
expect(status.vramTotalMb).toBeGreaterThan(0);
|
||||
expect(status.vramFreeMb).toBeGreaterThanOrEqual(0);
|
||||
expect(status.vramFreeMb).toBeLessThanOrEqual(status.vramTotalMb);
|
||||
});
|
||||
|
||||
it("rejects when the caller cancels", async () => {
|
||||
const request = { prompt: "x", width: 64, height: 64, signal: AbortSignal.abort() };
|
||||
await expect(create().generateImage(request)).rejects.toThrow();
|
||||
await expect(create().generateVideo({ ...request, durationSeconds: 1 })).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -23,6 +23,7 @@ export * from "./cloud-agent-provider-env.js";
|
||||
export * from "./cloud-agent-service.js";
|
||||
export * from "./cloud-agent-tools.js";
|
||||
export * from "./cloud-agent-tools-select.js";
|
||||
export * from "./comfyui-provider.js";
|
||||
export * from "./composio-catalog-cache.js";
|
||||
export * from "./composio-connector.js";
|
||||
export * from "./composio-emulator.js";
|
||||
|
||||
393
packages/adapters/src/testing/comfyui-emulator.ts
Normal file
393
packages/adapters/src/testing/comfyui-emulator.ts
Normal file
@ -0,0 +1,393 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
/** A 1 s, 32×32 H.264 MP4 (ffmpeg testsrc), small enough to inline. */
|
||||
const TINY_MP4_BASE64 = [
|
||||
"AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAM+bW9vdgAAAGxtdmhkAAAAAAAAAAAAAAAAAAAD6AAAA+gAAQAAAQAA",
|
||||
"AAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAA",
|
||||
"Ao10cmFrAAAAXHRraGQAAAADAAAAAAAAAAAAAAABAAAAAAAAA+gAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAA",
|
||||
"AAAAAAAAAAAAAABAAAAAACAAAAAgAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAAPoAAAQAAABAAAAAAIFbWRpYQAAACBtZGhk",
|
||||
"AAAAAAAAAAAAAAAAAABAAAAAQABVxAAAAAAALWhkbHIAAAAAAAAAAHZpZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAABsG1p",
|
||||
"bmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAAACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAAAXBzdGJsAAAAwHN0c2QA",
|
||||
"AAAAAAAAAQAAALBhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAACAAIABIAAAASAAAAAAAAAABDExhdmMgbGlieDI2NAAAAAAA",
|
||||
"AAAAAAAAAAAAAAAAAAAAGP//AAAANmF2Y0MBZAAK/+EAGGdkAAqscgRJbARAAAADAEAAAAQDxIlhGAEAB2joQ4OSyLD9+PgAAAAA",
|
||||
"EHBhc3AAAAABAAAAAQAAABRidHJ0AAAAAAAADAgAAAwIAAAAGHN0dHMAAAAAAAAAAQAAAAgAAAgAAAAAFHN0c3MAAAAAAAAAAQAA",
|
||||
"AAEAAAAYY3R0cwAAAAAAAAABAAAACAAAEAAAAAAcc3RzYwAAAAAAAAABAAAAAQAAAAgAAAABAAAANHN0c3oAAAAAAAAAAAAAAAgA",
|
||||
"AADTAAAADAAAABgAAAAdAAAAGgAAAB0AAAAfAAAAFwAAABRzdGNvAAAAAAAAAAEAAANuAAAAPXVkdGEAAAA1bWV0YQAAAAAAAAAh",
|
||||
"aGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAIaWxzdAAAAAhmcmVlAAABiW1kYXQAAADPZYiBABP/115QL0x5O3/uan23",
|
||||
"okcn80QwiyZelIq0iY0T1naTwGHVgXgsN7YBwlEikXvo+AyEjKk/afikvNFJbO7NjVpKr2f7e+xmO5rxkaelhrWKVT83gKOM5wKs",
|
||||
"QoFpCKDo56fFf6IYs3U1LAHAxZY3IIAUCFca5RyJ/ibKbmkxBCXXiBuUCCuL9FhMjcqhW2misGdpwuLgknHPr/37MowWA2zcOn9B",
|
||||
"OHR5mfgyt4/0x3o95hpnI+2axZFkXHhYwK55h134UUvr+r3B9b/xAAAACEGaCC2JP6KAAAAAFEGaEEvwIGTKYTe7qicJ6jU1YvCB",
|
||||
"AAAAGUGaGGk8CB5MpgTfu1UOB2sZPRDVFTCp+IAAAAAWQZogiXwIH5MpgER/wf8hVV5nVt8U2QAAABlBmiipvAgf5MpgBFfFjke1",
|
||||
"iXbU9gFuIojBAAAAG0GaMMn8CB/5MpgARn/KNfso3F7XaeLq3d3VjQAAABNBmjjojwIH/5MpgAEIf9P6j9Ph",
|
||||
].join("");
|
||||
|
||||
export const COMFYUI_EMULATOR_MP4 = new Uint8Array(Buffer.from(TINY_MP4_BASE64, "base64"));
|
||||
|
||||
const MIB = 1024 * 1024;
|
||||
|
||||
type EmulatedNode = { class_type: string; inputs: Record<string, unknown> };
|
||||
type EmulatedGraph = Record<string, EmulatedNode>;
|
||||
type FileRef = { filename: string; subfolder: string; type: string };
|
||||
|
||||
interface EmulatedPrompt {
|
||||
id: string;
|
||||
graph: EmulatedGraph;
|
||||
polls: number;
|
||||
failure?: string;
|
||||
interrupted?: boolean;
|
||||
/** Dropped from the queue before it ran. */
|
||||
removed?: boolean;
|
||||
/** Reported as finished by the history. */
|
||||
done?: boolean;
|
||||
outputs?: Record<string, Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface EmulatedComfyUiRequest {
|
||||
method: string;
|
||||
path: string;
|
||||
authorization: string | null;
|
||||
body?: unknown;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function json(body: unknown, status = 200): Response {
|
||||
return Response.json(body, { status });
|
||||
}
|
||||
|
||||
/**
|
||||
* In-process substitute for a ComfyUI server's HTTP API, so the real provider runs in offline
|
||||
* tests: it validates submitted workflows like ComfyUI does, "runs" them after a few history
|
||||
* polls, and returns small generated PNGs or an MP4 fixture.
|
||||
*/
|
||||
export class ComfyUiEmulator {
|
||||
readonly requests: EmulatedComfyUiRequest[] = [];
|
||||
readonly prompts: EmulatedPrompt[] = [];
|
||||
readonly uploads = new Map<string, Uint8Array>();
|
||||
readonly deleted: string[] = [];
|
||||
readonly interrupted: string[] = [];
|
||||
checkpoints = ["sd_xl_base_1.0.safetensors", "ltx-video-2b-v0.9.5.safetensors"];
|
||||
/** `options` answers like current ComfyUI (`["COMBO", {options}]`), `legacy` like older ones. */
|
||||
checkpointListFormat: "options" | "legacy" = "options";
|
||||
/** null reports a CPU-only server. */
|
||||
gpu: { totalMb: number; freeMb: number; torchReservedMb: number; torchFreeMb: number } | null = {
|
||||
totalMb: 24_576,
|
||||
freeMb: 20_000,
|
||||
torchReservedMb: 0,
|
||||
torchFreeMb: 0,
|
||||
};
|
||||
/** History polls that still report a submitted prompt as running. */
|
||||
pendingPolls = 1;
|
||||
/** Answer the next request with this HTTP status. */
|
||||
failNextStatus: number | undefined;
|
||||
/** Fail the next submitted prompt while it runs, with this exception message. */
|
||||
failNextExecution: string | undefined;
|
||||
private readonly files = new Map<string, Uint8Array>();
|
||||
private nextPrompt = 1;
|
||||
|
||||
/** `basePath` serves the API under a prefix, like ComfyUI behind a reverse proxy. */
|
||||
constructor(private readonly options: { apiKey?: string; basePath?: string } = {}) {}
|
||||
|
||||
readonly fetch: typeof fetch = async (input, init) => {
|
||||
init?.signal?.throwIfAborted();
|
||||
const url = new URL(input instanceof Request ? input.url : String(input));
|
||||
const method = init?.method ?? "GET";
|
||||
const headers = new Headers(init?.headers);
|
||||
const authorization = headers.get("authorization");
|
||||
const prefix = `/${(this.options.basePath ?? "").replace(/^\/+|\/+$/g, "")}/`.replace(
|
||||
"//",
|
||||
"/",
|
||||
);
|
||||
if (!url.pathname.startsWith(prefix)) return json({ error: "Not found" }, 404);
|
||||
const path = url.pathname.slice(prefix.length);
|
||||
const body =
|
||||
typeof init?.body === "string" && headers.get("content-type")?.includes("json")
|
||||
? JSON.parse(init.body)
|
||||
: undefined;
|
||||
this.requests.push({ method, path, authorization, body });
|
||||
if (this.failNextStatus) {
|
||||
const status = this.failNextStatus;
|
||||
this.failNextStatus = undefined;
|
||||
return new Response("Upstream failure", { status });
|
||||
}
|
||||
if (this.options.apiKey && authorization !== `Bearer ${this.options.apiKey}`) {
|
||||
return json({ error: "Unauthorized" }, 401);
|
||||
}
|
||||
const route = `${method} ${path.split("/")[0]}`;
|
||||
if (route === "GET system_stats") return this.systemStats();
|
||||
if (route === "GET object_info" && path === "object_info/CheckpointLoaderSimple") {
|
||||
return this.checkpointInfo();
|
||||
}
|
||||
if (route === "POST prompt") return this.submit(body);
|
||||
if (route === "GET history") return this.history(decodeURIComponent(path.slice(8)));
|
||||
if (route === "GET view") return this.view(url.searchParams);
|
||||
if (route === "POST upload") return this.upload(input, init);
|
||||
if (route === "GET queue") return this.queue();
|
||||
if (route === "POST queue") {
|
||||
for (const id of Array.isArray(body?.delete) ? body.delete : []) {
|
||||
this.deleted.push(id);
|
||||
// Like ComfyUI, deleting only drops pending prompts; a running one needs an interrupt.
|
||||
const prompt = this.prompts.find((entry) => entry.id === id);
|
||||
if (prompt && prompt !== this.unfinished()[0]) prompt.removed = true;
|
||||
}
|
||||
return json({});
|
||||
}
|
||||
if (route === "POST interrupt") {
|
||||
const id = String(body?.prompt_id ?? "");
|
||||
this.interrupted.push(id);
|
||||
const running = this.unfinished()[0];
|
||||
if (running && (!id || running.id === id)) running.interrupted = true;
|
||||
return json({});
|
||||
}
|
||||
return json({ error: "Not found" }, 404);
|
||||
};
|
||||
|
||||
private systemStats(): Response {
|
||||
const gpu = this.gpu;
|
||||
return json({
|
||||
system: { os: "posix", comfyui_version: "0.3.60", embedded_python: false },
|
||||
devices: [
|
||||
gpu
|
||||
? {
|
||||
name: "cuda:0 Emulated GPU",
|
||||
type: "cuda",
|
||||
index: 0,
|
||||
vram_total: gpu.totalMb * MIB,
|
||||
vram_free: gpu.freeMb * MIB,
|
||||
torch_vram_total: gpu.torchReservedMb * MIB,
|
||||
torch_vram_free: gpu.torchFreeMb * MIB,
|
||||
}
|
||||
: { name: "cpu", type: "cpu", index: null, vram_total: 64 * 1024 * MIB, vram_free: 0 },
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
private checkpointInfo(): Response {
|
||||
const spec =
|
||||
this.checkpointListFormat === "legacy"
|
||||
? [this.checkpoints, { tooltip: "Checkpoint" }]
|
||||
: ["COMBO", { options: this.checkpoints, tooltip: "Checkpoint" }];
|
||||
return json({
|
||||
CheckpointLoaderSimple: {
|
||||
input: { required: { ckpt_name: spec } },
|
||||
output: ["MODEL", "CLIP", "VAE"],
|
||||
name: "CheckpointLoaderSimple",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private submit(body: unknown): Response {
|
||||
const graph = isRecord(body) ? body.prompt : undefined;
|
||||
if (!isRecord(graph) || typeof (body as { client_id?: unknown }).client_id !== "string") {
|
||||
return json(
|
||||
{ error: { type: "invalid_prompt", message: "Cannot execute because prompt is invalid" } },
|
||||
400,
|
||||
);
|
||||
}
|
||||
const nodeErrors: Record<string, unknown> = {};
|
||||
const reject = (id: string, node: unknown, message: string, details: string) => {
|
||||
nodeErrors[id] = {
|
||||
errors: [{ type: "invalid", message, details, extra_info: {} }],
|
||||
dependent_outputs: [],
|
||||
class_type: isRecord(node) ? node.class_type : undefined,
|
||||
};
|
||||
};
|
||||
for (const [id, node] of Object.entries(graph)) {
|
||||
if (!isRecord(node) || typeof node.class_type !== "string" || !isRecord(node.inputs)) {
|
||||
reject(id, node, "Invalid node", "class_type and inputs are required");
|
||||
continue;
|
||||
}
|
||||
for (const [name, value] of Object.entries(node.inputs)) {
|
||||
if (typeof value === "string" && value.includes("{{")) {
|
||||
reject(id, node, "Unfilled placeholder", `${name}: ${value}`);
|
||||
}
|
||||
if (Array.isArray(value) && !isRecord(graph[String(value[0])])) {
|
||||
reject(id, node, "Missing linked node", `${name}: ${String(value[0])}`);
|
||||
}
|
||||
}
|
||||
const checkpoint = node.inputs.ckpt_name;
|
||||
if (typeof checkpoint === "string" && !this.checkpoints.includes(checkpoint)) {
|
||||
reject(id, node, "Value not in list", `ckpt_name: '${checkpoint}' not in list`);
|
||||
}
|
||||
if (node.class_type === "LoadImage" && !this.uploads.has(String(node.inputs.image))) {
|
||||
reject(id, node, "Invalid image file", `image - ${String(node.inputs.image)}`);
|
||||
}
|
||||
}
|
||||
if (Object.keys(nodeErrors).length > 0) {
|
||||
return json(
|
||||
{
|
||||
error: {
|
||||
type: "prompt_outputs_failed_validation",
|
||||
message: "Prompt outputs failed validation",
|
||||
},
|
||||
node_errors: nodeErrors,
|
||||
},
|
||||
400,
|
||||
);
|
||||
}
|
||||
const id = `prompt-${this.nextPrompt++}`;
|
||||
this.prompts.push({
|
||||
id,
|
||||
graph: graph as EmulatedGraph,
|
||||
polls: 0,
|
||||
failure: this.failNextExecution,
|
||||
});
|
||||
this.failNextExecution = undefined;
|
||||
return json({ prompt_id: id, number: this.nextPrompt, node_errors: {} });
|
||||
}
|
||||
|
||||
private async history(id: string): Promise<Response> {
|
||||
const prompt = this.prompts.find((entry) => entry.id === id);
|
||||
if (!prompt || prompt.removed) return json({});
|
||||
if (!prompt.done && !prompt.interrupted && prompt.polls < this.pendingPolls) {
|
||||
prompt.polls += 1;
|
||||
return json({});
|
||||
}
|
||||
prompt.done = true;
|
||||
if (prompt.interrupted) {
|
||||
return json({
|
||||
[id]: {
|
||||
prompt: [0, id, prompt.graph, {}, []],
|
||||
outputs: {},
|
||||
status: {
|
||||
status_str: "error",
|
||||
completed: false,
|
||||
messages: [["execution_interrupted", { prompt_id: id, node_id: "5" }]],
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
if (prompt.failure) {
|
||||
return json({
|
||||
[id]: {
|
||||
prompt: [0, id, prompt.graph, {}, []],
|
||||
outputs: {},
|
||||
status: {
|
||||
status_str: "error",
|
||||
completed: false,
|
||||
messages: [
|
||||
["execution_start", { prompt_id: id }],
|
||||
[
|
||||
"execution_error",
|
||||
{
|
||||
prompt_id: id,
|
||||
node_id: "5",
|
||||
node_type: "KSampler",
|
||||
exception_message: prompt.failure,
|
||||
exception_type: "RuntimeError",
|
||||
traceback: ["..."],
|
||||
},
|
||||
],
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
prompt.outputs ??= await this.execute(prompt);
|
||||
return json({
|
||||
[id]: {
|
||||
prompt: [0, id, prompt.graph, {}, Object.keys(prompt.outputs)],
|
||||
outputs: prompt.outputs,
|
||||
status: { status_str: "success", completed: true, messages: [] },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private async execute(prompt: EmulatedPrompt): Promise<Record<string, Record<string, unknown>>> {
|
||||
const outputs: Record<string, Record<string, unknown>> = {};
|
||||
let counter = 0;
|
||||
const save = (bytes: Uint8Array, extension: string, type: string): FileRef => {
|
||||
counter += 1;
|
||||
const ref = {
|
||||
filename: `${prompt.id}_${String(counter).padStart(5, "0")}_.${extension}`,
|
||||
subfolder: type === "output" ? "evoo" : "",
|
||||
type,
|
||||
};
|
||||
this.files.set(fileKey(ref), bytes);
|
||||
return ref;
|
||||
};
|
||||
for (const [id, node] of Object.entries(prompt.graph)) {
|
||||
if (node.class_type === "SaveImage" || node.class_type === "PreviewImage") {
|
||||
const type = node.class_type === "SaveImage" ? "output" : "temp";
|
||||
outputs[id] = { images: [save(await this.renderImage(prompt.graph), "png", type)] };
|
||||
} else if (node.class_type === "SaveVideo") {
|
||||
outputs[id] = { images: [save(COMFYUI_EMULATOR_MP4, "mp4", "output")], animated: [true] };
|
||||
} else if (node.class_type === "VHS_VideoCombine") {
|
||||
outputs[id] = { gifs: [save(COMFYUI_EMULATOR_MP4, "mp4", "output")] };
|
||||
}
|
||||
}
|
||||
return outputs;
|
||||
}
|
||||
|
||||
/** A flat color derived from the seed, at the workflow's size (or its input image's). */
|
||||
private async renderImage(graph: EmulatedGraph): Promise<Uint8Array> {
|
||||
const { default: sharp } = await import("sharp");
|
||||
const nodes = Object.values(graph);
|
||||
const input = (name: string) =>
|
||||
nodes.map((node) => node.inputs[name]).find((value) => typeof value === "number");
|
||||
let width = Number(input("width") ?? 0);
|
||||
let height = Number(input("height") ?? 0);
|
||||
const loaded = nodes.find((node) => node.class_type === "LoadImage");
|
||||
if (loaded && !(width && height)) {
|
||||
const metadata = await sharp(this.uploads.get(String(loaded.inputs.image))).metadata();
|
||||
width = metadata.width ?? 64;
|
||||
height = metadata.height ?? 64;
|
||||
}
|
||||
const seed = String(input("seed") ?? input("noise_seed") ?? 0);
|
||||
const [r = 0, g = 0, b = 0] = createHash("sha256").update(seed).digest();
|
||||
const png = await sharp({
|
||||
create: { width: width || 64, height: height || 64, channels: 3, background: { r, g, b } },
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
return new Uint8Array(png);
|
||||
}
|
||||
|
||||
private view(params: URLSearchParams): Response {
|
||||
const bytes = this.files.get(
|
||||
fileKey({
|
||||
filename: params.get("filename") ?? "",
|
||||
subfolder: params.get("subfolder") ?? "",
|
||||
type: params.get("type") ?? "",
|
||||
}),
|
||||
);
|
||||
if (!bytes) return new Response("Not found", { status: 404 });
|
||||
return new Response(Buffer.from(bytes), {
|
||||
headers: { "Content-Type": "application/octet-stream" },
|
||||
});
|
||||
}
|
||||
|
||||
private async upload(input: RequestInfo | URL, init?: RequestInit): Promise<Response> {
|
||||
const form = await new Request(String(input), {
|
||||
method: "POST",
|
||||
headers: init?.headers,
|
||||
body: init?.body,
|
||||
}).formData();
|
||||
const image = form.get("image");
|
||||
if (!(image instanceof File) || form.get("type") !== "input") {
|
||||
return json({ error: "Invalid upload" }, 400);
|
||||
}
|
||||
this.uploads.set(image.name, new Uint8Array(await image.arrayBuffer()));
|
||||
return json({ name: image.name, subfolder: "", type: "input" });
|
||||
}
|
||||
|
||||
private unfinished(): EmulatedPrompt[] {
|
||||
return this.prompts.filter((prompt) => !prompt.done && !prompt.removed);
|
||||
}
|
||||
|
||||
private queue(): Response {
|
||||
const running = this.unfinished();
|
||||
return json({
|
||||
queue_running: running.slice(0, 1).map((prompt) => [0, prompt.id, prompt.graph, {}, []]),
|
||||
queue_pending: running
|
||||
.slice(1)
|
||||
.map((prompt, index) => [index + 1, prompt.id, prompt.graph, {}, []]),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function fileKey(ref: FileRef): string {
|
||||
return `${ref.type}/${ref.subfolder}/${ref.filename}`;
|
||||
}
|
||||
@ -10,6 +10,7 @@ export * from "./integration-settings.js";
|
||||
export * from "./local-settings.js";
|
||||
export * from "./mcp.js";
|
||||
export * from "./media.js";
|
||||
export * from "./media-generation.js";
|
||||
export * from "./model-tiers.js";
|
||||
export * from "./openai-compatible-ui.js";
|
||||
export * from "./project-goals.js";
|
||||
|
||||
43
packages/contracts/src/media-generation.test.ts
Normal file
43
packages/contracts/src/media-generation.test.ts
Normal file
@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
EditImageInputSchema,
|
||||
GenerateImageInputSchema,
|
||||
GenerateVideoInputSchema,
|
||||
} from "./media-generation.js";
|
||||
|
||||
describe("media generation inputs", () => {
|
||||
it("fills defaults and drops unknown fields such as continue_after", () => {
|
||||
expect(GenerateImageInputSchema.parse({ prompt: " a fox ", continue_after: true })).toEqual({
|
||||
prompt: "a fox",
|
||||
count: 1,
|
||||
});
|
||||
expect(GenerateVideoInputSchema.parse({ prompt: "waves" })).toEqual({
|
||||
prompt: "waves",
|
||||
aspect: "landscape",
|
||||
durationSeconds: 5,
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds image count, size and video duration", () => {
|
||||
expect(GenerateImageInputSchema.safeParse({ prompt: "x", count: 5 }).success).toBe(false);
|
||||
expect(GenerateImageInputSchema.safeParse({ prompt: "x", width: 512 }).success).toBe(false);
|
||||
expect(
|
||||
GenerateImageInputSchema.safeParse({ prompt: "x", width: 512, height: 4096 }).success,
|
||||
).toBe(false);
|
||||
expect(GenerateVideoInputSchema.safeParse({ prompt: "x", durationSeconds: 11 }).success).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("needs at least one edit and valid edit values", () => {
|
||||
expect(EditImageInputSchema.safeParse({ artifactId: "a" }).success).toBe(false);
|
||||
expect(EditImageInputSchema.safeParse({ artifactId: "a", rotate: 45 }).success).toBe(false);
|
||||
expect(EditImageInputSchema.safeParse({ artifactId: "a", resize: {} }).success).toBe(false);
|
||||
expect(
|
||||
EditImageInputSchema.safeParse({ artifactId: "a", extend: { top: 4, color: "red" } }).success,
|
||||
).toBe(false);
|
||||
expect(
|
||||
EditImageInputSchema.parse({ artifactId: "a", resize: { aspect: "vertical" }, rotate: 90 }),
|
||||
).toEqual({ artifactId: "a", resize: { aspect: "vertical", fit: "crop" }, rotate: 90 });
|
||||
});
|
||||
});
|
||||
108
packages/contracts/src/media-generation.ts
Normal file
108
packages/contracts/src/media-generation.ts
Normal file
@ -0,0 +1,108 @@
|
||||
import * as z from "zod";
|
||||
import { Id } from "./ids.js";
|
||||
import { MediaFitSchema, MediaPresetIdSchema } from "./media.js";
|
||||
|
||||
export const MEDIA_IMAGE_MAX_COUNT = 4;
|
||||
export const MEDIA_VIDEO_MAX_SECONDS = 10;
|
||||
|
||||
const Prompt = z.string().trim().min(1).max(4000);
|
||||
const NegativePrompt = z.string().trim().max(4000).optional().describe("What to avoid.");
|
||||
const Seed = z
|
||||
.number()
|
||||
.int()
|
||||
.min(0)
|
||||
.max(Number.MAX_SAFE_INTEGER)
|
||||
.optional()
|
||||
.describe("Reuse a seed to reproduce a result.");
|
||||
const Strength = z.number().min(0).max(1).optional();
|
||||
const Aspect = MediaPresetIdSchema.describe(
|
||||
"vertical 9:16, landscape 16:9, square 1:1, portrait 4:5, classic 4:3, cinema 21:9.",
|
||||
);
|
||||
const Side = z.number().int().min(64).max(2048);
|
||||
const Pixels = z.number().int().min(0).max(16_384);
|
||||
|
||||
export const GenerateImageInputSchema = z
|
||||
.object({
|
||||
prompt: Prompt.describe("What the image shows."),
|
||||
negativePrompt: NegativePrompt,
|
||||
aspect: Aspect.optional(),
|
||||
width: Side.optional().describe("Exact width in pixels, with height; overrides aspect."),
|
||||
height: Side.optional(),
|
||||
count: z.number().int().min(1).max(MEDIA_IMAGE_MAX_COUNT).default(1),
|
||||
seed: Seed,
|
||||
sourceArtifactId: Id.optional().describe("Image artifact to start from (image-to-image)."),
|
||||
strength: Strength.describe("With a source: how far to move away from it (default 0.6)."),
|
||||
})
|
||||
.refine((input) => (input.width === undefined) === (input.height === undefined), {
|
||||
message: "Set width and height together",
|
||||
path: ["width"],
|
||||
});
|
||||
export type GenerateImageInput = z.infer<typeof GenerateImageInputSchema>;
|
||||
|
||||
export const GenerateVideoInputSchema = z.object({
|
||||
prompt: Prompt.describe("What happens in the clip: subject, motion, camera."),
|
||||
negativePrompt: NegativePrompt,
|
||||
aspect: Aspect.default("landscape"),
|
||||
durationSeconds: z.number().min(1).max(MEDIA_VIDEO_MAX_SECONDS).default(5),
|
||||
seed: Seed,
|
||||
sourceArtifactId: Id.optional().describe("Image artifact to use as the first frame."),
|
||||
});
|
||||
export type GenerateVideoInput = z.infer<typeof GenerateVideoInputSchema>;
|
||||
|
||||
export const ImageFormatSchema = z.enum(["png", "jpeg", "webp"]);
|
||||
export type ImageFormat = z.infer<typeof ImageFormatSchema>;
|
||||
|
||||
/** Edits apply in this order: crop, rotate, flip, resize, extend, repaint, format. */
|
||||
export const EditImageInputSchema = z
|
||||
.object({
|
||||
artifactId: Id.describe("Image artifact to edit."),
|
||||
crop: z
|
||||
.object({ left: Pixels, top: Pixels, width: Pixels.min(1), height: Pixels.min(1) })
|
||||
.optional()
|
||||
.describe("Area to keep, in source pixels."),
|
||||
rotate: z.literal([90, 180, 270]).optional().describe("Clockwise, in degrees."),
|
||||
flip: z.enum(["horizontal", "vertical"]).optional(),
|
||||
resize: z
|
||||
.object({
|
||||
aspect: Aspect.optional().describe("Exact size of this format."),
|
||||
width: z.number().int().min(1).max(8192).optional(),
|
||||
height: z.number().int().min(1).max(8192).optional(),
|
||||
fit: MediaFitSchema.default("crop").describe(
|
||||
"How a different aspect ratio fills the size: crop, pad, or blur (blurred background).",
|
||||
),
|
||||
})
|
||||
.refine((resize) => resize.aspect || resize.width || resize.height, {
|
||||
message: "Set aspect, width or height",
|
||||
})
|
||||
.optional()
|
||||
.describe("Width or height alone keeps the aspect ratio."),
|
||||
extend: z
|
||||
.object({
|
||||
top: Pixels.default(0),
|
||||
right: Pixels.default(0),
|
||||
bottom: Pixels.default(0),
|
||||
left: Pixels.default(0),
|
||||
color: z
|
||||
.string()
|
||||
.regex(/^#[0-9a-fA-F]{6}$/)
|
||||
.optional()
|
||||
.describe("#RRGGBB; transparent when omitted and the format allows it."),
|
||||
})
|
||||
.optional()
|
||||
.describe("Borders to add, in pixels."),
|
||||
prompt: Prompt.optional().describe("Repaint the image with this prompt."),
|
||||
strength: Strength.describe("How far the repaint moves away from the image (default 0.6)."),
|
||||
format: ImageFormatSchema.optional().describe("Default keeps the source format."),
|
||||
})
|
||||
.refine(
|
||||
(input) =>
|
||||
input.crop ||
|
||||
input.rotate ||
|
||||
input.flip ||
|
||||
input.resize ||
|
||||
input.extend ||
|
||||
input.prompt ||
|
||||
input.format,
|
||||
{ message: "Nothing to edit" },
|
||||
);
|
||||
export type EditImageInput = z.infer<typeof EditImageInputSchema>;
|
||||
@ -34,6 +34,11 @@
|
||||
"development": "./src/plot/plot-spec.ts",
|
||||
"default": "./src/plot/plot-spec.ts"
|
||||
},
|
||||
"./media/workflows": {
|
||||
"types": "./src/media/workflow-template.ts",
|
||||
"development": "./src/media/workflow-template.ts",
|
||||
"default": "./src/media/workflow-template.ts"
|
||||
},
|
||||
"./node/desktop-runtime": {
|
||||
"types": "./src/node/desktop-runtime.ts",
|
||||
"development": "./src/node/desktop-runtime.ts",
|
||||
|
||||
113
packages/core/src/media/workflow-template.test.ts
Normal file
113
packages/core/src/media/workflow-template.test.ts
Normal file
@ -0,0 +1,113 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
DEFAULT_WORKFLOWS,
|
||||
fillWorkflow,
|
||||
parseWorkflowTemplate,
|
||||
WorkflowTemplateError,
|
||||
} from "./workflow-template.js";
|
||||
|
||||
const minimal = {
|
||||
"1": { class_type: "CLIPTextEncode", inputs: { text: "{{prompt}}", clip: ["2", 1] } },
|
||||
"2": { class_type: "EmptyLatentImage", inputs: { width: "{{width}}", height: 512 } },
|
||||
"3": { class_type: "SaveImage", inputs: { filename_prefix: "run/{{seed}}-{{prompt}}" } },
|
||||
};
|
||||
|
||||
describe("parseWorkflowTemplate", () => {
|
||||
it("accepts JSON text or parsed graphs and lists the placeholders", () => {
|
||||
for (const source of [minimal, JSON.stringify(minimal)]) {
|
||||
const template = parseWorkflowTemplate(source);
|
||||
expect([...template.placeholders].sort()).toEqual(["prompt", "seed", "width"]);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
["not json", "Workflow is not valid JSON"],
|
||||
["[]", "Workflow must be a JSON object"],
|
||||
["{}", "Workflow must have 1 to 1000 nodes"],
|
||||
[JSON.stringify({ nodes: [], links: [] }), "Workflow must be exported in ComfyUI's API format"],
|
||||
[JSON.stringify({ "1": { inputs: {} } }), "Workflow node 1 needs a class_type and inputs"],
|
||||
[
|
||||
JSON.stringify({ "1": { class_type: "SaveImage", inputs: { text: "plain" } } }),
|
||||
"Workflow needs a {{prompt}} placeholder",
|
||||
],
|
||||
[
|
||||
JSON.stringify({
|
||||
"1": { class_type: "X", inputs: { a: "{{prompt}}", b: "{{cfg}}" } },
|
||||
}),
|
||||
"Unknown workflow placeholders: {{cfg}}",
|
||||
],
|
||||
])("rejects %s", (source, message) => {
|
||||
expect(() => parseWorkflowTemplate(source)).toThrow(new WorkflowTemplateError(message));
|
||||
});
|
||||
|
||||
it("rejects placeholder names that only exist on Object.prototype", () => {
|
||||
const source = {
|
||||
"1": {
|
||||
class_type: "X",
|
||||
inputs: { a: "{{prompt}}", b: "{{constructor}}", c: "{{__proto__}}" },
|
||||
},
|
||||
};
|
||||
expect(() => parseWorkflowTemplate(source)).toThrow(
|
||||
"Unknown workflow placeholders: {{constructor}}, {{__proto__}}",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects oversized workflow text", () => {
|
||||
expect(() => parseWorkflowTemplate(" ".repeat(1_000_001))).toThrow("Workflow is too large");
|
||||
});
|
||||
});
|
||||
|
||||
describe("fillWorkflow", () => {
|
||||
it("replaces whole values with typed values and embedded ones with text, without mutating", () => {
|
||||
const template = parseWorkflowTemplate(minimal);
|
||||
const graph = fillWorkflow(template, { prompt: "a $& cat", width: 768, seed: 7 });
|
||||
expect(graph["1"]?.inputs).toEqual({ text: "a $& cat", clip: ["2", 1] });
|
||||
expect(graph["2"]?.inputs).toEqual({ width: 768, height: 512 });
|
||||
expect(graph["3"]?.inputs.filename_prefix).toBe("run/7-a $& cat");
|
||||
expect(minimal["2"].inputs.width).toBe("{{width}}");
|
||||
});
|
||||
|
||||
it("refuses missing or mistyped values", () => {
|
||||
const template = parseWorkflowTemplate(minimal);
|
||||
expect(() => fillWorkflow(template, { prompt: "cat", seed: 1 })).toThrow(
|
||||
"Workflow value for {{width}} is missing",
|
||||
);
|
||||
expect(() => fillWorkflow(template, { prompt: "cat", width: 1.5, seed: 1 })).toThrow(
|
||||
"{{width}}",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("DEFAULT_WORKFLOWS", () => {
|
||||
it("only image-to-* defaults need an input image", () => {
|
||||
expect(DEFAULT_WORKFLOWS.textToImage.placeholders.has("image")).toBe(false);
|
||||
expect(DEFAULT_WORKFLOWS.textToVideo.placeholders.has("image")).toBe(false);
|
||||
expect(DEFAULT_WORKFLOWS.imageToImage.placeholders.has("image")).toBe(true);
|
||||
expect(DEFAULT_WORKFLOWS.imageToVideo.placeholders.has("image")).toBe(true);
|
||||
});
|
||||
|
||||
it("fill into graphs whose links point at existing nodes", () => {
|
||||
const values = {
|
||||
prompt: "a lighthouse",
|
||||
negative_prompt: "",
|
||||
model: "model.safetensors",
|
||||
image: "input.png",
|
||||
width: 512,
|
||||
height: 512,
|
||||
seed: 1,
|
||||
steps: 20,
|
||||
strength: 0.6,
|
||||
frames: 25,
|
||||
fps: 24,
|
||||
};
|
||||
for (const template of Object.values(DEFAULT_WORKFLOWS)) {
|
||||
const graph = fillWorkflow(template, values);
|
||||
expect(JSON.stringify(graph)).not.toContain("{{");
|
||||
for (const node of Object.values(graph)) {
|
||||
for (const input of Object.values(node.inputs)) {
|
||||
if (Array.isArray(input)) expect(graph[String(input[0])]).toBeDefined();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
159
packages/core/src/media/workflow-template.ts
Normal file
159
packages/core/src/media/workflow-template.ts
Normal file
@ -0,0 +1,159 @@
|
||||
import imageToImage from "./workflows/image-to-image.json" with { type: "json" };
|
||||
import imageToVideo from "./workflows/image-to-video.json" with { type: "json" };
|
||||
import textToImage from "./workflows/text-to-image.json" with { type: "json" };
|
||||
import textToVideo from "./workflows/text-to-video.json" with { type: "json" };
|
||||
|
||||
/**
|
||||
* Values a ComfyUI workflow template can ask for. A string value that is exactly `{{name}}`
|
||||
* becomes the typed value (so `"{{width}}"` turns into the number 1024); `{{name}}` inside a
|
||||
* longer string is replaced by its text. Templates are data: nothing is ever evaluated.
|
||||
*/
|
||||
export const WORKFLOW_PLACEHOLDERS = {
|
||||
prompt: "string",
|
||||
negative_prompt: "string",
|
||||
/** Checkpoint file name: the configured image model or the first installed checkpoint. */
|
||||
model: "string",
|
||||
/** Uploaded input image (image-to-image, image-to-video). */
|
||||
image: "string",
|
||||
width: "integer",
|
||||
height: "integer",
|
||||
seed: "integer",
|
||||
steps: "integer",
|
||||
/** 0–1 denoise for image-to-image. */
|
||||
strength: "number",
|
||||
frames: "integer",
|
||||
fps: "number",
|
||||
} as const satisfies Record<string, "string" | "integer" | "number">;
|
||||
|
||||
export type WorkflowPlaceholder = keyof typeof WORKFLOW_PLACEHOLDERS;
|
||||
export type WorkflowValues = {
|
||||
[Name in WorkflowPlaceholder]?: (typeof WORKFLOW_PLACEHOLDERS)[Name] extends "string"
|
||||
? string
|
||||
: number;
|
||||
};
|
||||
|
||||
/** One node of a workflow in ComfyUI's API format ("Export (API)" in ComfyUI). */
|
||||
export interface ComfyWorkflowNode {
|
||||
class_type: string;
|
||||
inputs: Record<string, unknown>;
|
||||
}
|
||||
export type ComfyWorkflow = Record<string, ComfyWorkflowNode>;
|
||||
|
||||
export interface WorkflowTemplate {
|
||||
readonly graph: ComfyWorkflow;
|
||||
/** The placeholders the graph uses; `image` marks a template that needs an input image. */
|
||||
readonly placeholders: ReadonlySet<WorkflowPlaceholder>;
|
||||
}
|
||||
|
||||
export class WorkflowTemplateError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "WorkflowTemplateError";
|
||||
}
|
||||
}
|
||||
|
||||
const PLACEHOLDER_PATTERN = /\{\{([a-z_]+)\}\}/g;
|
||||
const WHOLE_PLACEHOLDER = /^\{\{([a-z_]+)\}\}$/;
|
||||
const MAX_WORKFLOW_CHARS = 1_000_000;
|
||||
const MAX_WORKFLOW_NODES = 1_000;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isPlaceholder(name: string): name is WorkflowPlaceholder {
|
||||
return Object.hasOwn(WORKFLOW_PLACEHOLDERS, name);
|
||||
}
|
||||
|
||||
function visitStrings(value: unknown, visit: (text: string) => void): void {
|
||||
if (typeof value === "string") visit(value);
|
||||
else if (Array.isArray(value)) for (const entry of value) visitStrings(entry, visit);
|
||||
else if (isRecord(value)) for (const entry of Object.values(value)) visitStrings(entry, visit);
|
||||
}
|
||||
|
||||
/** Validates an API-format workflow (JSON text or parsed) and lists its placeholders. */
|
||||
export function parseWorkflowTemplate(source: unknown): WorkflowTemplate {
|
||||
let value = source;
|
||||
if (typeof source === "string") {
|
||||
if (source.length > MAX_WORKFLOW_CHARS) {
|
||||
throw new WorkflowTemplateError("Workflow is too large");
|
||||
}
|
||||
try {
|
||||
value = JSON.parse(source);
|
||||
} catch {
|
||||
throw new WorkflowTemplateError("Workflow is not valid JSON");
|
||||
}
|
||||
}
|
||||
if (!isRecord(value)) throw new WorkflowTemplateError("Workflow must be a JSON object");
|
||||
if (Array.isArray(value.nodes)) {
|
||||
throw new WorkflowTemplateError("Workflow must be exported in ComfyUI's API format");
|
||||
}
|
||||
const nodes = Object.entries(value);
|
||||
if (nodes.length === 0 || nodes.length > MAX_WORKFLOW_NODES) {
|
||||
throw new WorkflowTemplateError(`Workflow must have 1 to ${MAX_WORKFLOW_NODES} nodes`);
|
||||
}
|
||||
for (const [id, node] of nodes) {
|
||||
if (!isRecord(node) || typeof node.class_type !== "string" || !isRecord(node.inputs)) {
|
||||
throw new WorkflowTemplateError(`Workflow node ${id} needs a class_type and inputs`);
|
||||
}
|
||||
}
|
||||
const placeholders = new Set<WorkflowPlaceholder>();
|
||||
const unknown = new Set<string>();
|
||||
visitStrings(value, (text) => {
|
||||
for (const [, name = ""] of text.matchAll(PLACEHOLDER_PATTERN)) {
|
||||
if (isPlaceholder(name)) placeholders.add(name);
|
||||
else unknown.add(`{{${name}}}`);
|
||||
}
|
||||
});
|
||||
if (unknown.size > 0) {
|
||||
throw new WorkflowTemplateError(`Unknown workflow placeholders: ${[...unknown].join(", ")}`);
|
||||
}
|
||||
if (!placeholders.has("prompt")) {
|
||||
throw new WorkflowTemplateError("Workflow needs a {{prompt}} placeholder");
|
||||
}
|
||||
return { graph: value as unknown as ComfyWorkflow, placeholders };
|
||||
}
|
||||
|
||||
function replacePlaceholders(value: unknown, values: WorkflowValues): unknown {
|
||||
if (typeof value === "string") {
|
||||
const whole = WHOLE_PLACEHOLDER.exec(value)?.[1];
|
||||
if (whole) return values[whole as WorkflowPlaceholder];
|
||||
return value.replace(PLACEHOLDER_PATTERN, (_, name: string) =>
|
||||
String(values[name as WorkflowPlaceholder]),
|
||||
);
|
||||
}
|
||||
if (Array.isArray(value)) return value.map((entry) => replacePlaceholders(entry, values));
|
||||
if (isRecord(value)) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, entry]) => [key, replacePlaceholders(entry, values)]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Returns a copy of the template's graph with every placeholder replaced by its typed value. */
|
||||
export function fillWorkflow(template: WorkflowTemplate, values: WorkflowValues): ComfyWorkflow {
|
||||
for (const name of template.placeholders) {
|
||||
const value = values[name];
|
||||
const type = WORKFLOW_PLACEHOLDERS[name];
|
||||
const valid =
|
||||
type === "string"
|
||||
? typeof value === "string"
|
||||
: typeof value === "number" &&
|
||||
Number.isFinite(value) &&
|
||||
(type === "number" || Number.isInteger(value));
|
||||
if (!valid) throw new WorkflowTemplateError(`Workflow value for {{${name}}} is missing`);
|
||||
}
|
||||
return replacePlaceholders(template.graph, values) as ComfyWorkflow;
|
||||
}
|
||||
|
||||
/**
|
||||
* Built-in workflows. Images use core nodes only and run on any SD-family checkpoint; video
|
||||
* uses ComfyUI's core LTX-Video nodes (model files in workflows/README.md).
|
||||
*/
|
||||
export const DEFAULT_WORKFLOWS = {
|
||||
textToImage: parseWorkflowTemplate(textToImage),
|
||||
imageToImage: parseWorkflowTemplate(imageToImage),
|
||||
textToVideo: parseWorkflowTemplate(textToVideo),
|
||||
imageToVideo: parseWorkflowTemplate(imageToVideo),
|
||||
} as const;
|
||||
35
packages/core/src/media/workflows/README.md
Normal file
35
packages/core/src/media/workflows/README.md
Normal file
@ -0,0 +1,35 @@
|
||||
# ComfyUI workflows
|
||||
|
||||
Default workflows for `generate_image`, `generate_video` and repainting in `edit_image`, in
|
||||
ComfyUI's API format. A custom workflow in the Media settings replaces the matching default:
|
||||
one that uses `{{image}}` replaces the image-to-image (or image-to-video) default, one without
|
||||
it replaces the text-to-image (or text-to-video) default.
|
||||
|
||||
## Placeholders
|
||||
|
||||
A string value that is exactly `{{name}}` becomes the typed value; `{{name}}` inside a longer
|
||||
string is replaced by its text. Every workflow needs `{{prompt}}`; other names are optional.
|
||||
|
||||
| Placeholder | Value |
|
||||
|---|---|
|
||||
| `{{prompt}}`, `{{negative_prompt}}` | text |
|
||||
| `{{model}}` | the configured image model, or the first installed checkpoint |
|
||||
| `{{image}}` | the uploaded input image (for `LoadImage`) |
|
||||
| `{{width}}`, `{{height}}` | output size in pixels |
|
||||
| `{{seed}}`, `{{steps}}` | integers |
|
||||
| `{{strength}}` | 0–1, image-to-image denoise |
|
||||
| `{{frames}}`, `{{fps}}` | video length in frames (8n+1) and frame rate |
|
||||
|
||||
To write your own, build it in ComfyUI, replace the values above with placeholders and export it
|
||||
with **Export (API)**.
|
||||
|
||||
## Model files
|
||||
|
||||
- Images: any SD 1.5 or SDXL checkpoint in `models/checkpoints/`.
|
||||
- Video (LTX-Video, core nodes, about 12 GB of VRAM):
|
||||
- `models/checkpoints/ltx-video-2b-v0.9.5.safetensors` from
|
||||
[Lightricks/LTX-Video](https://huggingface.co/Lightricks/LTX-Video)
|
||||
- `models/text_encoders/t5xxl_fp16.safetensors` from
|
||||
[comfyanonymous/flux_text_encoders](https://huggingface.co/comfyanonymous/flux_text_encoders)
|
||||
|
||||
The video workflows need a ComfyUI release with the `CreateVideo` and `SaveVideo` nodes.
|
||||
45
packages/core/src/media/workflows/image-to-image.json
Normal file
45
packages/core/src/media/workflows/image-to-image.json
Normal file
@ -0,0 +1,45 @@
|
||||
{
|
||||
"1": {
|
||||
"class_type": "CheckpointLoaderSimple",
|
||||
"inputs": { "ckpt_name": "{{model}}" }
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": { "text": "{{prompt}}", "clip": ["1", 1] }
|
||||
},
|
||||
"3": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": { "text": "{{negative_prompt}}", "clip": ["1", 1] }
|
||||
},
|
||||
"4": {
|
||||
"class_type": "LoadImage",
|
||||
"inputs": { "image": "{{image}}" }
|
||||
},
|
||||
"5": {
|
||||
"class_type": "VAEEncode",
|
||||
"inputs": { "pixels": ["4", 0], "vae": ["1", 2] }
|
||||
},
|
||||
"6": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"positive": ["2", 0],
|
||||
"negative": ["3", 0],
|
||||
"latent_image": ["5", 0],
|
||||
"seed": "{{seed}}",
|
||||
"steps": "{{steps}}",
|
||||
"cfg": 7,
|
||||
"sampler_name": "dpmpp_2m",
|
||||
"scheduler": "karras",
|
||||
"denoise": "{{strength}}"
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": { "samples": ["6", 0], "vae": ["1", 2] }
|
||||
},
|
||||
"8": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": { "images": ["7", 0], "filename_prefix": "evoo/image" }
|
||||
}
|
||||
}
|
||||
86
packages/core/src/media/workflows/image-to-video.json
Normal file
86
packages/core/src/media/workflows/image-to-video.json
Normal file
@ -0,0 +1,86 @@
|
||||
{
|
||||
"1": {
|
||||
"class_type": "CheckpointLoaderSimple",
|
||||
"inputs": { "ckpt_name": "ltx-video-2b-v0.9.5.safetensors" }
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoader",
|
||||
"inputs": { "clip_name": "t5xxl_fp16.safetensors", "type": "ltxv" }
|
||||
},
|
||||
"3": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": { "text": "{{prompt}}", "clip": ["2", 0] }
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": { "text": "{{negative_prompt}}", "clip": ["2", 0] }
|
||||
},
|
||||
"5": {
|
||||
"class_type": "LoadImage",
|
||||
"inputs": { "image": "{{image}}" }
|
||||
},
|
||||
"6": {
|
||||
"class_type": "LTXVImgToVideo",
|
||||
"inputs": {
|
||||
"positive": ["3", 0],
|
||||
"negative": ["4", 0],
|
||||
"vae": ["1", 2],
|
||||
"image": ["5", 0],
|
||||
"width": "{{width}}",
|
||||
"height": "{{height}}",
|
||||
"length": "{{frames}}",
|
||||
"batch_size": 1,
|
||||
"strength": 1
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "LTXVConditioning",
|
||||
"inputs": { "positive": ["6", 0], "negative": ["6", 1], "frame_rate": "{{fps}}" }
|
||||
},
|
||||
"8": {
|
||||
"class_type": "LTXVScheduler",
|
||||
"inputs": {
|
||||
"steps": "{{steps}}",
|
||||
"max_shift": 2.05,
|
||||
"base_shift": 0.95,
|
||||
"stretch": true,
|
||||
"terminal": 0.1,
|
||||
"latent": ["6", 2]
|
||||
}
|
||||
},
|
||||
"9": {
|
||||
"class_type": "KSamplerSelect",
|
||||
"inputs": { "sampler_name": "euler" }
|
||||
},
|
||||
"10": {
|
||||
"class_type": "SamplerCustom",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"add_noise": true,
|
||||
"noise_seed": "{{seed}}",
|
||||
"cfg": 3,
|
||||
"positive": ["7", 0],
|
||||
"negative": ["7", 1],
|
||||
"sampler": ["9", 0],
|
||||
"sigmas": ["8", 0],
|
||||
"latent_image": ["6", 2]
|
||||
}
|
||||
},
|
||||
"11": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": { "samples": ["10", 0], "vae": ["1", 2] }
|
||||
},
|
||||
"12": {
|
||||
"class_type": "CreateVideo",
|
||||
"inputs": { "images": ["11", 0], "fps": "{{fps}}" }
|
||||
},
|
||||
"13": {
|
||||
"class_type": "SaveVideo",
|
||||
"inputs": {
|
||||
"video": ["12", 0],
|
||||
"filename_prefix": "evoo/video",
|
||||
"format": "mp4",
|
||||
"codec": "h264"
|
||||
}
|
||||
}
|
||||
}
|
||||
41
packages/core/src/media/workflows/text-to-image.json
Normal file
41
packages/core/src/media/workflows/text-to-image.json
Normal file
@ -0,0 +1,41 @@
|
||||
{
|
||||
"1": {
|
||||
"class_type": "CheckpointLoaderSimple",
|
||||
"inputs": { "ckpt_name": "{{model}}" }
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": { "text": "{{prompt}}", "clip": ["1", 1] }
|
||||
},
|
||||
"3": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": { "text": "{{negative_prompt}}", "clip": ["1", 1] }
|
||||
},
|
||||
"4": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": { "width": "{{width}}", "height": "{{height}}", "batch_size": 1 }
|
||||
},
|
||||
"5": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"positive": ["2", 0],
|
||||
"negative": ["3", 0],
|
||||
"latent_image": ["4", 0],
|
||||
"seed": "{{seed}}",
|
||||
"steps": "{{steps}}",
|
||||
"cfg": 7,
|
||||
"sampler_name": "dpmpp_2m",
|
||||
"scheduler": "karras",
|
||||
"denoise": 1
|
||||
}
|
||||
},
|
||||
"6": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": { "samples": ["5", 0], "vae": ["1", 2] }
|
||||
},
|
||||
"7": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": { "images": ["6", 0], "filename_prefix": "evoo/image" }
|
||||
}
|
||||
}
|
||||
77
packages/core/src/media/workflows/text-to-video.json
Normal file
77
packages/core/src/media/workflows/text-to-video.json
Normal file
@ -0,0 +1,77 @@
|
||||
{
|
||||
"1": {
|
||||
"class_type": "CheckpointLoaderSimple",
|
||||
"inputs": { "ckpt_name": "ltx-video-2b-v0.9.5.safetensors" }
|
||||
},
|
||||
"2": {
|
||||
"class_type": "CLIPLoader",
|
||||
"inputs": { "clip_name": "t5xxl_fp16.safetensors", "type": "ltxv" }
|
||||
},
|
||||
"3": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": { "text": "{{prompt}}", "clip": ["2", 0] }
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": { "text": "{{negative_prompt}}", "clip": ["2", 0] }
|
||||
},
|
||||
"5": {
|
||||
"class_type": "LTXVConditioning",
|
||||
"inputs": { "positive": ["3", 0], "negative": ["4", 0], "frame_rate": "{{fps}}" }
|
||||
},
|
||||
"6": {
|
||||
"class_type": "EmptyLTXVLatentVideo",
|
||||
"inputs": {
|
||||
"width": "{{width}}",
|
||||
"height": "{{height}}",
|
||||
"length": "{{frames}}",
|
||||
"batch_size": 1
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"class_type": "LTXVScheduler",
|
||||
"inputs": {
|
||||
"steps": "{{steps}}",
|
||||
"max_shift": 2.05,
|
||||
"base_shift": 0.95,
|
||||
"stretch": true,
|
||||
"terminal": 0.1,
|
||||
"latent": ["6", 0]
|
||||
}
|
||||
},
|
||||
"8": {
|
||||
"class_type": "KSamplerSelect",
|
||||
"inputs": { "sampler_name": "euler" }
|
||||
},
|
||||
"9": {
|
||||
"class_type": "SamplerCustom",
|
||||
"inputs": {
|
||||
"model": ["1", 0],
|
||||
"add_noise": true,
|
||||
"noise_seed": "{{seed}}",
|
||||
"cfg": 3,
|
||||
"positive": ["5", 0],
|
||||
"negative": ["5", 1],
|
||||
"sampler": ["8", 0],
|
||||
"sigmas": ["7", 0],
|
||||
"latent_image": ["6", 0]
|
||||
}
|
||||
},
|
||||
"10": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": { "samples": ["9", 0], "vae": ["1", 2] }
|
||||
},
|
||||
"11": {
|
||||
"class_type": "CreateVideo",
|
||||
"inputs": { "images": ["10", 0], "fps": "{{fps}}" }
|
||||
},
|
||||
"12": {
|
||||
"class_type": "SaveVideo",
|
||||
"inputs": {
|
||||
"video": ["11", 0],
|
||||
"filename_prefix": "evoo/video",
|
||||
"format": "mp4",
|
||||
"codec": "h264"
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user