diff --git a/packages/adapters/src/comfyui-provider.test.ts b/packages/adapters/src/comfyui-provider.test.ts new file mode 100644 index 0000000..2773c1f --- /dev/null +++ b/packages/adapters/src/comfyui-provider.test.ts @@ -0,0 +1,227 @@ +import sharp from "sharp"; +import { describe, expect, it } from "vitest"; +import type { ComfyUiProviderConfig } from "./comfyui-provider.js"; +import { + createComfyUiProvider, + describeExecutionError, + parseGpuStatus, +} from "./comfyui-provider.js"; +import { ComfyUiEmulator } from "./testing/comfyui-emulator.js"; + +const BASE_URL = "http://127.0.0.1:8188"; +const IMAGE = { prompt: "a lighthouse", width: 512, height: 512 }; + +function setup(config: Partial = {}, emulator = new ComfyUiEmulator()) { + const provider = createComfyUiProvider({ + baseUrl: BASE_URL, + fetch: emulator.fetch, + pollIntervalMs: 1, + ...config, + }); + return { provider, emulator }; +} + +function submitted(emulator: ComfyUiEmulator, index = 0) { + const graph = emulator.prompts[index]?.graph ?? {}; + const byClass = (classType: string) => + Object.values(graph).find((node) => node.class_type === classType)?.inputs ?? {}; + return { graph, byClass }; +} + +describe("createComfyUiProvider", () => { + it("refuses metadata hosts, public hosts and keys over public http", () => { + expect(() => createComfyUiProvider({ baseUrl: "http://169.254.169.254" })).toThrow( + "blocked metadata or link-local host", + ); + expect(() => createComfyUiProvider({ baseUrl: "http://comfy.example.com" })).toThrow( + "Public model endpoints are blocked", + ); + expect(() => createComfyUiProvider({ baseUrl: "ftp://127.0.0.1" })).toThrow("http or https"); + }); + + it("validates custom workflows up front", () => { + expect(() => createComfyUiProvider({ baseUrl: BASE_URL, imageWorkflow: "{" })).toThrow( + "Workflow is not valid JSON", + ); + expect(() => + createComfyUiProvider({ baseUrl: BASE_URL, videoWorkflow: '{"nodes": [], "links": []}' }), + ).toThrow("API format"); + }); + + it("keeps a reverse-proxy path prefix and sends the API key", async () => { + const emulator = new ComfyUiEmulator({ apiKey: "comfy-key", basePath: "/comfy" }); + const { provider } = setup({ baseUrl: `${BASE_URL}/comfy/`, apiKey: "comfy-key" }, emulator); + await provider.generateImage(IMAGE); + expect(emulator.requests.map((request) => request.path)).toContain("prompt"); + expect(emulator.requests.every((request) => request.authorization === "Bearer comfy-key")).toBe( + true, + ); + const { provider: keyless } = setup({ baseUrl: `${BASE_URL}/comfy` }, emulator); + await expect(keyless.generateImage(IMAGE)).rejects.toThrow("ComfyUI returned 401"); + }); + + it("fills the default text-to-image workflow", async () => { + const { provider, emulator } = setup(); + await provider.generateImage({ ...IMAGE, width: 1001, negativePrompt: "fog", seed: 42 }); + const { byClass } = submitted(emulator); + expect(byClass("CheckpointLoaderSimple").ckpt_name).toBe("sd_xl_base_1.0.safetensors"); + expect(byClass("EmptyLatentImage")).toMatchObject({ width: 1000, height: 512 }); + expect(byClass("KSampler")).toMatchObject({ seed: 42, denoise: 1 }); + const texts = emulator.prompts[0]!.graph; + expect(Object.values(texts).map((node) => node.inputs.text)).toEqual( + expect.arrayContaining(["a lighthouse", "fog"]), + ); + }); + + it("uses the configured model, else the first checkpoint the video workflows do not use", async () => { + const emulator = new ComfyUiEmulator(); + emulator.checkpoints = ["ltx-video-2b-v0.9.5.safetensors", "v1-5-pruned.safetensors"]; + emulator.checkpointListFormat = "legacy"; + await setup({}, emulator).provider.generateImage(IMAGE); + expect(submitted(emulator, 0).byClass("CheckpointLoaderSimple").ckpt_name).toBe( + "v1-5-pruned.safetensors", + ); + emulator.checkpoints.push("dreamshaper.safetensors"); + await setup({ imageModel: "dreamshaper.safetensors" }, emulator).provider.generateImage(IMAGE); + expect(submitted(emulator, 1).byClass("CheckpointLoaderSimple").ckpt_name).toBe( + "dreamshaper.safetensors", + ); + emulator.checkpoints = ["ltx-video-2b-v0.9.5.safetensors"]; + await expect(setup({}, emulator).provider.generateImage(IMAGE)).rejects.toThrow( + "ComfyUI has no image checkpoint installed", + ); + }); + + it("uploads the source image for image-to-image", async () => { + const { provider, emulator } = setup(); + const bytes = new Uint8Array(Buffer.from((await sourcePng()).bytes)); + await provider.generateImage({ + ...IMAGE, + source: { bytes, mimeType: "image/png" }, + strength: 0.3, + }); + const { byClass } = submitted(emulator); + const uploaded = String(byClass("LoadImage").image); + expect(uploaded).toMatch(/^evoo-.+\.png$/); + expect(Buffer.from(emulator.uploads.get(uploaded)!).equals(Buffer.from(bytes))).toBe(true); + expect(byClass("KSampler").denoise).toBe(0.3); + }); + + it("lets a custom workflow replace only the default it matches", async () => { + const custom = { + "1": { + class_type: "CustomTextToImage", + inputs: { text: "style: {{prompt}}", seed: "{{seed}}" }, + }, + "2": { class_type: "SaveImage", inputs: { images: ["1", 0], filename_prefix: "x" } }, + }; + const { provider, emulator } = setup({ imageWorkflow: JSON.stringify(custom) }); + await provider.generateImage(IMAGE); + expect(submitted(emulator, 0).byClass("CustomTextToImage").text).toBe("style: a lighthouse"); + await provider.generateImage({ ...IMAGE, source: await sourcePng() }); + expect(submitted(emulator, 1).byClass("LoadImage").image).toBeDefined(); + }); + + it("fills the LTX-Video workflow with an 8n+1 frame count and reports the clip", async () => { + const { provider, emulator } = setup(); + const [clip] = await provider.generateVideo({ + prompt: "a paper boat", + width: 470, + height: 836, + durationSeconds: 5, + }); + const { byClass } = submitted(emulator); + expect(byClass("EmptyLTXVLatentVideo")).toMatchObject({ width: 480, height: 832, length: 121 }); + expect(byClass("CreateVideo").fps).toBe(24); + expect(clip).toMatchObject({ + mimeType: "video/mp4", + width: 480, + height: 832, + durationMs: 5042, + }); + }); + + it("reads video from a custom workflow's gifs output and skips frame previews", async () => { + const custom = { + "1": { class_type: "TextToFrames", inputs: { text: "{{prompt}}" } }, + "2": { class_type: "PreviewImage", inputs: { images: ["1", 0] } }, + "3": { class_type: "VHS_VideoCombine", inputs: { images: ["1", 0], frame_rate: 16 } }, + }; + const { provider } = setup({ videoWorkflow: JSON.stringify(custom) }); + const clips = await provider.generateVideo({ ...IMAGE, durationSeconds: 2 }); + expect(clips.map((clip) => [clip.mimeType, clip.durationMs])).toEqual([ + ["video/mp4", undefined], + ]); + }); + + it("explains rejected workflows and failed runs", async () => { + const emulator = new ComfyUiEmulator(); + const { provider } = setup({ imageModel: "missing.safetensors" }, emulator); + await expect(provider.generateImage(IMAGE)).rejects.toThrow( + "ComfyUI rejected the workflow: CheckpointLoaderSimple: Value not in list: ckpt_name: 'missing.safetensors' not in list", + ); + const working = setup({}, emulator).provider; + emulator.failNextExecution = "CUDA out of memory. Tried to allocate 2.00 GiB\nTraceback..."; + await expect(working.generateImage(IMAGE)).rejects.toThrow( + /^ComfyUI failed in KSampler: CUDA out of memory\. Tried to allocate 2\.00 GiB$/, + ); + emulator.failNextStatus = 500; + await expect(working.generateImage(IMAGE)).rejects.toThrow("ComfyUI returned 500"); + }); + + it("reports an unreachable server", async () => { + const provider = createComfyUiProvider({ + baseUrl: BASE_URL, + fetch: async () => { + throw new TypeError("fetch failed"); + }, + }); + await expect(provider.generateImage(IMAGE)).rejects.toThrow("ComfyUI is unreachable"); + await expect(provider.gpuStatus()).rejects.toThrow("ComfyUI is unreachable"); + }); + + it("interrupts its own running prompt when the caller cancels", async () => { + const emulator = new ComfyUiEmulator(); + emulator.pendingPolls = 1_000_000; + const { provider } = setup({ pollIntervalMs: 5 }, emulator); + const controller = new AbortController(); + const run = provider.generateImage({ ...IMAGE, signal: controller.signal }); + while (!emulator.requests.some((request) => request.path.startsWith("history/"))) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + controller.abort(new Error("cancelled by user")); + await expect(run).rejects.toThrow("cancelled by user"); + expect(emulator.deleted).toEqual(["prompt-1"]); + expect(emulator.interrupted).toEqual(["prompt-1"]); + }); + + it("reports GPU memory other processes leave to ComfyUI", async () => { + const emulator = new ComfyUiEmulator(); + emulator.gpu = { totalMb: 12_288, freeMb: 3_000, torchReservedMb: 7_000, torchFreeMb: 1_000 }; + await expect(setup({}, emulator).provider.gpuStatus()).resolves.toEqual({ + vramFreeMb: 9_000, + vramTotalMb: 12_288, + }); + emulator.gpu = null; + await expect(setup({}, emulator).provider.gpuStatus()).resolves.toBeUndefined(); + expect(parseGpuStatus({ devices: [{ type: "cuda", vram_total: 0, vram_free: 0 }] })).toBe( + undefined, + ); + }); + + it("names interrupted runs", () => { + expect(describeExecutionError([["execution_interrupted", {}]])).toBe( + "ComfyUI interrupted the run", + ); + expect(describeExecutionError(undefined)).toBe("ComfyUI failed"); + }); +}); + +async function sourcePng() { + const png = await sharp({ + create: { width: 32, height: 32, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }) + .png() + .toBuffer(); + return { bytes: new Uint8Array(png), mimeType: "image/png" }; +} diff --git a/packages/adapters/src/comfyui-provider.ts b/packages/adapters/src/comfyui-provider.ts new file mode 100644 index 0000000..ec35de7 --- /dev/null +++ b/packages/adapters/src/comfyui-provider.ts @@ -0,0 +1,596 @@ +import { randomInt, randomUUID } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; +import type { + AdapterDescriptor, + GeneratedMedia, + GpuStatus, + ImageGenerationCapabilities, + ImageGenerationProvider, + ImageGenerationRequest, + MediaFile, + MediaProgressListener, + VideoGenerationCapabilities, + VideoGenerationProvider, + VideoGenerationRequest, +} from "@rakazo/adapter-kit"; +import type { AttachmentKind } from "@rakazo/contracts"; +import { + ARTIFACT_MAX_BYTES_BY_KIND, + attachmentKindOf, + MEDIA_VIDEO_MAX_SECONDS, +} from "@rakazo/contracts"; +import { attachmentExtensionForMimeType, inferAttachmentMimeType } from "@rakazo/core"; +import type { ComfyWorkflow, WorkflowTemplate, WorkflowValues } from "@rakazo/core/media/workflows"; +import { + DEFAULT_WORKFLOWS, + fillWorkflow, + parseWorkflowTemplate, +} from "@rakazo/core/media/workflows"; +import { + assertAllowedOpenAiCompatibleRequestUrl, + assertHttpsForKeyedOpenAiCompatibleUrl, +} from "./openai-compatible-url.js"; +import { createOpenAiCompatibleFetch } from "./pi-openai-compatible-provider.js"; +import { readBodyCapped } from "./web-ssrf.js"; + +export const COMFYUI_PROVIDER_ID = "comfyui"; +/** Frame rate of generated video; frame counts follow LTX-Video's 8n+1 rule. */ +export const COMFYUI_VIDEO_FPS = 24; +const MAX_IMAGE_SIDE = 2048; +const REQUEST_TIMEOUT_MS = 30_000; +const TRANSFER_TIMEOUT_MS = 120_000; +const CANCEL_TIMEOUT_MS = 5_000; +const IMAGE_PROMPT_TIMEOUT_MS = 10 * 60_000; +const VIDEO_PROMPT_TIMEOUT_MS = 30 * 60_000; +const POLL_INTERVAL_MS = 1_000; +const MAX_JSON_BYTES = 8 * 1024 * 1024; +const MAX_ERROR_CHARS = 400; +const MIB = 1024 * 1024; +const IMAGE_STEPS = 25; +const VIDEO_STEPS = 30; +const DEFAULT_STRENGTH = 0.6; +const DEFAULT_NEGATIVE_PROMPTS = { + image: "blurry, low quality, distorted, watermark, text", + video: "worst quality, inconsistent motion, blurry, jittery, distorted", +}; +const JSON_HEADERS = { Accept: "application/json", "Content-Type": "application/json" }; + +export interface ComfyUiProviderConfig { + baseUrl: string; + apiKey?: string; + /** Checkpoint for `{{model}}`; the first installed image checkpoint when omitted. */ + imageModel?: string; + /** API-format workflow JSON that replaces the matching default image workflow. */ + imageWorkflow?: string; + /** API-format workflow JSON that replaces the matching default video workflow. */ + videoWorkflow?: string; + fetch?: typeof globalThis.fetch; + /** How long to wait between checks for a finished prompt; tests shorten it. */ + pollIntervalMs?: number; +} + +export interface ComfyUiProvider extends ImageGenerationProvider, VideoGenerationProvider { + describe(): AdapterDescriptor; + gpuStatus(signal?: AbortSignal): Promise; +} + +type MediaKind = "image" | "video"; + +interface OutputFile { + filename: string; + subfolder: string; + type: string; + mimeType: string; + kind: AttachmentKind; +} + +interface GenerationJob { + kind: MediaKind; + template: WorkflowTemplate; + prompt: string; + negativePrompt?: string; + width: number; + height: number; + seed?: number; + count: number; + source?: MediaFile; + strength?: number; + frames?: number; + signal?: AbortSignal; + onProgress?: MediaProgressListener; +} + +/** + * Image and video generation on a self-hosted ComfyUI server through its HTTP API: submit an + * API-format workflow, poll its history, download the outputs. Requests go through the same + * address-checked fetch as OpenAI-compatible model servers. + */ +export function createComfyUiProvider(config: ComfyUiProviderConfig): ComfyUiProvider { + return new ComfyUiClient(config); +} + +class ComfyUiClient implements ComfyUiProvider { + private readonly baseUrl: URL; + private readonly apiKey: string | undefined; + private readonly imageModel: string | undefined; + private readonly imageWorkflow: WorkflowTemplate | undefined; + private readonly videoWorkflow: WorkflowTemplate | undefined; + private readonly fetch: typeof globalThis.fetch; + private readonly pollIntervalMs: number; + private readonly clientId = randomUUID(); + + constructor(config: ComfyUiProviderConfig) { + this.baseUrl = comfyUiBaseUrl(config.baseUrl); + this.apiKey = config.apiKey?.trim() || undefined; + assertHttpsForKeyedOpenAiCompatibleUrl(this.baseUrl, this.apiKey); + this.imageModel = config.imageModel?.trim() || undefined; + this.imageWorkflow = config.imageWorkflow?.trim() + ? parseWorkflowTemplate(config.imageWorkflow) + : undefined; + this.videoWorkflow = config.videoWorkflow?.trim() + ? parseWorkflowTemplate(config.videoWorkflow) + : undefined; + this.fetch = createOpenAiCompatibleFetch(config.fetch); + this.pollIntervalMs = config.pollIntervalMs ?? POLL_INTERVAL_MS; + } + + describe() { + return { + id: COMFYUI_PROVIDER_ID, + contractVersion: "1", + adapterVersion: "0.1.0", + capabilities: { + textToImage: true, + imageToImage: true, + inpaint: false, + maxWidth: MAX_IMAGE_SIDE, + maxHeight: MAX_IMAGE_SIDE, + textToVideo: true, + imageToVideo: true, + maxDurationSeconds: MEDIA_VIDEO_MAX_SECONDS, + }, + }; + } + + async gpuStatus(signal?: AbortSignal): Promise { + return parseGpuStatus(await this.getJson("system_stats", signal)); + } + + async generateImage(request: ImageGenerationRequest): Promise { + if (request.mask) throw new Error("ComfyUI inpainting is not supported"); + return this.generate({ + ...request, + kind: "image", + template: pickTemplate( + this.imageWorkflow, + Boolean(request.source), + DEFAULT_WORKFLOWS.textToImage, + DEFAULT_WORKFLOWS.imageToImage, + ), + width: roundSide(request.width, 8), + height: roundSide(request.height, 8), + count: request.count ?? 1, + }); + } + + async generateVideo(request: VideoGenerationRequest): Promise { + const seconds = Math.min(Math.max(request.durationSeconds, 1), MEDIA_VIDEO_MAX_SECONDS); + return this.generate({ + ...request, + kind: "video", + template: pickTemplate( + this.videoWorkflow, + Boolean(request.source), + DEFAULT_WORKFLOWS.textToVideo, + DEFAULT_WORKFLOWS.imageToVideo, + ), + width: roundSide(request.width, 32), + height: roundSide(request.height, 32), + frames: 8 * Math.round((seconds * COMFYUI_VIDEO_FPS) / 8) + 1, + count: 1, + }); + } + + private async generate(job: GenerationJob): Promise { + const { template, signal } = job; + const values: WorkflowValues = { + prompt: job.prompt, + negative_prompt: job.negativePrompt ?? DEFAULT_NEGATIVE_PROMPTS[job.kind], + width: job.width, + height: job.height, + steps: job.kind === "image" ? IMAGE_STEPS : VIDEO_STEPS, + strength: job.strength ?? DEFAULT_STRENGTH, + frames: job.frames ?? 1, + fps: COMFYUI_VIDEO_FPS, + }; + if (template.placeholders.has("model")) values.model = await this.imageCheckpoint(signal); + if (template.placeholders.has("image")) { + if (!job.source) throw new Error("ComfyUI workflow needs an input image"); + values.image = await this.upload(job.source, signal); + } + const sized = template.placeholders.has("width") && template.placeholders.has("height"); + const timed = template.placeholders.has("frames") && template.placeholders.has("fps"); + const firstSeed = job.seed ?? randomInt(2 ** 48 - 1); + const results: GeneratedMedia[] = []; + for (let index = 0; index < job.count; index += 1) { + const graph = fillWorkflow(template, { ...values, seed: firstSeed + index }); + const outputs = await this.runPrompt(graph, job.kind, signal); + const files = outputFiles(outputs, job.kind); + if (files.length === 0) throw new Error(`ComfyUI produced no ${job.kind}`); + for (const file of files) { + results.push({ + bytes: await this.download(file, signal), + mimeType: file.mimeType, + name: file.filename, + ...(sized && file.kind === job.kind ? { width: job.width, height: job.height } : {}), + ...(timed && file.kind === "video" + ? { durationMs: Math.round(((values.frames ?? 1) / COMFYUI_VIDEO_FPS) * 1000) } + : {}), + }); + } + job.onProgress?.((index + 1) / job.count); + } + return results; + } + + /** Submits a prompt and waits for its outputs; a cancelled or timed-out prompt is dequeued. */ + private async runPrompt( + graph: ComfyWorkflow, + kind: MediaKind, + signal: AbortSignal | undefined, + ): Promise> { + const deadline = AbortSignal.timeout( + kind === "image" ? IMAGE_PROMPT_TIMEOUT_MS : VIDEO_PROMPT_TIMEOUT_MS, + ); + const runSignal = signal ? AbortSignal.any([signal, deadline]) : deadline; + const promptId = await this.submit(graph, runSignal); + try { + for (;;) { + const outputs = await this.finishedOutputs(promptId, runSignal); + if (outputs) return outputs; + await sleep(this.pollIntervalMs, undefined, { signal: runSignal }); + } + } catch (error) { + if (!runSignal.aborted) throw error; + await this.cancelPrompt(promptId); + if (signal?.aborted) throw signal.reason ?? error; + throw new Error("ComfyUI timed out"); + } + } + + private async submit(graph: ComfyWorkflow, signal: AbortSignal): Promise { + const body = JSON.stringify({ prompt: graph, client_id: this.clientId }); + return this.send("prompt", { method: "POST", headers: JSON_HEADERS, body }, signal, { + read: async (response, readSignal) => { + if (response.status === 400) { + const rejection = await readJson(response, readSignal).catch(() => undefined); + throw new Error(describeWorkflowRejection(rejection)); + } + const result = await readOkJson(response, readSignal); + const promptId = isRecord(result) ? result.prompt_id : undefined; + if (typeof promptId !== "string" || !promptId) { + throw new Error("ComfyUI did not accept the workflow"); + } + return promptId; + }, + }); + } + + private async finishedOutputs( + promptId: string, + signal: AbortSignal, + ): Promise | undefined> { + const history = await this.getJson(`history/${encodeURIComponent(promptId)}`, signal); + const entry = isRecord(history) ? history[promptId] : undefined; + if (!isRecord(entry)) return undefined; + const status = isRecord(entry.status) ? entry.status : {}; + if (status.status_str === "error") throw new Error(describeExecutionError(status.messages)); + if (status.completed === false) return undefined; + return isRecord(entry.outputs) ? entry.outputs : {}; + } + + /** Best effort: drops the prompt from the queue, or interrupts it when it is ours and running. */ + private async cancelPrompt(promptId: string): Promise { + const options = { timeoutMs: CANCEL_TIMEOUT_MS, read: discardBody }; + try { + const body = JSON.stringify({ delete: [promptId] }); + await this.send("queue", { method: "POST", headers: JSON_HEADERS, body }, undefined, options); + const queue = await this.send( + "queue", + { headers: { Accept: "application/json" } }, + undefined, + { + timeoutMs: CANCEL_TIMEOUT_MS, + read: readOkJson, + }, + ); + const running = + isRecord(queue) && Array.isArray(queue.queue_running) ? queue.queue_running : []; + if (running.some((entry) => Array.isArray(entry) && entry[1] === promptId)) { + const interrupt = JSON.stringify({ prompt_id: promptId }); + await this.send( + "interrupt", + { method: "POST", headers: JSON_HEADERS, body: interrupt }, + undefined, + options, + ); + } + } catch { + // The prompt then finishes on its own; nothing waits for it. + } + } + + private async imageCheckpoint(signal: AbortSignal | undefined): Promise { + if (this.imageModel) return this.imageModel; + const info = await this.getJson("object_info/CheckpointLoaderSimple", signal); + const videoCheckpoints = new Set( + [DEFAULT_WORKFLOWS.textToVideo, DEFAULT_WORKFLOWS.imageToVideo, this.videoWorkflow].flatMap( + (template) => (template ? checkpointNames(template.graph) : []), + ), + ); + const checkpoint = parseCheckpointList(info).find((name) => !videoCheckpoints.has(name)); + if (!checkpoint) throw new Error("ComfyUI has no image checkpoint installed"); + return checkpoint; + } + + private async upload(file: MediaFile, signal: AbortSignal | undefined): Promise { + const form = new FormData(); + const extension = attachmentExtensionForMimeType(file.mimeType) || ".png"; + form.append( + "image", + new Blob([Buffer.from(file.bytes)], { type: file.mimeType }), + `evoo-${randomUUID()}${extension}`, + ); + form.append("type", "input"); + form.append("overwrite", "true"); + // Serialize here: the address-checked fetch runs on the undici package, whose FormData + // class differs from the global one. + const encoded = new Response(form); + const headers = { + Accept: "application/json", + "Content-Type": encoded.headers.get("content-type") ?? "", + }; + const body = new Uint8Array(await encoded.arrayBuffer()); + return this.send("upload/image", { method: "POST", headers, body }, signal, { + timeoutMs: TRANSFER_TIMEOUT_MS, + read: async (response, readSignal) => { + const result = await readOkJson(response, readSignal); + const name = isRecord(result) && typeof result.name === "string" ? result.name : ""; + if (!name) throw new Error("ComfyUI did not accept the image"); + const subfolder = + isRecord(result) && typeof result.subfolder === "string" ? result.subfolder : ""; + return subfolder ? `${subfolder}/${name}` : name; + }, + }); + } + + private download(file: OutputFile, signal: AbortSignal | undefined): Promise { + const query = new URLSearchParams({ + filename: file.filename, + subfolder: file.subfolder, + type: file.type, + }); + return this.send(`view?${query}`, {}, signal, { + timeoutMs: TRANSFER_TIMEOUT_MS, + read: async (response, readSignal) => { + await assertOk(response); + return readBodyCapped(response, ARTIFACT_MAX_BYTES_BY_KIND[file.kind], readSignal); + }, + }); + } + + private getJson(path: string, signal: AbortSignal | undefined): Promise { + return this.send(path, { headers: { Accept: "application/json" } }, signal, { + read: readOkJson, + }); + } + + /** One request, with the timeout covering the response body too. */ + private async send( + path: string, + init: { + method?: string; + headers?: Record; + body?: string | Uint8Array; + }, + signal: AbortSignal | undefined, + options: { + timeoutMs?: number; + read: (response: Response, signal: AbortSignal) => Promise; + }, + ): Promise { + const timeout = AbortSignal.timeout(options.timeoutMs ?? REQUEST_TIMEOUT_MS); + const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout; + const headers: Record = { ...init.headers }; + if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; + try { + const response = await this.fetch(new URL(path, this.baseUrl).href, { + method: init.method ?? "GET", + headers, + body: init.body, + redirect: "error", + signal: requestSignal, + }); + return await options.read(response, requestSignal); + } catch (error) { + if (signal?.aborted) throw error; + if (timeout.aborted) throw new Error("ComfyUI timed out"); + // undici reports refused or reset connections as a bare "fetch failed". + if (error instanceof TypeError && error.message === "fetch failed") { + throw new Error("ComfyUI is unreachable"); + } + throw error; + } + } +} + +function comfyUiBaseUrl(raw: string): URL { + const url = assertAllowedOpenAiCompatibleRequestUrl(raw.trim()); + url.search = ""; + url.hash = ""; + url.pathname = `${url.pathname.replace(/\/+$/, "")}/`; + return url; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function truncate(text: string): string { + return text.length > MAX_ERROR_CHARS ? `${text.slice(0, MAX_ERROR_CHARS - 1)}…` : text; +} + +function roundSide(side: number, multiple: number): number { + const rounded = Math.round(side / multiple) * multiple; + return Math.min(Math.max(rounded, multiple * Math.ceil(64 / multiple)), MAX_IMAGE_SIDE); +} + +/** A custom workflow replaces the default that matches whether it takes an input image. */ +function pickTemplate( + custom: WorkflowTemplate | undefined, + withImage: boolean, + textDefault: WorkflowTemplate, + imageDefault: WorkflowTemplate, +): WorkflowTemplate { + if (custom && custom.placeholders.has("image") === withImage) return custom; + return withImage ? imageDefault : textDefault; +} + +function checkpointNames(graph: ComfyWorkflow): string[] { + return Object.values(graph).flatMap((node) => + typeof node.inputs.ckpt_name === "string" ? [node.inputs.ckpt_name] : [], + ); +} + +async function discardBody(response: Response): Promise { + await response.body?.cancel().catch(() => undefined); +} + +async function assertOk(response: Response): Promise { + if (response.ok) return; + await discardBody(response); + throw new Error(`ComfyUI returned ${response.status}`); +} + +async function readJson(response: Response, signal: AbortSignal): Promise { + const bytes = await readBodyCapped(response, MAX_JSON_BYTES, signal); + try { + return JSON.parse(new TextDecoder().decode(bytes)); + } catch { + throw new Error("ComfyUI returned invalid JSON"); + } +} + +async function readOkJson(response: Response, signal: AbortSignal): Promise { + await assertOk(response); + return readJson(response, signal); +} + +/** + * ComfyUI can evict its own models, so memory its PyTorch process holds counts as available; + * only memory other processes use (another model server on the same GPU) does not. + */ +export function parseGpuStatus(body: unknown): GpuStatus | undefined { + const devices = isRecord(body) && Array.isArray(body.devices) ? body.devices : []; + for (const device of devices) { + if (!isRecord(device) || device.type === "cpu") continue; + const total = finiteNumber(device.vram_total); + const free = finiteNumber(device.vram_free); + if (!total || free === undefined) continue; + const reserved = finiteNumber(device.torch_vram_total) ?? 0; + const reservedFree = finiteNumber(device.torch_vram_free) ?? 0; + const available = Math.min(total, Math.max(0, free - reservedFree + reserved)); + return { vramFreeMb: Math.floor(available / MIB), vramTotalMb: Math.floor(total / MIB) }; + } + return undefined; +} + +function finiteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +/** Older ComfyUI lists combo values as `[[...names], {...}]`, newer as `["COMBO", {options}]`. */ +export function parseCheckpointList(body: unknown): string[] { + const loader = isRecord(body) ? body.CheckpointLoaderSimple : undefined; + const input = isRecord(loader) ? loader.input : undefined; + const required = isRecord(input) ? input.required : undefined; + const spec = isRecord(required) ? required.ckpt_name : undefined; + if (!Array.isArray(spec)) return []; + const names = Array.isArray(spec[0]) + ? spec[0] + : isRecord(spec[1]) && Array.isArray(spec[1].options) + ? spec[1].options + : []; + return names.filter((name): name is string => typeof name === "string" && name.length > 0); +} + +/** Saved files win over previews; video runs return clips (or animations), not frames. */ +function outputFiles(outputs: Record, kind: MediaKind): OutputFile[] { + const files: OutputFile[] = []; + for (const node of Object.values(outputs)) { + if (!isRecord(node)) continue; + for (const key of ["images", "gifs", "videos"]) { + const entries = node[key]; + if (!Array.isArray(entries)) continue; + for (const entry of entries) { + if (!isRecord(entry) || typeof entry.filename !== "string" || !entry.filename) continue; + const mimeType = inferAttachmentMimeType(entry.filename); + if (!mimeType) continue; + files.push({ + filename: entry.filename, + subfolder: typeof entry.subfolder === "string" ? entry.subfolder : "", + type: typeof entry.type === "string" ? entry.type : "output", + mimeType, + kind: attachmentKindOf(mimeType), + }); + } + } + } + const videos = files.filter((file) => file.kind === "video"); + const wanted = + kind === "image" + ? files.filter((file) => file.kind === "image") + : videos.length > 0 + ? videos + : files.filter((file) => file.mimeType === "image/gif" || file.mimeType === "image/webp"); + const saved = wanted.filter((file) => file.type === "output"); + return saved.length > 0 ? saved : wanted; +} + +export function describeWorkflowRejection(body: unknown): string { + const problems: string[] = []; + const nodeErrors = isRecord(body) && isRecord(body.node_errors) ? body.node_errors : {}; + for (const node of Object.values(nodeErrors)) { + if (!isRecord(node) || !Array.isArray(node.errors)) continue; + for (const error of node.errors) { + if (!isRecord(error)) continue; + const parts = [node.class_type, error.message, error.details]; + problems.push(parts.filter((part) => typeof part === "string" && part).join(": ")); + } + } + if (problems.length === 0 && isRecord(body) && isRecord(body.error)) { + if (typeof body.error.message === "string") problems.push(body.error.message); + } + return truncate( + problems.length > 0 + ? `ComfyUI rejected the workflow: ${problems.join("; ")}` + : "ComfyUI rejected the workflow", + ); +} + +export function describeExecutionError(messages: unknown): string { + const events = Array.isArray(messages) ? messages : []; + for (const event of events) { + if (!Array.isArray(event)) continue; + const [name, data] = event; + if (name === "execution_interrupted") return "ComfyUI interrupted the run"; + if (name !== "execution_error" || !isRecord(data)) continue; + const node = typeof data.node_type === "string" ? data.node_type : "a node"; + const message = + typeof data.exception_message === "string" + ? data.exception_message.trim().split("\n")[0] + : ""; + return truncate(`ComfyUI failed in ${node}${message ? `: ${message}` : ""}`); + } + return "ComfyUI failed"; +} diff --git a/packages/adapters/src/image-generation-conformance.test.ts b/packages/adapters/src/image-generation-conformance.test.ts new file mode 100644 index 0000000..92883a5 --- /dev/null +++ b/packages/adapters/src/image-generation-conformance.test.ts @@ -0,0 +1,111 @@ +import type { + ImageGenerationProvider, + MediaFile, + VideoGenerationProvider, +} from "@rakazo/adapter-kit"; +import { attachmentKindOf } from "@rakazo/contracts"; +import sharp from "sharp"; +import { describe, expect, it } from "vitest"; +import { createComfyUiProvider } from "./comfyui-provider.js"; +import { ComfyUiEmulator } from "./testing/comfyui-emulator.js"; + +type MediaGenerationProvider = ImageGenerationProvider & VideoGenerationProvider; + +const factories: Record MediaGenerationProvider> = { + comfyui: () => + createComfyUiProvider({ + baseUrl: "http://127.0.0.1:8188", + fetch: new ComfyUiEmulator().fetch, + pollIntervalMs: 1, + }), +}; + +async function sourceImage(): Promise { + const png = await sharp({ + create: { width: 96, height: 64, channels: 3, background: { r: 10, g: 20, b: 30 } }, + }) + .png() + .toBuffer(); + return { bytes: new Uint8Array(png), mimeType: "image/png", name: "source.png" }; +} + +/** + * Offline conformance: every image/video generation provider describes its limits, returns + * the requested number of decodable images, is reproducible by seed, takes a source image, + * reports progress, returns video, and honors cancellation. + */ +for (const [name, create] of Object.entries(factories)) { + describe(`${name} media generation conformance (offline)`, () => { + it("describes the contract and its limits", () => { + const image: ImageGenerationProvider = create(); + const video: VideoGenerationProvider = create(); + expect(image.describe().contractVersion).toBe("1"); + expect(image.describe().capabilities.maxWidth).toBeGreaterThanOrEqual(1024); + expect(image.describe().capabilities.maxHeight).toBeGreaterThanOrEqual(1024); + expect(video.describe().contractVersion).toBe("1"); + expect(video.describe().capabilities.maxDurationSeconds).toBeGreaterThan(0); + }); + + it("returns the requested number of images, reproducible by seed", async () => { + const provider = create(); + const progress: number[] = []; + const request = { prompt: "a red kite", width: 256, height: 192, count: 2, seed: 7 }; + const images = await provider.generateImage({ + ...request, + onProgress: (fraction) => progress.push(fraction), + }); + expect(images).toHaveLength(2); + for (const image of images) { + expect(attachmentKindOf(image.mimeType)).toBe("image"); + const metadata = await sharp(image.bytes).metadata(); + expect([metadata.width, metadata.height]).toEqual([256, 192]); + } + expect(Buffer.from(images[0]!.bytes).equals(Buffer.from(images[1]!.bytes))).toBe(false); + expect(progress.at(-1)).toBe(1); + const again = await provider.generateImage({ ...request, count: 1 }); + expect(Buffer.from(again[0]!.bytes).equals(Buffer.from(images[0]!.bytes))).toBe(true); + }); + + it("generates from a source image", async () => { + const images = await create().generateImage({ + prompt: "the same scene at night", + width: 96, + height: 64, + source: await sourceImage(), + strength: 0.5, + }); + expect(images).toHaveLength(1); + expect(attachmentKindOf(images[0]!.mimeType)).toBe("image"); + }); + + it("returns video from text and from a start frame", async () => { + const provider = create(); + for (const source of [undefined, await sourceImage()]) { + const [clip, ...rest] = await provider.generateVideo({ + prompt: "waves rolling onto a beach", + width: 768, + height: 512, + durationSeconds: 2, + source, + }); + expect(rest).toHaveLength(0); + expect(attachmentKindOf(clip!.mimeType)).toBe("video"); + expect(clip!.bytes.byteLength).toBeGreaterThan(0); + } + }); + + it("reports GPU memory with free not above total", async () => { + const status = await create().gpuStatus?.(); + if (!status) return; + expect(status.vramTotalMb).toBeGreaterThan(0); + expect(status.vramFreeMb).toBeGreaterThanOrEqual(0); + expect(status.vramFreeMb).toBeLessThanOrEqual(status.vramTotalMb); + }); + + it("rejects when the caller cancels", async () => { + const request = { prompt: "x", width: 64, height: 64, signal: AbortSignal.abort() }; + await expect(create().generateImage(request)).rejects.toThrow(); + await expect(create().generateVideo({ ...request, durationSeconds: 1 })).rejects.toThrow(); + }); + }); +} diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 7b4ecd0..18253dc 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -23,6 +23,7 @@ export * from "./cloud-agent-provider-env.js"; export * from "./cloud-agent-service.js"; export * from "./cloud-agent-tools.js"; export * from "./cloud-agent-tools-select.js"; +export * from "./comfyui-provider.js"; export * from "./composio-catalog-cache.js"; export * from "./composio-connector.js"; export * from "./composio-emulator.js"; diff --git a/packages/adapters/src/testing/comfyui-emulator.ts b/packages/adapters/src/testing/comfyui-emulator.ts new file mode 100644 index 0000000..4da22a1 --- /dev/null +++ b/packages/adapters/src/testing/comfyui-emulator.ts @@ -0,0 +1,393 @@ +import { createHash } from "node:crypto"; + +/** A 1 s, 32×32 H.264 MP4 (ffmpeg testsrc), small enough to inline. */ +const TINY_MP4_BASE64 = [ + "AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAM+bW9vdgAAAGxtdmhkAAAAAAAAAAAAAAAAAAAD6AAAA+gAAQAAAQAA", + "AAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAA", + "Ao10cmFrAAAAXHRraGQAAAADAAAAAAAAAAAAAAABAAAAAAAAA+gAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAA", + "AAAAAAAAAAAAAABAAAAAACAAAAAgAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAAPoAAAQAAABAAAAAAIFbWRpYQAAACBtZGhk", + "AAAAAAAAAAAAAAAAAABAAAAAQABVxAAAAAAALWhkbHIAAAAAAAAAAHZpZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAABsG1p", + "bmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAAACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAAAXBzdGJsAAAAwHN0c2QA", + "AAAAAAAAAQAAALBhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAACAAIABIAAAASAAAAAAAAAABDExhdmMgbGlieDI2NAAAAAAA", + "AAAAAAAAAAAAAAAAAAAAGP//AAAANmF2Y0MBZAAK/+EAGGdkAAqscgRJbARAAAADAEAAAAQDxIlhGAEAB2joQ4OSyLD9+PgAAAAA", + "EHBhc3AAAAABAAAAAQAAABRidHJ0AAAAAAAADAgAAAwIAAAAGHN0dHMAAAAAAAAAAQAAAAgAAAgAAAAAFHN0c3MAAAAAAAAAAQAA", + "AAEAAAAYY3R0cwAAAAAAAAABAAAACAAAEAAAAAAcc3RzYwAAAAAAAAABAAAAAQAAAAgAAAABAAAANHN0c3oAAAAAAAAAAAAAAAgA", + "AADTAAAADAAAABgAAAAdAAAAGgAAAB0AAAAfAAAAFwAAABRzdGNvAAAAAAAAAAEAAANuAAAAPXVkdGEAAAA1bWV0YQAAAAAAAAAh", + "aGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAIaWxzdAAAAAhmcmVlAAABiW1kYXQAAADPZYiBABP/115QL0x5O3/uan23", + "okcn80QwiyZelIq0iY0T1naTwGHVgXgsN7YBwlEikXvo+AyEjKk/afikvNFJbO7NjVpKr2f7e+xmO5rxkaelhrWKVT83gKOM5wKs", + "QoFpCKDo56fFf6IYs3U1LAHAxZY3IIAUCFca5RyJ/ibKbmkxBCXXiBuUCCuL9FhMjcqhW2misGdpwuLgknHPr/37MowWA2zcOn9B", + "OHR5mfgyt4/0x3o95hpnI+2axZFkXHhYwK55h134UUvr+r3B9b/xAAAACEGaCC2JP6KAAAAAFEGaEEvwIGTKYTe7qicJ6jU1YvCB", + "AAAAGUGaGGk8CB5MpgTfu1UOB2sZPRDVFTCp+IAAAAAWQZogiXwIH5MpgER/wf8hVV5nVt8U2QAAABlBmiipvAgf5MpgBFfFjke1", + "iXbU9gFuIojBAAAAG0GaMMn8CB/5MpgARn/KNfso3F7XaeLq3d3VjQAAABNBmjjojwIH/5MpgAEIf9P6j9Ph", +].join(""); + +export const COMFYUI_EMULATOR_MP4 = new Uint8Array(Buffer.from(TINY_MP4_BASE64, "base64")); + +const MIB = 1024 * 1024; + +type EmulatedNode = { class_type: string; inputs: Record }; +type EmulatedGraph = Record; +type FileRef = { filename: string; subfolder: string; type: string }; + +interface EmulatedPrompt { + id: string; + graph: EmulatedGraph; + polls: number; + failure?: string; + interrupted?: boolean; + /** Dropped from the queue before it ran. */ + removed?: boolean; + /** Reported as finished by the history. */ + done?: boolean; + outputs?: Record>; +} + +export interface EmulatedComfyUiRequest { + method: string; + path: string; + authorization: string | null; + body?: unknown; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function json(body: unknown, status = 200): Response { + return Response.json(body, { status }); +} + +/** + * In-process substitute for a ComfyUI server's HTTP API, so the real provider runs in offline + * tests: it validates submitted workflows like ComfyUI does, "runs" them after a few history + * polls, and returns small generated PNGs or an MP4 fixture. + */ +export class ComfyUiEmulator { + readonly requests: EmulatedComfyUiRequest[] = []; + readonly prompts: EmulatedPrompt[] = []; + readonly uploads = new Map(); + readonly deleted: string[] = []; + readonly interrupted: string[] = []; + checkpoints = ["sd_xl_base_1.0.safetensors", "ltx-video-2b-v0.9.5.safetensors"]; + /** `options` answers like current ComfyUI (`["COMBO", {options}]`), `legacy` like older ones. */ + checkpointListFormat: "options" | "legacy" = "options"; + /** null reports a CPU-only server. */ + gpu: { totalMb: number; freeMb: number; torchReservedMb: number; torchFreeMb: number } | null = { + totalMb: 24_576, + freeMb: 20_000, + torchReservedMb: 0, + torchFreeMb: 0, + }; + /** History polls that still report a submitted prompt as running. */ + pendingPolls = 1; + /** Answer the next request with this HTTP status. */ + failNextStatus: number | undefined; + /** Fail the next submitted prompt while it runs, with this exception message. */ + failNextExecution: string | undefined; + private readonly files = new Map(); + private nextPrompt = 1; + + /** `basePath` serves the API under a prefix, like ComfyUI behind a reverse proxy. */ + constructor(private readonly options: { apiKey?: string; basePath?: string } = {}) {} + + readonly fetch: typeof fetch = async (input, init) => { + init?.signal?.throwIfAborted(); + const url = new URL(input instanceof Request ? input.url : String(input)); + const method = init?.method ?? "GET"; + const headers = new Headers(init?.headers); + const authorization = headers.get("authorization"); + const prefix = `/${(this.options.basePath ?? "").replace(/^\/+|\/+$/g, "")}/`.replace( + "//", + "/", + ); + if (!url.pathname.startsWith(prefix)) return json({ error: "Not found" }, 404); + const path = url.pathname.slice(prefix.length); + const body = + typeof init?.body === "string" && headers.get("content-type")?.includes("json") + ? JSON.parse(init.body) + : undefined; + this.requests.push({ method, path, authorization, body }); + if (this.failNextStatus) { + const status = this.failNextStatus; + this.failNextStatus = undefined; + return new Response("Upstream failure", { status }); + } + if (this.options.apiKey && authorization !== `Bearer ${this.options.apiKey}`) { + return json({ error: "Unauthorized" }, 401); + } + const route = `${method} ${path.split("/")[0]}`; + if (route === "GET system_stats") return this.systemStats(); + if (route === "GET object_info" && path === "object_info/CheckpointLoaderSimple") { + return this.checkpointInfo(); + } + if (route === "POST prompt") return this.submit(body); + if (route === "GET history") return this.history(decodeURIComponent(path.slice(8))); + if (route === "GET view") return this.view(url.searchParams); + if (route === "POST upload") return this.upload(input, init); + if (route === "GET queue") return this.queue(); + if (route === "POST queue") { + for (const id of Array.isArray(body?.delete) ? body.delete : []) { + this.deleted.push(id); + // Like ComfyUI, deleting only drops pending prompts; a running one needs an interrupt. + const prompt = this.prompts.find((entry) => entry.id === id); + if (prompt && prompt !== this.unfinished()[0]) prompt.removed = true; + } + return json({}); + } + if (route === "POST interrupt") { + const id = String(body?.prompt_id ?? ""); + this.interrupted.push(id); + const running = this.unfinished()[0]; + if (running && (!id || running.id === id)) running.interrupted = true; + return json({}); + } + return json({ error: "Not found" }, 404); + }; + + private systemStats(): Response { + const gpu = this.gpu; + return json({ + system: { os: "posix", comfyui_version: "0.3.60", embedded_python: false }, + devices: [ + gpu + ? { + name: "cuda:0 Emulated GPU", + type: "cuda", + index: 0, + vram_total: gpu.totalMb * MIB, + vram_free: gpu.freeMb * MIB, + torch_vram_total: gpu.torchReservedMb * MIB, + torch_vram_free: gpu.torchFreeMb * MIB, + } + : { name: "cpu", type: "cpu", index: null, vram_total: 64 * 1024 * MIB, vram_free: 0 }, + ], + }); + } + + private checkpointInfo(): Response { + const spec = + this.checkpointListFormat === "legacy" + ? [this.checkpoints, { tooltip: "Checkpoint" }] + : ["COMBO", { options: this.checkpoints, tooltip: "Checkpoint" }]; + return json({ + CheckpointLoaderSimple: { + input: { required: { ckpt_name: spec } }, + output: ["MODEL", "CLIP", "VAE"], + name: "CheckpointLoaderSimple", + }, + }); + } + + private submit(body: unknown): Response { + const graph = isRecord(body) ? body.prompt : undefined; + if (!isRecord(graph) || typeof (body as { client_id?: unknown }).client_id !== "string") { + return json( + { error: { type: "invalid_prompt", message: "Cannot execute because prompt is invalid" } }, + 400, + ); + } + const nodeErrors: Record = {}; + const reject = (id: string, node: unknown, message: string, details: string) => { + nodeErrors[id] = { + errors: [{ type: "invalid", message, details, extra_info: {} }], + dependent_outputs: [], + class_type: isRecord(node) ? node.class_type : undefined, + }; + }; + for (const [id, node] of Object.entries(graph)) { + if (!isRecord(node) || typeof node.class_type !== "string" || !isRecord(node.inputs)) { + reject(id, node, "Invalid node", "class_type and inputs are required"); + continue; + } + for (const [name, value] of Object.entries(node.inputs)) { + if (typeof value === "string" && value.includes("{{")) { + reject(id, node, "Unfilled placeholder", `${name}: ${value}`); + } + if (Array.isArray(value) && !isRecord(graph[String(value[0])])) { + reject(id, node, "Missing linked node", `${name}: ${String(value[0])}`); + } + } + const checkpoint = node.inputs.ckpt_name; + if (typeof checkpoint === "string" && !this.checkpoints.includes(checkpoint)) { + reject(id, node, "Value not in list", `ckpt_name: '${checkpoint}' not in list`); + } + if (node.class_type === "LoadImage" && !this.uploads.has(String(node.inputs.image))) { + reject(id, node, "Invalid image file", `image - ${String(node.inputs.image)}`); + } + } + if (Object.keys(nodeErrors).length > 0) { + return json( + { + error: { + type: "prompt_outputs_failed_validation", + message: "Prompt outputs failed validation", + }, + node_errors: nodeErrors, + }, + 400, + ); + } + const id = `prompt-${this.nextPrompt++}`; + this.prompts.push({ + id, + graph: graph as EmulatedGraph, + polls: 0, + failure: this.failNextExecution, + }); + this.failNextExecution = undefined; + return json({ prompt_id: id, number: this.nextPrompt, node_errors: {} }); + } + + private async history(id: string): Promise { + const prompt = this.prompts.find((entry) => entry.id === id); + if (!prompt || prompt.removed) return json({}); + if (!prompt.done && !prompt.interrupted && prompt.polls < this.pendingPolls) { + prompt.polls += 1; + return json({}); + } + prompt.done = true; + if (prompt.interrupted) { + return json({ + [id]: { + prompt: [0, id, prompt.graph, {}, []], + outputs: {}, + status: { + status_str: "error", + completed: false, + messages: [["execution_interrupted", { prompt_id: id, node_id: "5" }]], + }, + }, + }); + } + if (prompt.failure) { + return json({ + [id]: { + prompt: [0, id, prompt.graph, {}, []], + outputs: {}, + status: { + status_str: "error", + completed: false, + messages: [ + ["execution_start", { prompt_id: id }], + [ + "execution_error", + { + prompt_id: id, + node_id: "5", + node_type: "KSampler", + exception_message: prompt.failure, + exception_type: "RuntimeError", + traceback: ["..."], + }, + ], + ], + }, + }, + }); + } + prompt.outputs ??= await this.execute(prompt); + return json({ + [id]: { + prompt: [0, id, prompt.graph, {}, Object.keys(prompt.outputs)], + outputs: prompt.outputs, + status: { status_str: "success", completed: true, messages: [] }, + }, + }); + } + + private async execute(prompt: EmulatedPrompt): Promise>> { + const outputs: Record> = {}; + let counter = 0; + const save = (bytes: Uint8Array, extension: string, type: string): FileRef => { + counter += 1; + const ref = { + filename: `${prompt.id}_${String(counter).padStart(5, "0")}_.${extension}`, + subfolder: type === "output" ? "evoo" : "", + type, + }; + this.files.set(fileKey(ref), bytes); + return ref; + }; + for (const [id, node] of Object.entries(prompt.graph)) { + if (node.class_type === "SaveImage" || node.class_type === "PreviewImage") { + const type = node.class_type === "SaveImage" ? "output" : "temp"; + outputs[id] = { images: [save(await this.renderImage(prompt.graph), "png", type)] }; + } else if (node.class_type === "SaveVideo") { + outputs[id] = { images: [save(COMFYUI_EMULATOR_MP4, "mp4", "output")], animated: [true] }; + } else if (node.class_type === "VHS_VideoCombine") { + outputs[id] = { gifs: [save(COMFYUI_EMULATOR_MP4, "mp4", "output")] }; + } + } + return outputs; + } + + /** A flat color derived from the seed, at the workflow's size (or its input image's). */ + private async renderImage(graph: EmulatedGraph): Promise { + const { default: sharp } = await import("sharp"); + const nodes = Object.values(graph); + const input = (name: string) => + nodes.map((node) => node.inputs[name]).find((value) => typeof value === "number"); + let width = Number(input("width") ?? 0); + let height = Number(input("height") ?? 0); + const loaded = nodes.find((node) => node.class_type === "LoadImage"); + if (loaded && !(width && height)) { + const metadata = await sharp(this.uploads.get(String(loaded.inputs.image))).metadata(); + width = metadata.width ?? 64; + height = metadata.height ?? 64; + } + const seed = String(input("seed") ?? input("noise_seed") ?? 0); + const [r = 0, g = 0, b = 0] = createHash("sha256").update(seed).digest(); + const png = await sharp({ + create: { width: width || 64, height: height || 64, channels: 3, background: { r, g, b } }, + }) + .png() + .toBuffer(); + return new Uint8Array(png); + } + + private view(params: URLSearchParams): Response { + const bytes = this.files.get( + fileKey({ + filename: params.get("filename") ?? "", + subfolder: params.get("subfolder") ?? "", + type: params.get("type") ?? "", + }), + ); + if (!bytes) return new Response("Not found", { status: 404 }); + return new Response(Buffer.from(bytes), { + headers: { "Content-Type": "application/octet-stream" }, + }); + } + + private async upload(input: RequestInfo | URL, init?: RequestInit): Promise { + const form = await new Request(String(input), { + method: "POST", + headers: init?.headers, + body: init?.body, + }).formData(); + const image = form.get("image"); + if (!(image instanceof File) || form.get("type") !== "input") { + return json({ error: "Invalid upload" }, 400); + } + this.uploads.set(image.name, new Uint8Array(await image.arrayBuffer())); + return json({ name: image.name, subfolder: "", type: "input" }); + } + + private unfinished(): EmulatedPrompt[] { + return this.prompts.filter((prompt) => !prompt.done && !prompt.removed); + } + + private queue(): Response { + const running = this.unfinished(); + return json({ + queue_running: running.slice(0, 1).map((prompt) => [0, prompt.id, prompt.graph, {}, []]), + queue_pending: running + .slice(1) + .map((prompt, index) => [index + 1, prompt.id, prompt.graph, {}, []]), + }); + } +} + +function fileKey(ref: FileRef): string { + return `${ref.type}/${ref.subfolder}/${ref.filename}`; +} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index ba2c70b..3c000fb 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -10,6 +10,7 @@ export * from "./integration-settings.js"; export * from "./local-settings.js"; export * from "./mcp.js"; export * from "./media.js"; +export * from "./media-generation.js"; export * from "./model-tiers.js"; export * from "./openai-compatible-ui.js"; export * from "./project-goals.js"; diff --git a/packages/contracts/src/media-generation.test.ts b/packages/contracts/src/media-generation.test.ts new file mode 100644 index 0000000..4ab1a74 --- /dev/null +++ b/packages/contracts/src/media-generation.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { + EditImageInputSchema, + GenerateImageInputSchema, + GenerateVideoInputSchema, +} from "./media-generation.js"; + +describe("media generation inputs", () => { + it("fills defaults and drops unknown fields such as continue_after", () => { + expect(GenerateImageInputSchema.parse({ prompt: " a fox ", continue_after: true })).toEqual({ + prompt: "a fox", + count: 1, + }); + expect(GenerateVideoInputSchema.parse({ prompt: "waves" })).toEqual({ + prompt: "waves", + aspect: "landscape", + durationSeconds: 5, + }); + }); + + it("bounds image count, size and video duration", () => { + expect(GenerateImageInputSchema.safeParse({ prompt: "x", count: 5 }).success).toBe(false); + expect(GenerateImageInputSchema.safeParse({ prompt: "x", width: 512 }).success).toBe(false); + expect( + GenerateImageInputSchema.safeParse({ prompt: "x", width: 512, height: 4096 }).success, + ).toBe(false); + expect(GenerateVideoInputSchema.safeParse({ prompt: "x", durationSeconds: 11 }).success).toBe( + false, + ); + }); + + it("needs at least one edit and valid edit values", () => { + expect(EditImageInputSchema.safeParse({ artifactId: "a" }).success).toBe(false); + expect(EditImageInputSchema.safeParse({ artifactId: "a", rotate: 45 }).success).toBe(false); + expect(EditImageInputSchema.safeParse({ artifactId: "a", resize: {} }).success).toBe(false); + expect( + EditImageInputSchema.safeParse({ artifactId: "a", extend: { top: 4, color: "red" } }).success, + ).toBe(false); + expect( + EditImageInputSchema.parse({ artifactId: "a", resize: { aspect: "vertical" }, rotate: 90 }), + ).toEqual({ artifactId: "a", resize: { aspect: "vertical", fit: "crop" }, rotate: 90 }); + }); +}); diff --git a/packages/contracts/src/media-generation.ts b/packages/contracts/src/media-generation.ts new file mode 100644 index 0000000..6d33216 --- /dev/null +++ b/packages/contracts/src/media-generation.ts @@ -0,0 +1,108 @@ +import * as z from "zod"; +import { Id } from "./ids.js"; +import { MediaFitSchema, MediaPresetIdSchema } from "./media.js"; + +export const MEDIA_IMAGE_MAX_COUNT = 4; +export const MEDIA_VIDEO_MAX_SECONDS = 10; + +const Prompt = z.string().trim().min(1).max(4000); +const NegativePrompt = z.string().trim().max(4000).optional().describe("What to avoid."); +const Seed = z + .number() + .int() + .min(0) + .max(Number.MAX_SAFE_INTEGER) + .optional() + .describe("Reuse a seed to reproduce a result."); +const Strength = z.number().min(0).max(1).optional(); +const Aspect = MediaPresetIdSchema.describe( + "vertical 9:16, landscape 16:9, square 1:1, portrait 4:5, classic 4:3, cinema 21:9.", +); +const Side = z.number().int().min(64).max(2048); +const Pixels = z.number().int().min(0).max(16_384); + +export const GenerateImageInputSchema = z + .object({ + prompt: Prompt.describe("What the image shows."), + negativePrompt: NegativePrompt, + aspect: Aspect.optional(), + width: Side.optional().describe("Exact width in pixels, with height; overrides aspect."), + height: Side.optional(), + count: z.number().int().min(1).max(MEDIA_IMAGE_MAX_COUNT).default(1), + seed: Seed, + sourceArtifactId: Id.optional().describe("Image artifact to start from (image-to-image)."), + strength: Strength.describe("With a source: how far to move away from it (default 0.6)."), + }) + .refine((input) => (input.width === undefined) === (input.height === undefined), { + message: "Set width and height together", + path: ["width"], + }); +export type GenerateImageInput = z.infer; + +export const GenerateVideoInputSchema = z.object({ + prompt: Prompt.describe("What happens in the clip: subject, motion, camera."), + negativePrompt: NegativePrompt, + aspect: Aspect.default("landscape"), + durationSeconds: z.number().min(1).max(MEDIA_VIDEO_MAX_SECONDS).default(5), + seed: Seed, + sourceArtifactId: Id.optional().describe("Image artifact to use as the first frame."), +}); +export type GenerateVideoInput = z.infer; + +export const ImageFormatSchema = z.enum(["png", "jpeg", "webp"]); +export type ImageFormat = z.infer; + +/** Edits apply in this order: crop, rotate, flip, resize, extend, repaint, format. */ +export const EditImageInputSchema = z + .object({ + artifactId: Id.describe("Image artifact to edit."), + crop: z + .object({ left: Pixels, top: Pixels, width: Pixels.min(1), height: Pixels.min(1) }) + .optional() + .describe("Area to keep, in source pixels."), + rotate: z.literal([90, 180, 270]).optional().describe("Clockwise, in degrees."), + flip: z.enum(["horizontal", "vertical"]).optional(), + resize: z + .object({ + aspect: Aspect.optional().describe("Exact size of this format."), + width: z.number().int().min(1).max(8192).optional(), + height: z.number().int().min(1).max(8192).optional(), + fit: MediaFitSchema.default("crop").describe( + "How a different aspect ratio fills the size: crop, pad, or blur (blurred background).", + ), + }) + .refine((resize) => resize.aspect || resize.width || resize.height, { + message: "Set aspect, width or height", + }) + .optional() + .describe("Width or height alone keeps the aspect ratio."), + extend: z + .object({ + top: Pixels.default(0), + right: Pixels.default(0), + bottom: Pixels.default(0), + left: Pixels.default(0), + color: z + .string() + .regex(/^#[0-9a-fA-F]{6}$/) + .optional() + .describe("#RRGGBB; transparent when omitted and the format allows it."), + }) + .optional() + .describe("Borders to add, in pixels."), + prompt: Prompt.optional().describe("Repaint the image with this prompt."), + strength: Strength.describe("How far the repaint moves away from the image (default 0.6)."), + format: ImageFormatSchema.optional().describe("Default keeps the source format."), + }) + .refine( + (input) => + input.crop || + input.rotate || + input.flip || + input.resize || + input.extend || + input.prompt || + input.format, + { message: "Nothing to edit" }, + ); +export type EditImageInput = z.infer; diff --git a/packages/core/package.json b/packages/core/package.json index bcac6e5..562dfd6 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -34,6 +34,11 @@ "development": "./src/plot/plot-spec.ts", "default": "./src/plot/plot-spec.ts" }, + "./media/workflows": { + "types": "./src/media/workflow-template.ts", + "development": "./src/media/workflow-template.ts", + "default": "./src/media/workflow-template.ts" + }, "./node/desktop-runtime": { "types": "./src/node/desktop-runtime.ts", "development": "./src/node/desktop-runtime.ts", diff --git a/packages/core/src/media/workflow-template.test.ts b/packages/core/src/media/workflow-template.test.ts new file mode 100644 index 0000000..7865463 --- /dev/null +++ b/packages/core/src/media/workflow-template.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_WORKFLOWS, + fillWorkflow, + parseWorkflowTemplate, + WorkflowTemplateError, +} from "./workflow-template.js"; + +const minimal = { + "1": { class_type: "CLIPTextEncode", inputs: { text: "{{prompt}}", clip: ["2", 1] } }, + "2": { class_type: "EmptyLatentImage", inputs: { width: "{{width}}", height: 512 } }, + "3": { class_type: "SaveImage", inputs: { filename_prefix: "run/{{seed}}-{{prompt}}" } }, +}; + +describe("parseWorkflowTemplate", () => { + it("accepts JSON text or parsed graphs and lists the placeholders", () => { + for (const source of [minimal, JSON.stringify(minimal)]) { + const template = parseWorkflowTemplate(source); + expect([...template.placeholders].sort()).toEqual(["prompt", "seed", "width"]); + } + }); + + it.each([ + ["not json", "Workflow is not valid JSON"], + ["[]", "Workflow must be a JSON object"], + ["{}", "Workflow must have 1 to 1000 nodes"], + [JSON.stringify({ nodes: [], links: [] }), "Workflow must be exported in ComfyUI's API format"], + [JSON.stringify({ "1": { inputs: {} } }), "Workflow node 1 needs a class_type and inputs"], + [ + JSON.stringify({ "1": { class_type: "SaveImage", inputs: { text: "plain" } } }), + "Workflow needs a {{prompt}} placeholder", + ], + [ + JSON.stringify({ + "1": { class_type: "X", inputs: { a: "{{prompt}}", b: "{{cfg}}" } }, + }), + "Unknown workflow placeholders: {{cfg}}", + ], + ])("rejects %s", (source, message) => { + expect(() => parseWorkflowTemplate(source)).toThrow(new WorkflowTemplateError(message)); + }); + + it("rejects placeholder names that only exist on Object.prototype", () => { + const source = { + "1": { + class_type: "X", + inputs: { a: "{{prompt}}", b: "{{constructor}}", c: "{{__proto__}}" }, + }, + }; + expect(() => parseWorkflowTemplate(source)).toThrow( + "Unknown workflow placeholders: {{constructor}}, {{__proto__}}", + ); + }); + + it("rejects oversized workflow text", () => { + expect(() => parseWorkflowTemplate(" ".repeat(1_000_001))).toThrow("Workflow is too large"); + }); +}); + +describe("fillWorkflow", () => { + it("replaces whole values with typed values and embedded ones with text, without mutating", () => { + const template = parseWorkflowTemplate(minimal); + const graph = fillWorkflow(template, { prompt: "a $& cat", width: 768, seed: 7 }); + expect(graph["1"]?.inputs).toEqual({ text: "a $& cat", clip: ["2", 1] }); + expect(graph["2"]?.inputs).toEqual({ width: 768, height: 512 }); + expect(graph["3"]?.inputs.filename_prefix).toBe("run/7-a $& cat"); + expect(minimal["2"].inputs.width).toBe("{{width}}"); + }); + + it("refuses missing or mistyped values", () => { + const template = parseWorkflowTemplate(minimal); + expect(() => fillWorkflow(template, { prompt: "cat", seed: 1 })).toThrow( + "Workflow value for {{width}} is missing", + ); + expect(() => fillWorkflow(template, { prompt: "cat", width: 1.5, seed: 1 })).toThrow( + "{{width}}", + ); + }); +}); + +describe("DEFAULT_WORKFLOWS", () => { + it("only image-to-* defaults need an input image", () => { + expect(DEFAULT_WORKFLOWS.textToImage.placeholders.has("image")).toBe(false); + expect(DEFAULT_WORKFLOWS.textToVideo.placeholders.has("image")).toBe(false); + expect(DEFAULT_WORKFLOWS.imageToImage.placeholders.has("image")).toBe(true); + expect(DEFAULT_WORKFLOWS.imageToVideo.placeholders.has("image")).toBe(true); + }); + + it("fill into graphs whose links point at existing nodes", () => { + const values = { + prompt: "a lighthouse", + negative_prompt: "", + model: "model.safetensors", + image: "input.png", + width: 512, + height: 512, + seed: 1, + steps: 20, + strength: 0.6, + frames: 25, + fps: 24, + }; + for (const template of Object.values(DEFAULT_WORKFLOWS)) { + const graph = fillWorkflow(template, values); + expect(JSON.stringify(graph)).not.toContain("{{"); + for (const node of Object.values(graph)) { + for (const input of Object.values(node.inputs)) { + if (Array.isArray(input)) expect(graph[String(input[0])]).toBeDefined(); + } + } + } + }); +}); diff --git a/packages/core/src/media/workflow-template.ts b/packages/core/src/media/workflow-template.ts new file mode 100644 index 0000000..a7c74e6 --- /dev/null +++ b/packages/core/src/media/workflow-template.ts @@ -0,0 +1,159 @@ +import imageToImage from "./workflows/image-to-image.json" with { type: "json" }; +import imageToVideo from "./workflows/image-to-video.json" with { type: "json" }; +import textToImage from "./workflows/text-to-image.json" with { type: "json" }; +import textToVideo from "./workflows/text-to-video.json" with { type: "json" }; + +/** + * Values a ComfyUI workflow template can ask for. A string value that is exactly `{{name}}` + * becomes the typed value (so `"{{width}}"` turns into the number 1024); `{{name}}` inside a + * longer string is replaced by its text. Templates are data: nothing is ever evaluated. + */ +export const WORKFLOW_PLACEHOLDERS = { + prompt: "string", + negative_prompt: "string", + /** Checkpoint file name: the configured image model or the first installed checkpoint. */ + model: "string", + /** Uploaded input image (image-to-image, image-to-video). */ + image: "string", + width: "integer", + height: "integer", + seed: "integer", + steps: "integer", + /** 0–1 denoise for image-to-image. */ + strength: "number", + frames: "integer", + fps: "number", +} as const satisfies Record; + +export type WorkflowPlaceholder = keyof typeof WORKFLOW_PLACEHOLDERS; +export type WorkflowValues = { + [Name in WorkflowPlaceholder]?: (typeof WORKFLOW_PLACEHOLDERS)[Name] extends "string" + ? string + : number; +}; + +/** One node of a workflow in ComfyUI's API format ("Export (API)" in ComfyUI). */ +export interface ComfyWorkflowNode { + class_type: string; + inputs: Record; +} +export type ComfyWorkflow = Record; + +export interface WorkflowTemplate { + readonly graph: ComfyWorkflow; + /** The placeholders the graph uses; `image` marks a template that needs an input image. */ + readonly placeholders: ReadonlySet; +} + +export class WorkflowTemplateError extends Error { + constructor(message: string) { + super(message); + this.name = "WorkflowTemplateError"; + } +} + +const PLACEHOLDER_PATTERN = /\{\{([a-z_]+)\}\}/g; +const WHOLE_PLACEHOLDER = /^\{\{([a-z_]+)\}\}$/; +const MAX_WORKFLOW_CHARS = 1_000_000; +const MAX_WORKFLOW_NODES = 1_000; + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function isPlaceholder(name: string): name is WorkflowPlaceholder { + return Object.hasOwn(WORKFLOW_PLACEHOLDERS, name); +} + +function visitStrings(value: unknown, visit: (text: string) => void): void { + if (typeof value === "string") visit(value); + else if (Array.isArray(value)) for (const entry of value) visitStrings(entry, visit); + else if (isRecord(value)) for (const entry of Object.values(value)) visitStrings(entry, visit); +} + +/** Validates an API-format workflow (JSON text or parsed) and lists its placeholders. */ +export function parseWorkflowTemplate(source: unknown): WorkflowTemplate { + let value = source; + if (typeof source === "string") { + if (source.length > MAX_WORKFLOW_CHARS) { + throw new WorkflowTemplateError("Workflow is too large"); + } + try { + value = JSON.parse(source); + } catch { + throw new WorkflowTemplateError("Workflow is not valid JSON"); + } + } + if (!isRecord(value)) throw new WorkflowTemplateError("Workflow must be a JSON object"); + if (Array.isArray(value.nodes)) { + throw new WorkflowTemplateError("Workflow must be exported in ComfyUI's API format"); + } + const nodes = Object.entries(value); + if (nodes.length === 0 || nodes.length > MAX_WORKFLOW_NODES) { + throw new WorkflowTemplateError(`Workflow must have 1 to ${MAX_WORKFLOW_NODES} nodes`); + } + for (const [id, node] of nodes) { + if (!isRecord(node) || typeof node.class_type !== "string" || !isRecord(node.inputs)) { + throw new WorkflowTemplateError(`Workflow node ${id} needs a class_type and inputs`); + } + } + const placeholders = new Set(); + const unknown = new Set(); + visitStrings(value, (text) => { + for (const [, name = ""] of text.matchAll(PLACEHOLDER_PATTERN)) { + if (isPlaceholder(name)) placeholders.add(name); + else unknown.add(`{{${name}}}`); + } + }); + if (unknown.size > 0) { + throw new WorkflowTemplateError(`Unknown workflow placeholders: ${[...unknown].join(", ")}`); + } + if (!placeholders.has("prompt")) { + throw new WorkflowTemplateError("Workflow needs a {{prompt}} placeholder"); + } + return { graph: value as unknown as ComfyWorkflow, placeholders }; +} + +function replacePlaceholders(value: unknown, values: WorkflowValues): unknown { + if (typeof value === "string") { + const whole = WHOLE_PLACEHOLDER.exec(value)?.[1]; + if (whole) return values[whole as WorkflowPlaceholder]; + return value.replace(PLACEHOLDER_PATTERN, (_, name: string) => + String(values[name as WorkflowPlaceholder]), + ); + } + if (Array.isArray(value)) return value.map((entry) => replacePlaceholders(entry, values)); + if (isRecord(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [key, replacePlaceholders(entry, values)]), + ); + } + return value; +} + +/** Returns a copy of the template's graph with every placeholder replaced by its typed value. */ +export function fillWorkflow(template: WorkflowTemplate, values: WorkflowValues): ComfyWorkflow { + for (const name of template.placeholders) { + const value = values[name]; + const type = WORKFLOW_PLACEHOLDERS[name]; + const valid = + type === "string" + ? typeof value === "string" + : typeof value === "number" && + Number.isFinite(value) && + (type === "number" || Number.isInteger(value)); + if (!valid) throw new WorkflowTemplateError(`Workflow value for {{${name}}} is missing`); + } + return replacePlaceholders(template.graph, values) as ComfyWorkflow; +} + +/** + * Built-in workflows. Images use core nodes only and run on any SD-family checkpoint; video + * uses ComfyUI's core LTX-Video nodes (model files in workflows/README.md). + */ +export const DEFAULT_WORKFLOWS = { + textToImage: parseWorkflowTemplate(textToImage), + imageToImage: parseWorkflowTemplate(imageToImage), + textToVideo: parseWorkflowTemplate(textToVideo), + imageToVideo: parseWorkflowTemplate(imageToVideo), +} as const; diff --git a/packages/core/src/media/workflows/README.md b/packages/core/src/media/workflows/README.md new file mode 100644 index 0000000..af97d1b --- /dev/null +++ b/packages/core/src/media/workflows/README.md @@ -0,0 +1,35 @@ +# ComfyUI workflows + +Default workflows for `generate_image`, `generate_video` and repainting in `edit_image`, in +ComfyUI's API format. A custom workflow in the Media settings replaces the matching default: +one that uses `{{image}}` replaces the image-to-image (or image-to-video) default, one without +it replaces the text-to-image (or text-to-video) default. + +## Placeholders + +A string value that is exactly `{{name}}` becomes the typed value; `{{name}}` inside a longer +string is replaced by its text. Every workflow needs `{{prompt}}`; other names are optional. + +| Placeholder | Value | +|---|---| +| `{{prompt}}`, `{{negative_prompt}}` | text | +| `{{model}}` | the configured image model, or the first installed checkpoint | +| `{{image}}` | the uploaded input image (for `LoadImage`) | +| `{{width}}`, `{{height}}` | output size in pixels | +| `{{seed}}`, `{{steps}}` | integers | +| `{{strength}}` | 0–1, image-to-image denoise | +| `{{frames}}`, `{{fps}}` | video length in frames (8n+1) and frame rate | + +To write your own, build it in ComfyUI, replace the values above with placeholders and export it +with **Export (API)**. + +## Model files + +- Images: any SD 1.5 or SDXL checkpoint in `models/checkpoints/`. +- Video (LTX-Video, core nodes, about 12 GB of VRAM): + - `models/checkpoints/ltx-video-2b-v0.9.5.safetensors` from + [Lightricks/LTX-Video](https://huggingface.co/Lightricks/LTX-Video) + - `models/text_encoders/t5xxl_fp16.safetensors` from + [comfyanonymous/flux_text_encoders](https://huggingface.co/comfyanonymous/flux_text_encoders) + +The video workflows need a ComfyUI release with the `CreateVideo` and `SaveVideo` nodes. diff --git a/packages/core/src/media/workflows/image-to-image.json b/packages/core/src/media/workflows/image-to-image.json new file mode 100644 index 0000000..e762d24 --- /dev/null +++ b/packages/core/src/media/workflows/image-to-image.json @@ -0,0 +1,45 @@ +{ + "1": { + "class_type": "CheckpointLoaderSimple", + "inputs": { "ckpt_name": "{{model}}" } + }, + "2": { + "class_type": "CLIPTextEncode", + "inputs": { "text": "{{prompt}}", "clip": ["1", 1] } + }, + "3": { + "class_type": "CLIPTextEncode", + "inputs": { "text": "{{negative_prompt}}", "clip": ["1", 1] } + }, + "4": { + "class_type": "LoadImage", + "inputs": { "image": "{{image}}" } + }, + "5": { + "class_type": "VAEEncode", + "inputs": { "pixels": ["4", 0], "vae": ["1", 2] } + }, + "6": { + "class_type": "KSampler", + "inputs": { + "model": ["1", 0], + "positive": ["2", 0], + "negative": ["3", 0], + "latent_image": ["5", 0], + "seed": "{{seed}}", + "steps": "{{steps}}", + "cfg": 7, + "sampler_name": "dpmpp_2m", + "scheduler": "karras", + "denoise": "{{strength}}" + } + }, + "7": { + "class_type": "VAEDecode", + "inputs": { "samples": ["6", 0], "vae": ["1", 2] } + }, + "8": { + "class_type": "SaveImage", + "inputs": { "images": ["7", 0], "filename_prefix": "evoo/image" } + } +} diff --git a/packages/core/src/media/workflows/image-to-video.json b/packages/core/src/media/workflows/image-to-video.json new file mode 100644 index 0000000..6bf53a7 --- /dev/null +++ b/packages/core/src/media/workflows/image-to-video.json @@ -0,0 +1,86 @@ +{ + "1": { + "class_type": "CheckpointLoaderSimple", + "inputs": { "ckpt_name": "ltx-video-2b-v0.9.5.safetensors" } + }, + "2": { + "class_type": "CLIPLoader", + "inputs": { "clip_name": "t5xxl_fp16.safetensors", "type": "ltxv" } + }, + "3": { + "class_type": "CLIPTextEncode", + "inputs": { "text": "{{prompt}}", "clip": ["2", 0] } + }, + "4": { + "class_type": "CLIPTextEncode", + "inputs": { "text": "{{negative_prompt}}", "clip": ["2", 0] } + }, + "5": { + "class_type": "LoadImage", + "inputs": { "image": "{{image}}" } + }, + "6": { + "class_type": "LTXVImgToVideo", + "inputs": { + "positive": ["3", 0], + "negative": ["4", 0], + "vae": ["1", 2], + "image": ["5", 0], + "width": "{{width}}", + "height": "{{height}}", + "length": "{{frames}}", + "batch_size": 1, + "strength": 1 + } + }, + "7": { + "class_type": "LTXVConditioning", + "inputs": { "positive": ["6", 0], "negative": ["6", 1], "frame_rate": "{{fps}}" } + }, + "8": { + "class_type": "LTXVScheduler", + "inputs": { + "steps": "{{steps}}", + "max_shift": 2.05, + "base_shift": 0.95, + "stretch": true, + "terminal": 0.1, + "latent": ["6", 2] + } + }, + "9": { + "class_type": "KSamplerSelect", + "inputs": { "sampler_name": "euler" } + }, + "10": { + "class_type": "SamplerCustom", + "inputs": { + "model": ["1", 0], + "add_noise": true, + "noise_seed": "{{seed}}", + "cfg": 3, + "positive": ["7", 0], + "negative": ["7", 1], + "sampler": ["9", 0], + "sigmas": ["8", 0], + "latent_image": ["6", 2] + } + }, + "11": { + "class_type": "VAEDecode", + "inputs": { "samples": ["10", 0], "vae": ["1", 2] } + }, + "12": { + "class_type": "CreateVideo", + "inputs": { "images": ["11", 0], "fps": "{{fps}}" } + }, + "13": { + "class_type": "SaveVideo", + "inputs": { + "video": ["12", 0], + "filename_prefix": "evoo/video", + "format": "mp4", + "codec": "h264" + } + } +} diff --git a/packages/core/src/media/workflows/text-to-image.json b/packages/core/src/media/workflows/text-to-image.json new file mode 100644 index 0000000..633cdc5 --- /dev/null +++ b/packages/core/src/media/workflows/text-to-image.json @@ -0,0 +1,41 @@ +{ + "1": { + "class_type": "CheckpointLoaderSimple", + "inputs": { "ckpt_name": "{{model}}" } + }, + "2": { + "class_type": "CLIPTextEncode", + "inputs": { "text": "{{prompt}}", "clip": ["1", 1] } + }, + "3": { + "class_type": "CLIPTextEncode", + "inputs": { "text": "{{negative_prompt}}", "clip": ["1", 1] } + }, + "4": { + "class_type": "EmptyLatentImage", + "inputs": { "width": "{{width}}", "height": "{{height}}", "batch_size": 1 } + }, + "5": { + "class_type": "KSampler", + "inputs": { + "model": ["1", 0], + "positive": ["2", 0], + "negative": ["3", 0], + "latent_image": ["4", 0], + "seed": "{{seed}}", + "steps": "{{steps}}", + "cfg": 7, + "sampler_name": "dpmpp_2m", + "scheduler": "karras", + "denoise": 1 + } + }, + "6": { + "class_type": "VAEDecode", + "inputs": { "samples": ["5", 0], "vae": ["1", 2] } + }, + "7": { + "class_type": "SaveImage", + "inputs": { "images": ["6", 0], "filename_prefix": "evoo/image" } + } +} diff --git a/packages/core/src/media/workflows/text-to-video.json b/packages/core/src/media/workflows/text-to-video.json new file mode 100644 index 0000000..55a3b3c --- /dev/null +++ b/packages/core/src/media/workflows/text-to-video.json @@ -0,0 +1,77 @@ +{ + "1": { + "class_type": "CheckpointLoaderSimple", + "inputs": { "ckpt_name": "ltx-video-2b-v0.9.5.safetensors" } + }, + "2": { + "class_type": "CLIPLoader", + "inputs": { "clip_name": "t5xxl_fp16.safetensors", "type": "ltxv" } + }, + "3": { + "class_type": "CLIPTextEncode", + "inputs": { "text": "{{prompt}}", "clip": ["2", 0] } + }, + "4": { + "class_type": "CLIPTextEncode", + "inputs": { "text": "{{negative_prompt}}", "clip": ["2", 0] } + }, + "5": { + "class_type": "LTXVConditioning", + "inputs": { "positive": ["3", 0], "negative": ["4", 0], "frame_rate": "{{fps}}" } + }, + "6": { + "class_type": "EmptyLTXVLatentVideo", + "inputs": { + "width": "{{width}}", + "height": "{{height}}", + "length": "{{frames}}", + "batch_size": 1 + } + }, + "7": { + "class_type": "LTXVScheduler", + "inputs": { + "steps": "{{steps}}", + "max_shift": 2.05, + "base_shift": 0.95, + "stretch": true, + "terminal": 0.1, + "latent": ["6", 0] + } + }, + "8": { + "class_type": "KSamplerSelect", + "inputs": { "sampler_name": "euler" } + }, + "9": { + "class_type": "SamplerCustom", + "inputs": { + "model": ["1", 0], + "add_noise": true, + "noise_seed": "{{seed}}", + "cfg": 3, + "positive": ["5", 0], + "negative": ["5", 1], + "sampler": ["8", 0], + "sigmas": ["7", 0], + "latent_image": ["6", 0] + } + }, + "10": { + "class_type": "VAEDecode", + "inputs": { "samples": ["9", 0], "vae": ["1", 2] } + }, + "11": { + "class_type": "CreateVideo", + "inputs": { "images": ["10", 0], "fps": "{{fps}}" } + }, + "12": { + "class_type": "SaveVideo", + "inputs": { + "video": ["11", 0], + "filename_prefix": "evoo/video", + "format": "mp4", + "codec": "h264" + } + } +}