Claude 4217e4b4af
feat(media): generate images and video on ComfyUI and edit images
Bots can now make images and short clips on the user's own GPU server and
rework images locally, through the media job runtime.

- generate_image: prompt, aspect preset or exact size, up to 4 images, optional
  source image for image-to-image. Sizes target what the default models are
  tuned for, and sources are fitted to that size before upload, so large
  photos cannot exhaust GPU memory.
- generate_video: prompt, aspect, duration up to 10 s, optional start frame.
- edit_image: crop, rotate, flip, resize to a size or format (crop, pad or
  blurred background), extend, convert, in that order, with sharp; an optional
  prompt repaints the result on ComfyUI. Edits keep the source as parent.
- Registrations declare the input artifacts so jobs refuse ones the user
  cannot read, and generation declares a fixed GPU memory floor (6 GB images,
  12 GB video) that a custom workflow cannot lower.
- Tool definitions reuse the operation input schemas plus continue_after.
- Saving media settings rejects custom workflows the provider could not fill.
- Tests: sharp edits on generated fixtures, handlers against the ComfyUI
  emulator, and a PostgreSQL suite running the operations through the media
  runtime (card outputs, artifact versions, foreign artifacts, GPU wait).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
2026-10-03 13:42:35 +00:00

157 lines
5.7 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import type { EditImageInput, ImageFormat } from "@rakazo/contracts";
import { MEDIA_PRESETS } from "@rakazo/contracts";
import type sharpFactory from "sharp";
import type { Sharp } from "sharp";
type SharpFactory = typeof sharpFactory;
export type ImageGeometryEdits = Pick<
EditImageInput,
"crop" | "rotate" | "flip" | "resize" | "extend"
>;
/** Edited images stay below this many pixels (8192²). */
const MAX_EDIT_PIXELS = 8192 * 8192;
const TRANSPARENT = { r: 0, g: 0, b: 0, alpha: 0 };
const BLACK = { r: 0, g: 0, b: 0 };
let sharpModule: Promise<SharpFactory> | undefined;
/** sharp is a native module; load it on first use so a platform without it only loses edits. */
export function loadSharp(): Promise<SharpFactory> {
sharpModule ??= import("sharp").then((module) => module.default);
return sharpModule;
}
export async function sharpAvailable(): Promise<boolean> {
return loadSharp().then(
() => true,
() => false,
);
}
/** Displayed size, with EXIF orientation applied. */
export async function imageSize(
input: string | Uint8Array,
): Promise<{ width: number; height: number }> {
const sharp = await loadSharp();
const { autoOrient } = await sharp(input).metadata();
return { width: autoOrient.width, height: autoOrient.height };
}
/** Covers exactly width×height (center crop) as PNG: generation inputs at the size asked for. */
export async function coverImage(
input: string | Uint8Array,
width: number,
height: number,
): Promise<Uint8Array> {
const sharp = await loadSharp();
const png = await sharp(input, { autoOrient: true })
.resize(width, height, { fit: "cover" })
.png()
.toBuffer();
return new Uint8Array(png);
}
function parseColor(hex: string | undefined) {
if (!hex) return TRANSPARENT;
const value = Number.parseInt(hex.slice(1), 16);
return { r: (value >> 16) & 255, g: (value >> 8) & 255, b: value & 255, alpha: 1 };
}
function assertSize(width: number, height: number): void {
if (width * height > MAX_EDIT_PIXELS) throw new Error("The edited image would be too large");
}
/** Runs each step on decoded pixels, so steps apply in the order given, not sharp's own order. */
async function materialize(sharp: SharpFactory, image: Sharp) {
const { data, info } = await image.raw().toBuffer({ resolveWithObject: true });
const { width, height, channels } = info;
return { image: sharp(data, { raw: { width, height, channels } }), width, height };
}
async function resizeTo(
sharp: SharpFactory,
image: Sharp,
width: number,
height: number,
fit: "crop" | "pad" | "blur",
): Promise<Sharp> {
if (fit === "crop") return image.resize(width, height, { fit: "cover" });
if (fit === "pad")
return image.resize(width, height, { fit: "contain", background: TRANSPARENT });
const source = await image.png().toBuffer();
const background = await sharp(source)
.resize(width, height, { fit: "cover" })
.blur(Math.max(width, height) / 40)
.toBuffer();
const foreground = await sharp(source).resize(width, height, { fit: "inside" }).toBuffer();
return sharp(background).composite([{ input: foreground, gravity: "centre" }]);
}
/**
* Applies crop, rotate, flip, resize and extend in that order and returns a lossless PNG.
* EXIF orientation is applied first, so coordinates match what viewers show.
*/
export async function editImageGeometry(
input: string | Uint8Array,
edits: ImageGeometryEdits,
): Promise<Buffer> {
const sharp = await loadSharp();
let current = await materialize(sharp, sharp(input, { autoOrient: true }));
const step = async (apply: (image: Sharp) => Sharp | Promise<Sharp>) => {
current = await materialize(sharp, await apply(current.image));
};
const { crop, rotate, flip, resize, extend } = edits;
if (crop) {
if (crop.left + crop.width > current.width || crop.top + crop.height > current.height) {
throw new Error(`The crop area is outside the ${current.width}×${current.height} image`);
}
await step((image) => image.extract(crop));
}
if (rotate) await step((image) => image.rotate(rotate));
if (flip) await step((image) => (flip === "vertical" ? image.flip() : image.flop()));
if (resize) {
const preset = resize.aspect ? MEDIA_PRESETS[resize.aspect] : undefined;
const width = preset?.width ?? resize.width;
const height = preset?.height ?? resize.height;
const scale = width ? width / current.width : height! / current.height;
assertSize(width ?? current.width * scale, height ?? current.height * scale);
await step((image) =>
width && height
? resizeTo(sharp, image, width, height, resize.fit)
: image.resize(width ?? null, height ?? null),
);
}
if (extend) {
const { top, right, bottom, left, color } = extend;
assertSize(current.width + left + right, current.height + top + bottom);
await step((image) =>
image.extend({ top, right, bottom, left, background: parseColor(color) }),
);
}
return current.image.png().toBuffer();
}
export const IMAGE_FORMAT_MIME_TYPES: Record<ImageFormat, string> = {
png: "image/png",
jpeg: "image/jpeg",
webp: "image/webp",
};
/** Encodes the final image; JPEG has no transparency, so transparent areas turn black. */
export async function encodeImage(
input: Uint8Array,
format: ImageFormat,
): Promise<{ bytes: Buffer; width: number; height: number }> {
const sharp = await loadSharp();
const image = sharp(input);
const encoded =
format === "jpeg"
? image.flatten({ background: BLACK }).jpeg({ quality: 90, mozjpeg: true })
: format === "webp"
? image.webp({ quality: 90 })
: image.png();
const { data, info } = await encoded.toBuffer({ resolveWithObject: true });
return { bytes: data, width: info.width, height: info.height };
}