Bots can now make images and short clips on the user's own GPU server and rework images locally, through the media job runtime. - generate_image: prompt, aspect preset or exact size, up to 4 images, optional source image for image-to-image. Sizes target what the default models are tuned for, and sources are fitted to that size before upload, so large photos cannot exhaust GPU memory. - generate_video: prompt, aspect, duration up to 10 s, optional start frame. - edit_image: crop, rotate, flip, resize to a size or format (crop, pad or blurred background), extend, convert, in that order, with sharp; an optional prompt repaints the result on ComfyUI. Edits keep the source as parent. - Registrations declare the input artifacts so jobs refuse ones the user cannot read, and generation declares a fixed GPU memory floor (6 GB images, 12 GB video) that a custom workflow cannot lower. - Tool definitions reuse the operation input schemas plus continue_after. - Saving media settings rejects custom workflows the provider could not fill. - Tests: sharp edits on generated fixtures, handlers against the ComfyUI emulator, and a PostgreSQL suite running the operations through the media runtime (card outputs, artifact versions, foreign artifacts, GPU wait). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
157 lines
5.7 KiB
TypeScript
157 lines
5.7 KiB
TypeScript
import type { EditImageInput, ImageFormat } from "@rakazo/contracts";
|
||
import { MEDIA_PRESETS } from "@rakazo/contracts";
|
||
import type sharpFactory from "sharp";
|
||
import type { Sharp } from "sharp";
|
||
|
||
type SharpFactory = typeof sharpFactory;
|
||
export type ImageGeometryEdits = Pick<
|
||
EditImageInput,
|
||
"crop" | "rotate" | "flip" | "resize" | "extend"
|
||
>;
|
||
|
||
/** Edited images stay below this many pixels (8192²). */
|
||
const MAX_EDIT_PIXELS = 8192 * 8192;
|
||
const TRANSPARENT = { r: 0, g: 0, b: 0, alpha: 0 };
|
||
const BLACK = { r: 0, g: 0, b: 0 };
|
||
|
||
let sharpModule: Promise<SharpFactory> | undefined;
|
||
|
||
/** sharp is a native module; load it on first use so a platform without it only loses edits. */
|
||
export function loadSharp(): Promise<SharpFactory> {
|
||
sharpModule ??= import("sharp").then((module) => module.default);
|
||
return sharpModule;
|
||
}
|
||
|
||
export async function sharpAvailable(): Promise<boolean> {
|
||
return loadSharp().then(
|
||
() => true,
|
||
() => false,
|
||
);
|
||
}
|
||
|
||
/** Displayed size, with EXIF orientation applied. */
|
||
export async function imageSize(
|
||
input: string | Uint8Array,
|
||
): Promise<{ width: number; height: number }> {
|
||
const sharp = await loadSharp();
|
||
const { autoOrient } = await sharp(input).metadata();
|
||
return { width: autoOrient.width, height: autoOrient.height };
|
||
}
|
||
|
||
/** Covers exactly width×height (center crop) as PNG: generation inputs at the size asked for. */
|
||
export async function coverImage(
|
||
input: string | Uint8Array,
|
||
width: number,
|
||
height: number,
|
||
): Promise<Uint8Array> {
|
||
const sharp = await loadSharp();
|
||
const png = await sharp(input, { autoOrient: true })
|
||
.resize(width, height, { fit: "cover" })
|
||
.png()
|
||
.toBuffer();
|
||
return new Uint8Array(png);
|
||
}
|
||
|
||
function parseColor(hex: string | undefined) {
|
||
if (!hex) return TRANSPARENT;
|
||
const value = Number.parseInt(hex.slice(1), 16);
|
||
return { r: (value >> 16) & 255, g: (value >> 8) & 255, b: value & 255, alpha: 1 };
|
||
}
|
||
|
||
function assertSize(width: number, height: number): void {
|
||
if (width * height > MAX_EDIT_PIXELS) throw new Error("The edited image would be too large");
|
||
}
|
||
|
||
/** Runs each step on decoded pixels, so steps apply in the order given, not sharp's own order. */
|
||
async function materialize(sharp: SharpFactory, image: Sharp) {
|
||
const { data, info } = await image.raw().toBuffer({ resolveWithObject: true });
|
||
const { width, height, channels } = info;
|
||
return { image: sharp(data, { raw: { width, height, channels } }), width, height };
|
||
}
|
||
|
||
async function resizeTo(
|
||
sharp: SharpFactory,
|
||
image: Sharp,
|
||
width: number,
|
||
height: number,
|
||
fit: "crop" | "pad" | "blur",
|
||
): Promise<Sharp> {
|
||
if (fit === "crop") return image.resize(width, height, { fit: "cover" });
|
||
if (fit === "pad")
|
||
return image.resize(width, height, { fit: "contain", background: TRANSPARENT });
|
||
const source = await image.png().toBuffer();
|
||
const background = await sharp(source)
|
||
.resize(width, height, { fit: "cover" })
|
||
.blur(Math.max(width, height) / 40)
|
||
.toBuffer();
|
||
const foreground = await sharp(source).resize(width, height, { fit: "inside" }).toBuffer();
|
||
return sharp(background).composite([{ input: foreground, gravity: "centre" }]);
|
||
}
|
||
|
||
/**
|
||
* Applies crop, rotate, flip, resize and extend in that order and returns a lossless PNG.
|
||
* EXIF orientation is applied first, so coordinates match what viewers show.
|
||
*/
|
||
export async function editImageGeometry(
|
||
input: string | Uint8Array,
|
||
edits: ImageGeometryEdits,
|
||
): Promise<Buffer> {
|
||
const sharp = await loadSharp();
|
||
let current = await materialize(sharp, sharp(input, { autoOrient: true }));
|
||
const step = async (apply: (image: Sharp) => Sharp | Promise<Sharp>) => {
|
||
current = await materialize(sharp, await apply(current.image));
|
||
};
|
||
const { crop, rotate, flip, resize, extend } = edits;
|
||
if (crop) {
|
||
if (crop.left + crop.width > current.width || crop.top + crop.height > current.height) {
|
||
throw new Error(`The crop area is outside the ${current.width}×${current.height} image`);
|
||
}
|
||
await step((image) => image.extract(crop));
|
||
}
|
||
if (rotate) await step((image) => image.rotate(rotate));
|
||
if (flip) await step((image) => (flip === "vertical" ? image.flip() : image.flop()));
|
||
if (resize) {
|
||
const preset = resize.aspect ? MEDIA_PRESETS[resize.aspect] : undefined;
|
||
const width = preset?.width ?? resize.width;
|
||
const height = preset?.height ?? resize.height;
|
||
const scale = width ? width / current.width : height! / current.height;
|
||
assertSize(width ?? current.width * scale, height ?? current.height * scale);
|
||
await step((image) =>
|
||
width && height
|
||
? resizeTo(sharp, image, width, height, resize.fit)
|
||
: image.resize(width ?? null, height ?? null),
|
||
);
|
||
}
|
||
if (extend) {
|
||
const { top, right, bottom, left, color } = extend;
|
||
assertSize(current.width + left + right, current.height + top + bottom);
|
||
await step((image) =>
|
||
image.extend({ top, right, bottom, left, background: parseColor(color) }),
|
||
);
|
||
}
|
||
return current.image.png().toBuffer();
|
||
}
|
||
|
||
export const IMAGE_FORMAT_MIME_TYPES: Record<ImageFormat, string> = {
|
||
png: "image/png",
|
||
jpeg: "image/jpeg",
|
||
webp: "image/webp",
|
||
};
|
||
|
||
/** Encodes the final image; JPEG has no transparency, so transparent areas turn black. */
|
||
export async function encodeImage(
|
||
input: Uint8Array,
|
||
format: ImageFormat,
|
||
): Promise<{ bytes: Buffer; width: number; height: number }> {
|
||
const sharp = await loadSharp();
|
||
const image = sharp(input);
|
||
const encoded =
|
||
format === "jpeg"
|
||
? image.flatten({ background: BLACK }).jpeg({ quality: 90, mozjpeg: true })
|
||
: format === "webp"
|
||
? image.webp({ quality: 90 })
|
||
: image.png();
|
||
const { data, info } = await encoded.toBuffer({ resolveWithObject: true });
|
||
return { bytes: data, width: info.width, height: info.height };
|
||
}
|