import type { MediaJobStatus, MediaOperation, MediaOutput, MediaOutputKind, MediaPresetId, } from "@rakazo/contracts"; import type { AdapterDescriptor } from "./types.js"; /** Bytes handed to or returned by a media backend. */ export interface MediaFile { bytes: Uint8Array; mimeType: string; name?: string; } export interface GeneratedMedia extends MediaFile { width?: number; height?: number; durationMs?: number; } /** Reports progress as a fraction from 0 to 1. */ export type MediaProgressListener = (fraction: number) => void; /** Free and total memory of the backend's GPU, when the backend can tell. */ export interface GpuStatus { vramFreeMb: number; vramTotalMb: number; } export interface ImageGenerationRequest { prompt: string; negativePrompt?: string; width: number; height: number; /** Number of images, default 1. */ count?: number; seed?: number; /** Source image for image-to-image and inpainting. */ source?: MediaFile; /** 0–1: how far the result may move away from the source. */ strength?: number; /** Inpainting mask, same size as the source; white areas are repainted. */ mask?: MediaFile; /** Workflow template id; the provider's default when omitted. */ workflow?: string; signal?: AbortSignal; onProgress?: MediaProgressListener; } export interface VideoGenerationRequest { prompt: string; negativePrompt?: string; width: number; height: number; durationSeconds: number; fps?: number; seed?: number; /** Start frame for image-to-video. */ source?: MediaFile; workflow?: string; signal?: AbortSignal; onProgress?: MediaProgressListener; } export interface ImageGenerationCapabilities { textToImage: boolean; imageToImage: boolean; inpaint: boolean; maxWidth: number; maxHeight: number; } export interface VideoGenerationCapabilities { textToVideo: boolean; imageToVideo: boolean; maxDurationSeconds: number; } export interface ImageGenerationProvider { describe(): AdapterDescriptor; generateImage(request: ImageGenerationRequest): Promise; /** Lets the media runner wait for free GPU memory instead of failing mid-generation. */ gpuStatus?(signal?: AbortSignal): Promise; } export interface VideoGenerationProvider { describe(): AdapterDescriptor; generateVideo(request: VideoGenerationRequest): Promise; gpuStatus?(signal?: AbortSignal): Promise; } export interface TranscriptSegment { startMs: number; endMs: number; text: string; } export interface Transcript { text: string; /** Detected or requested language (BCP 47), when the backend reports it. */ language?: string; /** Timed segments; empty when the backend returns plain text only. */ segments: TranscriptSegment[]; } export interface TranscriptionRequest { audio: Uint8Array; mimeType: string; fileName?: string; /** Language hint (BCP 47); detected when omitted. */ language?: string; signal?: AbortSignal; } export interface TranscriptionCapabilities { segments: boolean; languageDetection: boolean; maxBytes: number; } export interface TranscriptionProvider { describe(): AdapterDescriptor; transcribe(request: TranscriptionRequest): Promise; } /** A file an operation produced in its temp directory, to be stored as an artifact. */ export interface MediaJobOutputFile { path: string; name: string; mimeType: string; width?: number; height?: number; durationMs?: number; /** Poster frame for a video, as a file in the temp directory. */ posterPath?: string; parentArtifactId?: string; preset?: MediaPresetId; } /** An artifact the job may read, copied into the job's temp directory. */ export interface MediaJobInputFile { artifactId: string; path: string; name: string; mimeType: string; kind: MediaOutputKind; size: number; width?: number; height?: number; durationMs?: number; } /** What the media runner hands an operation while a job runs. */ export interface MediaJobContext { jobId: string; spaceId: string; userId: string; botId: string | null; /** The input stored with the job; each operation validates it with its own schema. */ input: unknown; /** Private scratch directory, removed when the job ends. */ tempDir: string; signal: AbortSignal; progress(fraction: number): Promise; /** Copies an artifact of the job's space that the job's user may read into tempDir. */ fetchArtifact(artifactId: string): Promise; /** Stores a produced file as an artifact and adds it to the media card. */ addOutput(file: MediaJobOutputFile): Promise; } export interface MediaOperationResult { /** One or two sentences for the bot that started the job. */ summary: string; } export interface MediaOperationHandler { operation: MediaOperation; run(context: MediaJobContext): Promise; } /** Thrown by an operation that cannot start yet, e.g. while the GPU lacks free memory. */ export class MediaJobDeferredError extends Error { constructor( message: string, readonly retryAfterMs: number, ) { super(message); this.name = "MediaJobDeferredError"; } } export interface StartMediaJobInput { operation: MediaOperation; /** Validated by the operation's input schema before the job is created. */ input: unknown; /** Card label; the operation derives one from the input when omitted. */ title?: string; spaceId: string; userId: string; /** Author of the media card; a job started from a card keeps that card's bot. */ botId: string | null; groupId: string | null; threadId: string; runId: string | null; /** Wake the starting bot with the job's summary when the job ends. */ continueAfter?: boolean; /** A replay with the same key returns the job created first (tool call idempotency). */ operationKey?: string; } export interface MediaJobSnapshot { id: string; operation: MediaOperation; status: MediaJobStatus; title: string; progress: number | null; error: string | null; outputs: MediaOutput[]; } /** Which media operations the space can run right now. */ export type MediaCapabilities = Record; /** * Starts media jobs and answers which operations are available. Tools call this; the job runs * in a background worker and keeps its media card current. */ export interface MediaRuntime { capabilities(spaceId: string): Promise; start(input: StartMediaJobInput): Promise<{ jobId: string; messageId: string }>; get(jobId: string): Promise; cancel(jobId: string): Promise; }