evOo-local/packages/adapter-kit/src/media.ts

import type {
  MediaCheck,
  MediaJobStatus,
  MediaOperation,
  MediaOutput,
  MediaOutputKind,
  MediaPlatformId,
  MediaPresetId,
  MediaSkippedTarget,
} from "@rakazo/contracts";
import type { AdapterDescriptor } from "./types.js";

/** Bytes handed to or returned by a media backend. */
export interface MediaFile {
  bytes: Uint8Array;
  mimeType: string;
  name?: string;
}

export interface GeneratedMedia extends MediaFile {
  width?: number;
  height?: number;
  durationMs?: number;
}

/** Reports progress as a fraction from 0 to 1. */
export type MediaProgressListener = (fraction: number) => void;

/** Free and total memory of the backend's GPU, when the backend can tell. */
export interface GpuStatus {
  vramFreeMb: number;
  vramTotalMb: number;
}

export interface ImageGenerationRequest {
  prompt: string;
  negativePrompt?: string;
  width: number;
  height: number;
  /** Number of images, default 1. */
  count?: number;
  seed?: number;
  /** Source image for image-to-image and inpainting. */
  source?: MediaFile;
  /** 0–1: how far the result may move away from the source. */
  strength?: number;
  /** Inpainting mask, same size as the source; white areas are repainted. */
  mask?: MediaFile;
  /** Workflow template id; the provider's default when omitted. */
  workflow?: string;
  signal?: AbortSignal;
  onProgress?: MediaProgressListener;
}

export interface VideoGenerationRequest {
  prompt: string;
  negativePrompt?: string;
  width: number;
  height: number;
  durationSeconds: number;
  fps?: number;
  seed?: number;
  /** Start frame for image-to-video. */
  source?: MediaFile;
  workflow?: string;
  signal?: AbortSignal;
  onProgress?: MediaProgressListener;
}

export interface ImageGenerationCapabilities {
  textToImage: boolean;
  imageToImage: boolean;
  inpaint: boolean;
  maxWidth: number;
  maxHeight: number;
}

export interface VideoGenerationCapabilities {
  textToVideo: boolean;
  imageToVideo: boolean;
  maxDurationSeconds: number;
}

export interface ImageGenerationProvider {
  describe(): AdapterDescriptor<ImageGenerationCapabilities>;
  generateImage(request: ImageGenerationRequest): Promise<GeneratedMedia[]>;
  /** Lets the media runner wait for free GPU memory instead of failing mid-generation. */
  gpuStatus?(signal?: AbortSignal): Promise<GpuStatus | undefined>;
}

export interface VideoGenerationProvider {
  describe(): AdapterDescriptor<VideoGenerationCapabilities>;
  generateVideo(request: VideoGenerationRequest): Promise<GeneratedMedia[]>;
  gpuStatus?(signal?: AbortSignal): Promise<GpuStatus | undefined>;
}

export interface TranscriptSegment {
  startMs: number;
  endMs: number;
  text: string;
}

export interface Transcript {
  text: string;
  /** Detected or requested language (BCP 47), when the backend reports it. */
  language?: string;
  /** Timed segments; empty when the backend returns plain text only. */
  segments: TranscriptSegment[];
}

export interface TranscriptionRequest {
  audio: Uint8Array;
  mimeType: string;
  fileName?: string;
  /** Language hint (BCP 47); detected when omitted. */
  language?: string;
  signal?: AbortSignal;
}

export interface TranscriptionCapabilities {
  segments: boolean;
  languageDetection: boolean;
  maxBytes: number;
}

export interface TranscriptionProvider {
  describe(): AdapterDescriptor<TranscriptionCapabilities>;
  transcribe(request: TranscriptionRequest): Promise<Transcript>;
}

/** A file an operation produced in its temp directory, to be stored as an artifact. */
export interface MediaJobOutputFile {
  path: string;
  name: string;
  mimeType: string;
  width?: number;
  height?: number;
  durationMs?: number;
  /** Poster frame for a video, as a file in the temp directory. */
  posterPath?: string;
  parentArtifactId?: string;
  preset?: MediaPresetId;
  platform?: MediaPlatformId;
  check?: MediaCheck;
}

/** An artifact the job may read, copied into the job's temp directory. */
export interface MediaJobInputFile {
  artifactId: string;
  path: string;
  name: string;
  mimeType: string;
  kind: MediaOutputKind;
  size: number;
  width?: number;
  height?: number;
  durationMs?: number;
}

/** What the media runner hands an operation while a job runs. */
export interface MediaJobContext {
  jobId: string;
  spaceId: string;
  userId: string;
  botId: string | null;
  /** The input stored with the job; each operation validates it with its own schema. */
  input: unknown;
  /** Private scratch directory, removed when the job ends. */
  tempDir: string;
  signal: AbortSignal;
  progress(fraction: number): Promise<void>;
  /** Copies an artifact of the job's space that the job's user may read into tempDir. */
  fetchArtifact(artifactId: string): Promise<MediaJobInputFile>;
  /**
   * Fetches the newest artifact made from `artifactId` (a transcript of a video, say) with one
   * of these MIME types, like `fetchArtifact`; null when the user has none.
   */
  fetchDerivedArtifact(
    artifactId: string,
    mimeTypes: readonly string[],
  ): Promise<MediaJobInputFile | null>;
  /** Stores a produced file as an artifact and adds it to the media card. */
  addOutput(file: MediaJobOutputFile): Promise<MediaOutput>;
  /** Shows a platform the job did not render on the media card, with the reason. */
  addSkipped(target: MediaSkippedTarget): Promise<void>;
}

export interface MediaOperationResult {
  /** One or two sentences for the bot that started the job. */
  summary: string;
}

export interface MediaOperationHandler {
  operation: MediaOperation;
  run(context: MediaJobContext): Promise<MediaOperationResult>;
}

/** Thrown by an operation that cannot start yet, e.g. while the GPU lacks free memory. */
export class MediaJobDeferredError extends Error {
  constructor(
    message: string,
    readonly retryAfterMs: number,
  ) {
    super(message);
    this.name = "MediaJobDeferredError";
  }
}

export interface StartMediaJobInput {
  operation: MediaOperation;
  /** Validated by the operation's input schema before the job is created. */
  input: unknown;
  /** Card label; the operation derives one from the input when omitted. */
  title?: string;
  spaceId: string;
  userId: string;
  /** Author of the media card; a job started from a card keeps that card's bot. */
  botId: string | null;
  groupId: string | null;
  threadId: string;
  runId: string | null;
  /** Wake the starting bot with the job's summary when the job ends. */
  continueAfter?: boolean;
  /** A replay with the same key returns the job created first (tool call idempotency). */
  operationKey?: string;
}

export interface MediaJobSnapshot {
  id: string;
  operation: MediaOperation;
  status: MediaJobStatus;
  title: string;
  progress: number | null;
  error: string | null;
  outputs: MediaOutput[];
}

/** Which media operations the space can run right now. */
export type MediaCapabilities = Record<MediaOperation, boolean>;

/**
 * Starts media jobs and answers which operations are available. Tools call this; the job runs
 * in a background worker and keeps its media card current.
 */
export interface MediaRuntime {
  capabilities(spaceId: string): Promise<MediaCapabilities>;
  start(input: StartMediaJobInput): Promise<{ jobId: string; messageId: string }>;
  get(jobId: string): Promise<MediaJobSnapshot | null>;
  cancel(jobId: string): Promise<void>;
}