import type { AnonymizeMode, BoxKeyframe, BrandKit, ContentCategory, FfprobeReport, MediaAssetKind, MediaCapabilities, MediaInferenceOperation, MediaMarkPrompt, MediaSourceKind, MotionTemplateInfo, MotionTemplateRef, NormalizedBox, NormalizedPoint, PlatformProfile, RenderPlan, } from "@rakazo/contracts"; /* * Media Studio provider boundaries. Every implementation (local sidecar, * emulator, optional cloud adapters) satisfies the same contracts, which * media-conformance tests exercise offline. * * Paths are absolute host paths inside the configured media root. Providers * must reject any path outside that root. */ // --------------------------------------------------------------------------- // Sources // --------------------------------------------------------------------------- export interface MediaSourceItem { externalId: string; kind: MediaAssetKind; capturedAt: Date | null; durationMs: number | null; width: number | null; height: number | null; fps: number | null; gpsLat: number | null; gpsLon: number | null; deviceModel: string | null; albumNames: string[]; isFavorite: boolean; fileName: string | null; sizeBytes: number | null; /** Cheap change detector (mtime + size or provider revision); not a content hash. */ revision: string; } export interface MediaSourceListRequest { config: Record; cursor: string | null; limit: number; signal?: AbortSignal; } export interface MediaSourceListResult { items: MediaSourceItem[]; nextCursor: string | null; } /** Server-side sources the worker can enumerate and read directly. */ export interface MediaSourceProvider { readonly kind: Extract; list(request: MediaSourceListRequest): Promise; /** Absolute readable path of the original file. Throws when it escaped the source root. */ resolve(config: Record, externalId: string): Promise; } // --------------------------------------------------------------------------- // Inference // --------------------------------------------------------------------------- export class UnsupportedMediaOperationError extends Error { constructor( readonly operation: MediaInferenceOperation, readonly reason: string, ) { super(`${operation} is not available: ${reason}`); this.name = "UnsupportedMediaOperationError"; } } export interface MediaTimeSpan { startMs: number; endMs: number; } export interface DetectScenesResult { shots: Array; scenes: MediaTimeSpan[]; } export interface TranscriptWord { startMs: number; endMs: number; text: string; confidence: number; } export interface TranscribeResult { language: string | null; words: TranscriptWord[]; } export interface FaceObservation { box: NormalizedBox; confidence: number; /** Five-point landmarks: eyes, nose, mouth corners. */ landmarks?: NormalizedPoint[]; } export interface DetectFacesResult { frames: Array<{ tMs: number; faces: FaceObservation[] }>; } export interface FaceTrackResult { trackId: string; keyframes: Array; /** Identity embedding for local person clustering. Never leaves the host. */ embedding: number[] | null; } export interface TrackFacesResult { sampleFps: number; tracks: FaceTrackResult[]; } export interface FaceLandmarksResult { frames: Array<{ tMs: number; faces: Array<{ box: NormalizedBox; landmarks: NormalizedPoint[] }>; }>; } export interface EmbedFramesResult { model: string; dim: number; vectors: number[][]; } export interface ClassifyContentResult { model: string; frames: Array<{ tMs: number; scores: Partial> }>; } export interface SegmentPromptedResult { /** Directory of per-frame PNG masks named by timestamp in ms. */ maskDir: string; keyframes: Array; score: number; } export interface InpaintQuality { /** 1 = no flicker inside the mask region between neighbouring frames. */ temporalStability: number; /** 1 = no visible seams at the mask edge. */ edgeQuality: number; score: number; } export interface InpaintVideoResult { outPath: string; quality: InpaintQuality; } export interface AudioFeaturesResult { integratedLufs: number | null; truePeakDb: number | null; tempoBpm: number | null; beatsMs: number[]; energy: Array<{ tMs: number; value: number }>; silences: MediaTimeSpan[]; } export interface SaliencyResult { points: Array<{ tMs: number; cx: number; cy: number; strength: number }>; } export interface RedactionInstruction { keyframes: BoxKeyframe[]; mode: Exclude | "cover"; paddingPct: number; featherPct: number; /** Solid and cover fill. */ color?: string; /** Landmarks per keyframe for partial modes. */ landmarks?: Array<{ tMs: number; points: NormalizedPoint[] }>; /** Mask directory for region marks instead of boxes. */ maskDir?: string; } interface MediaPathRequest { path: string; signal?: AbortSignal; } /** * Local-first ML operations. Callers check `capabilities()` and fall back * (blur, cover, reframe) when an operation reports unavailable; providers * throw UnsupportedMediaOperationError instead of guessing. */ export interface MediaInferenceProvider { readonly id: string; capabilities(signal?: AbortSignal): Promise; detectScenes(request: MediaPathRequest): Promise; transcribe(request: MediaPathRequest & { language?: string }): Promise; detectFaces( request: MediaPathRequest & { sampleFps: number; timesMs?: number[] }, ): Promise; trackFaces(request: MediaPathRequest & { sampleFps: number }): Promise; faceLandmarks(request: MediaPathRequest & { timesMs: number[] }): Promise; embedFrames(request: MediaPathRequest & { timesMs: number[] }): Promise; classifyContent( request: MediaPathRequest & { sampleFps: number; extraTimesMs?: number[] }, ): Promise; segmentPrompted( request: MediaPathRequest & { prompt: MediaMarkPrompt; propagate: boolean; outDir: string }, ): Promise; inpaintVideo( request: MediaPathRequest & { maskDir: string; outPath: string }, ): Promise; /** * Apply face and region redactions frame-accurately and write a near-lossless * intermediate. Fails closed: a frame it cannot process is an error, not a pass. */ redact( request: MediaPathRequest & { redactions: RedactionInstruction[]; outPath: string }, ): Promise<{ outPath: string; framesProcessed: number }>; /** * Replace a tracked face with a generated, non-existent identity derived from * `seed`. Deliberately takes no target-face image. */ synthesizeIdentity( request: MediaPathRequest & { track: BoxKeyframe[]; seed: number; outPath: string }, ): Promise<{ outPath: string }>; audioFeatures(request: MediaPathRequest): Promise; saliency(request: MediaPathRequest & { timesMs: number[] }): Promise; } // --------------------------------------------------------------------------- // Rendering // --------------------------------------------------------------------------- export interface MediaRenderProgress { stepId: string; /** Overall progress 0..1. */ progress: number; } export interface MediaRenderer { readonly id: string; /** Hardware encoders usable on this host, e.g. h264_vaapi, h264_videotoolbox. */ hardwareEncoders(): Promise; /** * Run every ffmpeg step of the plan. Motion, inpaint, and synthetic identity * steps must already be satisfied (their outPath exists) or the renderer throws. */ render( plan: RenderPlan, options: { signal?: AbortSignal; onProgress?: (progress: MediaRenderProgress) => void }, ): Promise; probe( path: string, options?: { measureLoudness?: boolean; signal?: AbortSignal }, ): Promise; /** Extract a still frame as PNG (EXIF-free). */ extractFrame(input: { path: string; tMs: number; outPath: string; width?: number; signal?: AbortSignal; }): Promise; } export interface MotionRenderRequest { template: MotionTemplateRef; brandKit: BrandKit; /** Resolved font and logo files from the brand kit artifacts. */ assets: { fonts: Array<{ family: string; path: string }>; logoPaths: string[] }; width: number; height: number; fps: number; outPath: string; signal?: AbortSignal; } export interface MotionGraphicsRenderer { readonly id: string; templates(): MotionTemplateInfo[]; /** Render an overlay clip with alpha (MOV, qtrle or prores 4444). */ render(request: MotionRenderRequest): Promise<{ path: string; hasAlpha: boolean }>; } // --------------------------------------------------------------------------- // Publishing // --------------------------------------------------------------------------- export interface MediaPublishRequest { bundleDir: string; profile: PlatformProfile; publicationId: string; title: string | null; caption: string | null; tags: string[]; signal?: AbortSignal; } export interface MediaPublishResult { status: "published" | "bundled"; externalUrl: string | null; } export interface MediaPublisher { readonly id: string; publish(request: MediaPublishRequest): Promise; } export interface LinkShortener { readonly id: string; shorten(url: string, signal?: AbortSignal): Promise; }