evOo-local/packages/adapter-kit/src/media.ts

import type {
  AnonymizeMode,
  BoxKeyframe,
  BrandKit,
  ContentCategory,
  FfprobeReport,
  MediaAssetKind,
  MediaCapabilities,
  MediaInferenceOperation,
  MediaMarkPrompt,
  MediaSourceKind,
  MotionTemplateInfo,
  MotionTemplateRef,
  NormalizedBox,
  NormalizedPoint,
  PlatformProfile,
  RenderPlan,
} from "@rakazo/contracts";

/*
 * Media Studio provider boundaries. Every implementation (local sidecar,
 * emulator, optional cloud adapters) satisfies the same contracts, which
 * media-conformance tests exercise offline.
 *
 * Paths are absolute host paths inside the configured media root. Providers
 * must reject any path outside that root.
 */

// ---------------------------------------------------------------------------
// Sources
// ---------------------------------------------------------------------------

export interface MediaSourceItem {
  externalId: string;
  kind: MediaAssetKind;
  capturedAt: Date | null;
  durationMs: number | null;
  width: number | null;
  height: number | null;
  fps: number | null;
  gpsLat: number | null;
  gpsLon: number | null;
  deviceModel: string | null;
  albumNames: string[];
  isFavorite: boolean;
  fileName: string | null;
  sizeBytes: number | null;
  /** Cheap change detector (mtime + size or provider revision); not a content hash. */
  revision: string;
}

export interface MediaSourceListRequest {
  config: Record<string, unknown>;
  cursor: string | null;
  limit: number;
  signal?: AbortSignal;
}

export interface MediaSourceListResult {
  items: MediaSourceItem[];
  nextCursor: string | null;
}

/** Server-side sources the worker can enumerate and read directly. */
export interface MediaSourceProvider {
  readonly kind: Extract<MediaSourceKind, "folder" | "emulator">;
  list(request: MediaSourceListRequest): Promise<MediaSourceListResult>;
  /** Absolute readable path of the original file. Throws when it escaped the source root. */
  resolve(config: Record<string, unknown>, externalId: string): Promise<string>;
}

// ---------------------------------------------------------------------------
// Inference
// ---------------------------------------------------------------------------

export class UnsupportedMediaOperationError extends Error {
  constructor(
    readonly operation: MediaInferenceOperation,
    readonly reason: string,
  ) {
    super(`${operation} is not available: ${reason}`);
    this.name = "UnsupportedMediaOperationError";
  }
}

export interface MediaTimeSpan {
  startMs: number;
  endMs: number;
}

export interface DetectScenesResult {
  shots: Array<MediaTimeSpan & { score: number }>;
  scenes: MediaTimeSpan[];
}

export interface TranscriptWord {
  startMs: number;
  endMs: number;
  text: string;
  confidence: number;
}

export interface TranscribeResult {
  language: string | null;
  words: TranscriptWord[];
}

export interface FaceObservation {
  box: NormalizedBox;
  confidence: number;
  /** Five-point landmarks: eyes, nose, mouth corners. */
  landmarks?: NormalizedPoint[];
}

export interface DetectFacesResult {
  frames: Array<{ tMs: number; faces: FaceObservation[] }>;
}

export interface FaceTrackResult {
  trackId: string;
  keyframes: Array<BoxKeyframe & { confidence: number }>;
  /** Identity embedding for local person clustering. Never leaves the host. */
  embedding: number[] | null;
}

export interface TrackFacesResult {
  sampleFps: number;
  tracks: FaceTrackResult[];
}

export interface FaceLandmarksResult {
  frames: Array<{
    tMs: number;
    faces: Array<{ box: NormalizedBox; landmarks: NormalizedPoint[] }>;
  }>;
}

export interface EmbedFramesResult {
  model: string;
  dim: number;
  vectors: number[][];
}

export interface ClassifyContentResult {
  model: string;
  frames: Array<{ tMs: number; scores: Partial<Record<ContentCategory, number>> }>;
}

export interface SegmentPromptedResult {
  /** Directory of per-frame PNG masks named by timestamp in ms. */
  maskDir: string;
  keyframes: Array<BoxKeyframe & { confidence: number }>;
  score: number;
}

export interface InpaintQuality {
  /** 1 = no flicker inside the mask region between neighbouring frames. */
  temporalStability: number;
  /** 1 = no visible seams at the mask edge. */
  edgeQuality: number;
  score: number;
}

export interface InpaintVideoResult {
  outPath: string;
  quality: InpaintQuality;
}

export interface AudioFeaturesResult {
  integratedLufs: number | null;
  truePeakDb: number | null;
  tempoBpm: number | null;
  beatsMs: number[];
  energy: Array<{ tMs: number; value: number }>;
  silences: MediaTimeSpan[];
}

export interface SaliencyResult {
  points: Array<{ tMs: number; cx: number; cy: number; strength: number }>;
}

export interface RedactionInstruction {
  keyframes: BoxKeyframe[];
  mode: Exclude<AnonymizeMode, "none" | "reframe_out" | "synthetic_identity"> | "cover";
  paddingPct: number;
  featherPct: number;
  /** Solid and cover fill. */
  color?: string;
  /** Landmarks per keyframe for partial modes. */
  landmarks?: Array<{ tMs: number; points: NormalizedPoint[] }>;
  /** Mask directory for region marks instead of boxes. */
  maskDir?: string;
}

interface MediaPathRequest {
  path: string;
  signal?: AbortSignal;
}

/**
 * Local-first ML operations. Callers check `capabilities()` and fall back
 * (blur, cover, reframe) when an operation reports unavailable; providers
 * throw UnsupportedMediaOperationError instead of guessing.
 */
export interface MediaInferenceProvider {
  readonly id: string;
  capabilities(signal?: AbortSignal): Promise<MediaCapabilities>;
  detectScenes(request: MediaPathRequest): Promise<DetectScenesResult>;
  transcribe(request: MediaPathRequest & { language?: string }): Promise<TranscribeResult>;
  detectFaces(
    request: MediaPathRequest & { sampleFps: number; timesMs?: number[] },
  ): Promise<DetectFacesResult>;
  trackFaces(request: MediaPathRequest & { sampleFps: number }): Promise<TrackFacesResult>;
  faceLandmarks(request: MediaPathRequest & { timesMs: number[] }): Promise<FaceLandmarksResult>;
  embedFrames(request: MediaPathRequest & { timesMs: number[] }): Promise<EmbedFramesResult>;
  classifyContent(
    request: MediaPathRequest & { sampleFps: number; extraTimesMs?: number[] },
  ): Promise<ClassifyContentResult>;
  segmentPrompted(
    request: MediaPathRequest & { prompt: MediaMarkPrompt; propagate: boolean; outDir: string },
  ): Promise<SegmentPromptedResult>;
  inpaintVideo(
    request: MediaPathRequest & { maskDir: string; outPath: string },
  ): Promise<InpaintVideoResult>;
  /**
   * Apply face and region redactions frame-accurately and write a near-lossless
   * intermediate. Fails closed: a frame it cannot process is an error, not a pass.
   */
  redact(
    request: MediaPathRequest & { redactions: RedactionInstruction[]; outPath: string },
  ): Promise<{ outPath: string; framesProcessed: number }>;
  /**
   * Replace a tracked face with a generated, non-existent identity derived from
   * `seed`. Deliberately takes no target-face image.
   */
  synthesizeIdentity(
    request: MediaPathRequest & { track: BoxKeyframe[]; seed: number; outPath: string },
  ): Promise<{ outPath: string }>;
  audioFeatures(request: MediaPathRequest): Promise<AudioFeaturesResult>;
  saliency(request: MediaPathRequest & { timesMs: number[] }): Promise<SaliencyResult>;
}

// ---------------------------------------------------------------------------
// Rendering
// ---------------------------------------------------------------------------

export interface MediaRenderProgress {
  stepId: string;
  /** Overall progress 0..1. */
  progress: number;
}

export interface MediaRenderer {
  readonly id: string;
  /** Hardware encoders usable on this host, e.g. h264_vaapi, h264_videotoolbox. */
  hardwareEncoders(): Promise<string[]>;
  /**
   * Run every ffmpeg step of the plan. Motion, inpaint, and synthetic identity
   * steps must already be satisfied (their outPath exists) or the renderer throws.
   */
  render(
    plan: RenderPlan,
    options: { signal?: AbortSignal; onProgress?: (progress: MediaRenderProgress) => void },
  ): Promise<void>;
  probe(
    path: string,
    options?: { measureLoudness?: boolean; signal?: AbortSignal },
  ): Promise<FfprobeReport>;
  /** Extract a still frame as PNG (EXIF-free). */
  extractFrame(input: {
    path: string;
    tMs: number;
    outPath: string;
    width?: number;
    signal?: AbortSignal;
  }): Promise<void>;
}

export interface MotionRenderRequest {
  template: MotionTemplateRef;
  brandKit: BrandKit;
  /** Resolved font and logo files from the brand kit artifacts. */
  assets: { fonts: Array<{ family: string; path: string }>; logoPaths: string[] };
  width: number;
  height: number;
  fps: number;
  outPath: string;
  signal?: AbortSignal;
}

export interface MotionGraphicsRenderer {
  readonly id: string;
  templates(): MotionTemplateInfo[];
  /** Render an overlay clip with alpha (MOV, qtrle or prores 4444). */
  render(request: MotionRenderRequest): Promise<{ path: string; hasAlpha: boolean }>;
}

// ---------------------------------------------------------------------------
// Publishing
// ---------------------------------------------------------------------------

export interface MediaPublishRequest {
  bundleDir: string;
  profile: PlatformProfile;
  publicationId: string;
  title: string | null;
  caption: string | null;
  tags: string[];
  signal?: AbortSignal;
}

export interface MediaPublishResult {
  status: "published" | "bundled";
  externalUrl: string | null;
}

export interface MediaPublisher {
  readonly id: string;
  publish(request: MediaPublishRequest): Promise<MediaPublishResult>;
}

export interface LinkShortener {
  readonly id: string;
  shorten(url: string, signal?: AbortSignal): Promise<string>;
}