import type { AgentInputImage, AgentRunModel, AgentRuntime, ArtifactStore, JobPublisher, LinkShortener, MediaInferenceProvider, MediaPublisher, MediaRenderer, MediaSourceProvider, MotionGraphicsRenderer, } from "@rakazo/adapter-kit"; import type { AiAlteration, ClientMediaItem, CreateMarkInput, Edl, EdlError, FfprobeReport, GateReport, IntroVariant, MarketingCopy, MarkTreatment, MediaEditPlan, MediaMark, MediaPublication, MediaRender, MediaRenderOutput, MediaRenderQuality, ModelTier, PlatformProfile, Utm, } from "@rakazo/contracts"; import type { MediaRepos, PrismaClient } from "@rakazo/db"; import { getLogger } from "@rakazo/logging"; import type { CompileSource, ProfileSet } from "@rakazo/media"; import type { z } from "zod"; import { aiRecipient } from "./ai-consent.js"; /* * Shared wiring for Media Studio services. Composition roots (API, worker) * build one MediaServiceDeps; every media service, job handler, and agent tool * receives it instead of reaching for globals. */ /** Realtime notifications the web and mobile clients subscribe to per space. */ export type MediaRealtimeEvent = | { type: "media.source"; sourceId: string; status: string; error?: string | null } | { type: "media.asset"; assetId: string; ingestState: string } | { type: "media.collections"; kind: "session" | "theme" } | { type: "media.people" } | { type: "media.render"; renderId: string; projectId: string; status: string; progress: number } | { type: "media.publication"; publicationId: string; status: string }; /** * Structured model calls on behalf of a user in a space. Returns null instead of * calling a model when no model is configured, the runtime is scripted, or the * user has not consented to sharing data with the resolved model recipient. * Callers must then fall back to deterministic behavior. */ export interface MediaModelClient { generate(input: { spaceId: string; userId: string; /** Short system instructions; the reply must be one JSON document. */ instructions: string; prompt: string; images?: AgentInputImage[]; schema: z.ZodType; tier?: ModelTier; signal?: AbortSignal; }): Promise; } export interface MediaServiceDeps { prisma: PrismaClient; repos: MediaRepos; jobs: JobPublisher; /** Null when no sidecar is configured; media then stays hidden in the UI. */ inference: MediaInferenceProvider | null; renderer: MediaRenderer; motion: MotionGraphicsRenderer; publisher: MediaPublisher; shortener: LinkShortener | null; sources: Partial>; /** Absolute DATA_DIR/media. Every stored media path is relative to it. */ mediaRoot: string; /** Builtin plus private platform profiles, cached by the composition root. */ profiles: () => Promise; artifacts: ArtifactStore; model: MediaModelClient; publish: (spaceId: string, event: MediaRealtimeEvent) => Promise; now: () => Date; } /** Realtime topic for a space's media events. */ export function mediaTopic(spaceId: string): string { return `media:${spaceId}`; } const MODEL_TIMEOUT_MS = 120_000; const RUNTIME_FAILURE = /^(?:I hit a problem:|Unknown model )/i; /** Extract the first JSON object or array from a model reply (models may wrap or think aloud). */ export function extractJson(text: string): unknown { const trimmed = text.trim(); const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(trimmed); const body = fenced?.[1] ?? trimmed; const start = body.search(/[[{]/); if (start < 0) return undefined; const open = body[start]; const close = open === "{" ? "}" : "]"; const end = body.lastIndexOf(close); if (end <= start) return undefined; try { return JSON.parse(body.slice(start, end + 1)); } catch { return undefined; } } export function createMediaModelClient(deps: { prisma: PrismaClient; runtime: AgentRuntime; resolveModel: (scope: { userId: string; spaceId: string; tier?: ModelTier; }) => Promise; }): MediaModelClient { return { async generate(input) { if (!deps.runtime.describe().capabilities.compaction) return null; let model: AgentRunModel; try { model = await deps.resolveModel({ userId: input.userId, spaceId: input.spaceId, tier: input.tier ?? "fast", }); } catch { return null; } if (model.provider === "scripted") return null; const recipient = aiRecipient({ provider: model.provider, use: "model", modelId: model.id, baseUrl: model.baseUrl, }); if (!recipient) return null; const consent = await deps.prisma.aiDataConsent.findUnique({ where: { userId_spaceId_recipientKey: { userId: input.userId, spaceId: input.spaceId, recipientKey: recipient.key, }, }, }); if (!consent) return null; const images = input.images && model.acceptsImages !== false ? input.images.slice(0, model.maxImagesPerPrompt ?? input.images.length) : undefined; const runId = `media-${crypto.randomUUID()}`; const signal = input.signal ? AbortSignal.any([input.signal, AbortSignal.timeout(MODEL_TIMEOUT_MS)]) : AbortSignal.timeout(MODEL_TIMEOUT_MS); let reply = ""; try { for await (const event of deps.runtime.run( { botId: "media-studio", threadId: runId, runId, prompt: input.prompt, instructions: input.instructions, history: [], tools: [], model, ...(images && images.length > 0 ? { currentTurnImages: images } : {}), }, { operationId: runId, traceId: runId, spaceId: input.spaceId, userId: input.userId, signal, }, )) { if (event.type === "done" && event.text) reply = event.text.trim(); } } catch (error) { getLogger().warn("media model call failed", { reason: error instanceof Error ? error.message : String(error), }); return null; } if (!reply || RUNTIME_FAILURE.test(reply)) return null; const parsed = input.schema.safeParse(extractJson(reply)); return parsed.success ? parsed.data : null; }, }; } // --------------------------------------------------------------------------- // Service contracts between media slices. Each slice implements its part; the // API, worker, and agent tools only depend on these interfaces. // --------------------------------------------------------------------------- export type CreateMarkInputType = z.infer; export interface MediaActor { spaceId: string; userId: string; } export interface MediaIngestService { /** Client-pushed metadata (Apple Photos, iOS): upsert, apply deletions, enqueue work. */ pushItems( actor: MediaActor, input: { sourceId: string; items: ClientMediaItem[]; deletedExternalIds: string[]; syncToken: string | null; complete: boolean; }, ): Promise<{ created: number; updated: number; unchanged: number; deleted: number }>; /** Enqueue an incremental sync of a server-side source. */ requestSync(actor: MediaActor, sourceId: string): Promise; /** An upload finished: move the file into place and enqueue proxy and analysis. */ uploadCompleted( actor: MediaActor, input: { assetId: string; tempPath: string; sha256: string; fileName: string }, ): Promise; recluster(actor: MediaActor): Promise; /** Remove a source with its assets and every derived file. */ removeSource(actor: MediaActor, sourceId: string): Promise; } export interface MediaPreparedSources { sources: Map; /** Alterations applied while preparing (inpaint, synthetic identity, voice). */ aiAlterations: AiAlteration[]; /** Redactions that could not be honored and fell back, for the plan rationale. */ fallbacks: string[]; } export interface MediaPrivacyService { /** * Resolve every clip's source file and apply all face, mark, and region * redactions from the EDL plus person and mark defaults. Fails closed: a * redaction that cannot be applied throws instead of returning an unredacted source. */ prepareSources(input: { actor: MediaActor; projectId: string; edl: Edl; quality: MediaRenderQuality; workDir: string; signal?: AbortSignal; }): Promise; /** Re-detect on the rendered output and evaluate the full gate. */ gate(input: { actor: MediaActor; projectId: string; edl: Edl; profile: PlatformProfile; outputPath: string; ffprobe: FfprobeReport; copy: { title: string | null; caption: string | null; tags: string[] }; bundleHasDisclosure: boolean; autoFixRound: number; signal?: AbortSignal; }): Promise; /** Create a mark: segment from the prompt, propagate, score, and suggest a fallback. */ createMark(actor: MediaActor, input: CreateMarkInputType): Promise; updateMark( actor: MediaActor, input: { markId: string; treatment?: MarkTreatment; label?: string | null; personId?: string | null; }, ): Promise; /** Delete a person, their biometric data, and their files. */ removePerson(actor: MediaActor, personId: string): Promise; } export interface MediaRenderService { start( actor: MediaActor, input: { planId: string; quality: MediaRenderQuality }, ): Promise; cancel(actor: MediaActor, renderId: string): Promise; /** Absolute path of a finished output for streaming, or null. */ outputPath(actor: MediaActor, renderId: string, fileName: string): Promise; } export interface MediaMarketingService { autoPlan( actor: MediaActor, input: { projectId: string; profileIds?: string[] }, ): Promise; submitPlan( actor: MediaActor, input: { projectId: string; edl: unknown; rationale: string | null; createdBy: "agent" | "user"; }, ): Promise<{ ok: true; plan: MediaEditPlan } | { ok: false; errors: EdlError[] }>; introVariants( actor: MediaActor, input: { brandKitId: string; count: number }, ): Promise; thumbnails( actor: MediaActor, input: { renderId: string; count: number }, ): Promise; copy(actor: MediaActor, input: { renderId: string; language: string }): Promise; createPublication( actor: MediaActor, input: { renderId: string; destination: string; targetUrl: string | null; utm: Utm | null; title: string | null; caption: string | null; tags: string[]; thumbnailVariant: string | null; scheduledAt: string | null; }, ): Promise; bundle(actor: MediaActor, publicationId: string): Promise; cancelPublication(actor: MediaActor, publicationId: string): Promise; importMetricsCsv( actor: MediaActor, input: { mappingId: string; csv: string }, ): Promise<{ imported: number; skipped: Array<{ row: number; reason: string }> }>; } export interface MediaServices { deps: MediaServiceDeps; ingest: MediaIngestService; privacy: MediaPrivacyService; render: MediaRenderService; marketing: MediaMarketingService; }