From 475710e27e6889c14dc3371ffa8a8468a22662fd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:10:02 +0000 Subject: [PATCH] feat(media): shared service deps and consent-gated model client Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- packages/adapters/src/index.ts | 1 + packages/adapters/src/media-context.test.ts | 82 +++++++++ packages/adapters/src/media-context.ts | 186 ++++++++++++++++++++ 3 files changed, 269 insertions(+) create mode 100644 packages/adapters/src/media-context.test.ts create mode 100644 packages/adapters/src/media-context.ts diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 3848b49..f6a9368 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -76,6 +76,7 @@ export * from "./mcp-emulator.js"; export * from "./mcp-oauth.js"; export * from "./mcp-server-tool.js"; export * from "./mcp-transport.js"; +export * from "./media-context.js"; export * from "./media-emulator.js"; export * from "./media-emulator-clips.js"; export * from "./media-emulator-source.js"; diff --git a/packages/adapters/src/media-context.test.ts b/packages/adapters/src/media-context.test.ts new file mode 100644 index 0000000..37eec33 --- /dev/null +++ b/packages/adapters/src/media-context.test.ts @@ -0,0 +1,82 @@ +import type { AgentRuntime, AgentRuntimeEvent } from "@rakazo/adapter-kit"; +import type { PrismaClient } from "@rakazo/db"; +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; +import { createMediaModelClient, extractJson } from "./media-context.js"; + +function runtime(reply: string, compaction = true): AgentRuntime { + return { + describe: () => ({ capabilities: { compaction } }), + run: vi.fn(async function* (): AsyncGenerator { + yield { type: "done", text: reply }; + }), + } as unknown as AgentRuntime; +} + +function prisma(consented: boolean): PrismaClient { + return { + aiDataConsent: { findUnique: vi.fn(async () => (consented ? { userId: "u" } : null)) }, + } as unknown as PrismaClient; +} + +const model = async () => ({ provider: "openai", id: "gpt-test" }); +const input = { + spaceId: "s", + userId: "u", + instructions: "Reply with JSON.", + prompt: "Label", + schema: z.object({ title: z.string() }), +}; + +describe("extractJson", () => { + it("reads fenced, wrapped, and bare JSON", () => { + expect(extractJson('```json\n{"a":1}\n```')).toEqual({ a: 1 }); + expect(extractJson('Thinking... {"a":[1,2]} done')).toEqual({ a: [1, 2] }); + expect(extractJson("[1]")).toEqual([1]); + expect(extractJson("no json")).toBeUndefined(); + expect(extractJson("{broken")).toBeUndefined(); + }); +}); + +describe("createMediaModelClient", () => { + it("returns parsed output only with consent for the resolved recipient", async () => { + const withConsent = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"title":"Beach"}'), + resolveModel: model, + }); + expect(await withConsent.generate(input)).toEqual({ title: "Beach" }); + + const run = runtime('{"title":"Beach"}'); + const withoutConsent = createMediaModelClient({ + prisma: prisma(false), + runtime: run, + resolveModel: model, + }); + expect(await withoutConsent.generate(input)).toBeNull(); + expect(run.run).not.toHaveBeenCalled(); + }); + + it("falls back to null for scripted runtimes, missing models, and invalid replies", async () => { + const scripted = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"title":"x"}', false), + resolveModel: model, + }); + expect(await scripted.generate(input)).toBeNull(); + const noModel = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"title":"x"}'), + resolveModel: async () => { + throw new Error("no model"); + }, + }); + expect(await noModel.generate(input)).toBeNull(); + const invalid = createMediaModelClient({ + prisma: prisma(true), + runtime: runtime('{"name":"x"}'), + resolveModel: model, + }); + expect(await invalid.generate(input)).toBeNull(); + }); +}); diff --git a/packages/adapters/src/media-context.ts b/packages/adapters/src/media-context.ts new file mode 100644 index 0000000..78f39cd --- /dev/null +++ b/packages/adapters/src/media-context.ts @@ -0,0 +1,186 @@ +import type { + AgentInputImage, + AgentRunModel, + AgentRuntime, + ArtifactStore, + JobPublisher, + LinkShortener, + MediaInferenceProvider, + MediaPublisher, + MediaRenderer, + MediaSourceProvider, + MotionGraphicsRenderer, +} from "@rakazo/adapter-kit"; +import type { ModelTier } from "@rakazo/contracts"; +import type { MediaRepos, PrismaClient } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import type { ProfileSet } from "@rakazo/media"; +import type { z } from "zod"; +import { aiRecipient } from "./ai-consent.js"; + +/* + * Shared wiring for Media Studio services. Composition roots (API, worker) + * build one MediaServiceDeps; every media service, job handler, and agent tool + * receives it instead of reaching for globals. + */ + +/** Realtime notifications the web and mobile clients subscribe to per space. */ +export type MediaRealtimeEvent = + | { type: "media.source"; sourceId: string; status: string; error?: string | null } + | { type: "media.asset"; assetId: string; ingestState: string } + | { type: "media.collections"; kind: "session" | "theme" } + | { type: "media.people" } + | { type: "media.render"; renderId: string; projectId: string; status: string; progress: number } + | { type: "media.publication"; publicationId: string; status: string }; + +/** + * Structured model calls on behalf of a user in a space. Returns null instead of + * calling a model when no model is configured, the runtime is scripted, or the + * user has not consented to sharing data with the resolved model recipient. + * Callers must then fall back to deterministic behavior. + */ +export interface MediaModelClient { + generate(input: { + spaceId: string; + userId: string; + /** Short system instructions; the reply must be one JSON document. */ + instructions: string; + prompt: string; + images?: AgentInputImage[]; + schema: z.ZodType; + tier?: ModelTier; + signal?: AbortSignal; + }): Promise; +} + +export interface MediaServiceDeps { + prisma: PrismaClient; + repos: MediaRepos; + jobs: JobPublisher; + /** Null when no sidecar is configured; media then stays hidden in the UI. */ + inference: MediaInferenceProvider | null; + renderer: MediaRenderer; + motion: MotionGraphicsRenderer; + publisher: MediaPublisher; + shortener: LinkShortener | null; + sources: Partial>; + /** Absolute DATA_DIR/media. Every stored media path is relative to it. */ + mediaRoot: string; + /** Builtin plus private platform profiles, cached by the composition root. */ + profiles: () => Promise; + artifacts: ArtifactStore; + model: MediaModelClient; + publish: (spaceId: string, event: MediaRealtimeEvent) => Promise; + now: () => Date; +} + +/** Realtime topic for a space's media events. */ +export function mediaTopic(spaceId: string): string { + return `media:${spaceId}`; +} + +const MODEL_TIMEOUT_MS = 120_000; +const RUNTIME_FAILURE = /^(?:I hit a problem:|Unknown model )/i; + +/** Extract the first JSON object or array from a model reply (models may wrap or think aloud). */ +export function extractJson(text: string): unknown { + const trimmed = text.trim(); + const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(trimmed); + const body = fenced?.[1] ?? trimmed; + const start = body.search(/[[{]/); + if (start < 0) return undefined; + const open = body[start]; + const close = open === "{" ? "}" : "]"; + const end = body.lastIndexOf(close); + if (end <= start) return undefined; + try { + return JSON.parse(body.slice(start, end + 1)); + } catch { + return undefined; + } +} + +export function createMediaModelClient(deps: { + prisma: PrismaClient; + runtime: AgentRuntime; + resolveModel: (scope: { + userId: string; + spaceId: string; + tier?: ModelTier; + }) => Promise; +}): MediaModelClient { + return { + async generate(input) { + if (!deps.runtime.describe().capabilities.compaction) return null; + let model: AgentRunModel; + try { + model = await deps.resolveModel({ + userId: input.userId, + spaceId: input.spaceId, + tier: input.tier ?? "fast", + }); + } catch { + return null; + } + if (model.provider === "scripted") return null; + const recipient = aiRecipient({ + provider: model.provider, + use: "model", + modelId: model.id, + baseUrl: model.baseUrl, + }); + if (!recipient) return null; + const consent = await deps.prisma.aiDataConsent.findUnique({ + where: { + userId_spaceId_recipientKey: { + userId: input.userId, + spaceId: input.spaceId, + recipientKey: recipient.key, + }, + }, + }); + if (!consent) return null; + const images = + input.images && model.acceptsImages !== false + ? input.images.slice(0, model.maxImagesPerPrompt ?? input.images.length) + : undefined; + const runId = `media-${crypto.randomUUID()}`; + const signal = input.signal + ? AbortSignal.any([input.signal, AbortSignal.timeout(MODEL_TIMEOUT_MS)]) + : AbortSignal.timeout(MODEL_TIMEOUT_MS); + let reply = ""; + try { + for await (const event of deps.runtime.run( + { + botId: "media-studio", + threadId: runId, + runId, + prompt: input.prompt, + instructions: input.instructions, + history: [], + tools: [], + model, + ...(images && images.length > 0 ? { currentTurnImages: images } : {}), + }, + { + operationId: runId, + traceId: runId, + spaceId: input.spaceId, + userId: input.userId, + signal, + }, + )) { + if (event.type === "done" && event.text) reply = event.text.trim(); + } + } catch (error) { + getLogger().warn("media model call failed", { + reason: error instanceof Error ? error.message : String(error), + }); + return null; + } + if (!reply || RUNTIME_FAILURE.test(reply)) return null; + const parsed = input.schema.safeParse(extractJson(reply)); + return parsed.success ? parsed.data : null; + }, + }; +}