diff --git a/packages/adapters/src/media-skills.ts b/packages/adapters/src/media-skills.ts new file mode 100644 index 0000000..a554485 --- /dev/null +++ b/packages/adapters/src/media-skills.ts @@ -0,0 +1,75 @@ +/* + * Built-in Media Studio skills (spec 10.2 and 11). builtin-skills.ts appends + * them to its catalog. They are recipes for the media_* tools, which only exist + * when Media Studio is configured; filterMediaSkills hides them otherwise. + */ + +/** Hard limits from the Media Studio spec, section 0, verbatim. */ +export const MEDIA_HARD_LIMITS = `Hard limits (Media Studio spec, section 0, verbatim): + +- **Fail closed.** Wenn Anonymisierung, Compliance-Gate oder Einwilligungsprüfung keinen sicheren Zustand nachweisen, wird nicht exportiert. Ein Fehler wird nie verschluckt, und das System täuscht nie Erfolg vor. +- Funktionen, die Moderation oder Klassifikatoren einer Plattform umgehen. Beispiele: adversariale Overlays, eingestreute Einzelframes, Farb- oder Rauschtricks gegen Filter, Umgehen von Altersschranken. „Compliance“ heißt hier ausschließlich, dass Inhalte die Regeln tatsächlich einhalten. +- Gesichtstausch mit dem Gesicht einer realen dritten Person, also Deepfakes. Zulässig ist nur eine **synthetische, nicht existierende** Identität. Die API nimmt keinerlei Zielgesichts-Bild entgegen. +- Export für Bezahl-Plattformen ohne dokumentierte Einwilligung und Altersnachweis aller erkannten Mitwirkenden. +- Hochladen biometrischer Daten (Gesichts-Embeddings, Personen-Cluster) an externe Dienste. Diese Daten fallen unter DSGVO Art. 9 und bleiben lokal und löschbar. + +The second to fifth items are forbidden: never do them, never help work around them, and refuse requests that need them. Consent and age proof are recorded only by the user in the People view; you can read them but never set them. Model calls on media only happen with the user's AI consent.`; + +const VIDEO_PRODUCER = `Produce platform-ready videos from the user's media library with the media_* tools. Call a tool with {"help": true} the first time you use it. + +1. Clarify the goal and the target platforms. Ask only when the request leaves them truly open; otherwise pick the closest goal and profiles and say which. +2. Find the material: media_collections (sessions, themes) and media_search. Create the project with media_project create. +3. Check people and mark rules: media_people list for consent and anonymization, media_marks list for identifying features. Anyone without consent stays anonymized. For paid-video profiles, every detected participant needs consent and age proof; if someone lacks it, tell the user to record it in the People view and do not export. +4. Build one EDL per profile: media_project auto_plan for a first cut, then refine with media_plan. Fix every returned validation error at its path. Never use paid_only segments in teasers; on short vertical profiles the hook comes before the intro. +5. Render a draft: media_render start with quality draft, then media_render status until it leaves queued, rendering, and gating. +6. Gate: media_gate_report on the draft. +7. Fix blockers only with compliant changes (trim or cut the flagged range, add redactions, shorten text, add disclosure) and render the draft again. Stop after three rounds and report what still blocks. +8. Final render: media_render start with quality final once the draft passes. +9. Thumbnails and copy: media_thumbnails, then media_copy per profile (follow the Video Copy skill). +10. Bundle and schedule: media_publish create (with scheduledAt to plan it) and media_publish bundle. Uploads to connected accounts go through that connector's tool and the user's approval; paid-video platforms only get the bundle. + +Report renders, gate results, and bundles by id and status. Never claim a step succeeded unless its tool result says so. + +${MEDIA_HARD_LIMITS}`; + +const VIDEO_COPY = `Write titles, hooks, captions, hashtags, descriptions, chapters, and alt text for a finished render with media_copy, then check and revise them. + +1. Generate with media_copy {"renderId","language"}; use the language the audience reads. +2. Read every returned issue (banned patterns, lengths, link policy) and rewrite until none remain. +3. Rules: + - No clickbait that breaks the platform's rules, no misleading thumbnails or titles, no false promises about content, prices, or results. + - Describe only what is in the video. No claims about people's identity, age, or consent. + - Follow the profile's link policy: no links where it forbids them, links only in the bio where it says bio_only. + - Keep the AI disclosure, age restriction, and paid partnership labels the profile requires. + - Never use spelling tricks, symbols, or coded words to slip past filters or age gates. +4. Pass the final title, caption, and tags to media_publish create. + +${MEDIA_HARD_LIMITS}`; + +export const MEDIA_AGENT_SKILLS: Array<{ name: string; description: string; body: string }> = [ + { + name: "Video Producer", + description: "Find, cut, gate, render, and bundle videos per platform.", + body: VIDEO_PRODUCER, + }, + { + name: "Video Copy", + description: "Platform-safe titles, captions, and hashtags for a render.", + body: VIDEO_COPY, + }, +]; + +const MEDIA_SKILL_NAMES: ReadonlySet = new Set( + MEDIA_AGENT_SKILLS.map((skill) => skill.name), +); + +/** Built-in media skills only appear where Media Studio is configured; user skills stay. */ +export function filterMediaSkills( + skills: T[], + configured: boolean, +): T[] { + if (configured) return skills; + return skills.filter( + (skill) => !(MEDIA_SKILL_NAMES.has(skill.name) && (skill.source ?? "builtin") === "builtin"), + ); +} diff --git a/packages/adapters/src/media-tools-execute.ts b/packages/adapters/src/media-tools-execute.ts new file mode 100644 index 0000000..5de94fc --- /dev/null +++ b/packages/adapters/src/media-tools-execute.ts @@ -0,0 +1,755 @@ +import { rm } from "node:fs/promises"; +import { UnsupportedMediaOperationError } from "@rakazo/adapter-kit"; +import type { + BrandKit, + BrandKitRecord, + FunnelRow, + GateIssue, + MediaAsset, + MediaCollection, + MediaEditPlan, + MediaMark, + MediaPerson, + MediaProject, + MediaPublication, + MediaRender, + MediaRenderOutput, + MediaToolName, +} from "@rakazo/contracts"; +import { CreateMarkInput, CreateMediaProjectInput, MediaToolInputSchemas } from "@rakazo/contracts"; +import { InvalidMediaInputError, IsolationError, MediaStateConflictError } from "@rakazo/db"; +import { PathOutsideRootError, resolveInside } from "@rakazo/media/node"; +import type { z } from "zod"; +import { saveBrandKit } from "./media-brand.js"; +import type { MediaActor, MediaServices } from "./media-context.js"; +import { recordMetric } from "./media-metrics.js"; +import { PrivacyUnavailableError } from "./media-privacy-io.js"; +import { isMediaToolName, MEDIA_TOOL_GUIDES } from "./media-tools.js"; + +/* + * Execution of the media_* agent tools (spec 11). Every call is scoped to the + * run's space and user (the actor) and goes through the same services and + * repos as the media RPCs, with the same ownership checks. Results are compact: + * ids and summaries, never file paths, embeddings, or raw analysis. + * + * Two boundaries hold here regardless of what the model asks for: + * - No tool records consent or age proof. media_people can only rename, set an + * anonymization mode, and merge people without a recorded consent. + * - No tool uploads. media_publish creates, schedules, and bundles; uploads to + * connected accounts go through connector tools and their approval. + */ + +export const MEDIA_NOT_CONFIGURED_TOOL_ERROR = "Media Studio is not configured."; +export const BUNDLE_PUBLISHER_ID = "bundle"; + +const DEFAULT_SEARCH_LIMIT = 25; +const LIST_LIMIT = 50; +const MAX_THEME_MATCHES = 5; +const COLLECTION_ASSET_PREVIEW = 100; + +type Args = z.infer<(typeof MediaToolInputSchemas)[N]>; +type ToolResult = Record; + +class ToolInputError extends Error {} + +function need(value: T | undefined | null, field: string): T { + if (value === undefined || value === null) throw new ToolInputError(`${field} is required.`); + return value; +} + +/** Drop null, undefined, and empty arrays so results stay small. */ +function compact>(value: T): Partial { + return Object.fromEntries( + Object.entries(value).filter( + ([, entry]) => + entry !== null && entry !== undefined && !(Array.isArray(entry) && entry.length === 0), + ), + ) as Partial; +} + +// --------------------------------------------------------------------------- +// Compact views +// --------------------------------------------------------------------------- + +function assetView(asset: MediaAsset) { + return compact({ + id: asset.id, + kind: asset.kind, + capturedAt: asset.capturedAt, + durationMs: asset.durationMs, + size: asset.width && asset.height ? `${asset.width}x${asset.height}` : null, + quality: asset.qualityScore === null ? null : Math.round(asset.qualityScore * 100) / 100, + state: asset.ingestState, + favorite: asset.isFavorite || null, + }); +} + +function collectionView(collection: MediaCollection) { + return compact({ + id: collection.id, + kind: collection.kind, + title: collection.title, + startAt: collection.startAt, + endAt: collection.endAt, + assets: collection.assetCount, + videos: collection.videoCount, + durationMs: collection.totalDurationMs, + }); +} + +function personView(person: MediaPerson) { + return { + id: person.id, + name: person.displayName, + self: person.isSelf, + confirmed: person.confirmed, + anonymize: person.anonymizeByDefault, + consent: person.consentRecordedAt !== null, + ageVerified: person.ageVerifiedAt !== null && person.ageVerificationMethod !== "none", + tracks: person.faceTrackCount, + }; +} + +function markView(mark: MediaMark) { + return compact({ + id: mark.id, + kind: mark.kind, + label: mark.label, + personId: mark.personId, + assetId: mark.assetId, + treatment: mark.treatment, + qualityScore: mark.qualityScore, + suggestedFallback: mark.suggestedFallback, + }); +} + +function projectView(project: MediaProject) { + return compact({ + id: project.id, + title: project.title, + goal: project.goal, + status: project.status, + brandKitId: project.brandKitId, + collectionIds: project.collectionIds, + profileIds: project.profileIds, + updatedAt: project.updatedAt, + }); +} + +function planView(plan: MediaEditPlan) { + return compact({ + id: plan.id, + profileId: plan.profileId, + version: plan.version, + createdBy: plan.createdBy, + clips: plan.edl.tracks.video.length, + durationTargetMs: plan.edl.output.durationTargetMs, + rationale: plan.rationale, + }); +} + +function gateSummary(render: MediaRender) { + const report = render.gateReport; + if (!report) return null; + return { + passed: report.passed, + blockers: report.issues.filter((issue) => issue.severity === "block").length, + warnings: report.issues.filter((issue) => issue.severity === "warn").length, + }; +} + +function renderView(render: MediaRender) { + return compact({ + id: render.id, + planId: render.planId, + profileId: render.profileId, + quality: render.quality, + status: render.status, + progress: Math.round(render.progress * 100) / 100, + gate: gateSummary(render), + outputs: render.outputs.map((output) => output.fileName), + error: render.error, + }); +} + +function outputView(output: MediaRenderOutput) { + return compact({ kind: output.kind, fileName: output.fileName, artifactId: output.artifactId }); +} + +function issueView(issue: GateIssue) { + return compact({ ...issue }); +} + +function publicationView(publication: MediaPublication) { + return compact({ + id: publication.id, + renderId: publication.renderId, + profileId: publication.profileId, + destination: publication.destination, + status: publication.status, + scheduledAt: publication.scheduledAt, + publishedAt: publication.publishedAt, + trackingUrl: publication.trackingUrl, + title: publication.title, + thumbnailVariant: publication.thumbnailVariant, + bundleArtifactId: publication.bundleArtifactId, + }); +} + +function brandKitView(record: BrandKitRecord) { + return { id: record.id, kit: record.kit, updatedAt: record.updatedAt }; +} + +function funnelTotals(rows: FunnelRow[]) { + const totals = new Map< + string, + { currency: string; views: number; clicks: number; conversions: number; revenueCents: number } + >(); + for (const row of rows) { + const total = totals.get(row.currency) ?? { + currency: row.currency, + views: 0, + clicks: 0, + conversions: 0, + revenueCents: 0, + }; + total.views += row.views; + total.clicks += row.clicks; + total.conversions += row.conversions; + total.revenueCents += row.revenueCents; + totals.set(row.currency, total); + } + return [...totals.values()]; +} + +// --------------------------------------------------------------------------- +// Handlers +// --------------------------------------------------------------------------- + +interface ToolContext { + services: MediaServices; + actor: MediaActor; + rawArgs: Record; +} + +async function assertKnownProfiles(services: MediaServices, profileIds: string[] | undefined) { + if (!profileIds) return; + const { profiles } = await services.deps.profiles(); + const unknown = profileIds.filter((id) => !profiles.has(id)); + if (unknown.length > 0) { + throw new InvalidMediaInputError(`Unknown profile: ${unknown.join(", ")}`); + } +} + +/** Artifacts a brand kit references must belong to the run's space. */ +async function assertSpaceArtifacts(services: MediaServices, spaceId: string, kit: BrandKit) { + const ids = [ + ...new Set([ + ...kit.fonts.map((font) => font.artifactId), + ...kit.logoArtifactIds, + ...(kit.lutArtifactId ? [kit.lutArtifactId] : []), + ...(kit.watermark?.artifactId ? [kit.watermark.artifactId] : []), + ...kit.musicBeds.map((bed) => bed.artifactId), + ]), + ]; + if (ids.length === 0) return; + const owned = await services.deps.prisma.artifact.count({ where: { spaceId, id: { in: ids } } }); + if (owned !== ids.length) throw new IsolationError(); +} + +async function search({ services, actor, rawArgs }: ToolContext, args: Args<"media_search">) { + const { repos } = services.deps; + const { help: _help, themeText, ...query } = args; + if (rawArgs.limit === undefined) query.limit = DEFAULT_SEARCH_LIMIT; + const theme = themeText?.trim().toLowerCase(); + if (!theme || query.collectionId) { + const page = await repos.searchAssets(actor.spaceId, query); + return { items: page.items.map(assetView), nextCursor: page.nextCursor }; + } + const themes = ( + await repos.listCollections(actor.spaceId, { kind: "theme", cursor: null, limit: 500 }) + ).items + .filter((collection) => collection.title.toLowerCase().includes(theme)) + .slice(0, MAX_THEME_MATCHES); + const seen = new Map(); + for (const collection of themes) { + const page = await repos.searchAssets(actor.spaceId, { + ...query, + cursor: null, + collectionId: collection.id, + }); + for (const asset of page.items) seen.set(asset.id, asset); + } + const items = [...seen.values()] + .sort( + (a, b) => (b.capturedAt ?? "").localeCompare(a.capturedAt ?? "") || b.id.localeCompare(a.id), + ) + .slice(0, query.limit); + return { + themes: themes.map((collection) => ({ id: collection.id, title: collection.title })), + items: items.map(assetView), + nextCursor: null, + }; +} + +async function collections({ services, actor }: ToolContext, args: Args<"media_collections">) { + const { repos } = services.deps; + const { spaceId } = actor; + switch (args.action) { + case "list": { + const page = await repos.listCollections(spaceId, { + kind: args.kind, + cursor: null, + limit: LIST_LIMIT, + }); + return { items: page.items.map(collectionView), more: page.nextCursor !== null }; + } + case "get": { + const found = await repos.getCollection(spaceId, need(args.collectionId, "collectionId")); + return { + collection: collectionView(found.collection), + assets: found.assets.slice(0, COLLECTION_ASSET_PREVIEW).map(assetView), + more: found.assets.length > COLLECTION_ASSET_PREVIEW, + }; + } + case "rename": { + const title = need(args.title?.trim() || undefined, "title"); + return { + collection: collectionView( + await repos.renameCollection(spaceId, need(args.collectionId, "collectionId"), title), + ), + }; + } + case "merge": { + const ids = need(args.collectionIds, "collectionIds"); + if (ids.length < 2) throw new ToolInputError("collectionIds needs at least two ids."); + return { collection: collectionView(await repos.mergeCollections(spaceId, ids)) }; + } + case "split": { + const parts = await repos.splitCollection( + spaceId, + need(args.collectionId, "collectionId"), + need(args.atAssetId, "atAssetId"), + ); + return { collections: parts.map(collectionView) }; + } + } +} + +async function project({ services, actor }: ToolContext, args: Args<"media_project">) { + const { repos } = services.deps; + const { spaceId } = actor; + switch (args.action) { + case "create": { + const input = CreateMediaProjectInput.parse({ + title: need(args.title, "title"), + goal: need(args.goal, "goal"), + profileIds: need(args.profileIds, "profileIds"), + collectionIds: args.collectionIds, + assetIds: args.assetIds, + brandKitId: args.brandKitId, + }); + await assertKnownProfiles(services, input.profileIds); + return { project: projectView(await repos.createProject(spaceId, actor.userId, input)) }; + } + case "update": { + const projectId = need(args.projectId, "projectId"); + if (args.profileIds && args.profileIds.length === 0) { + throw new ToolInputError("profileIds needs at least one profile."); + } + await assertKnownProfiles(services, args.profileIds); + const title = args.title?.trim(); + if (args.title !== undefined && !title) throw new ToolInputError("title cannot be empty."); + const updated = await repos.updateProject(spaceId, projectId, { + ...(title ? { title } : {}), + ...(args.goal ? { goal: args.goal } : {}), + ...(args.collectionIds ? { collectionIds: args.collectionIds } : {}), + ...(args.profileIds ? { profileIds: args.profileIds } : {}), + ...(args.brandKitId !== undefined ? { brandKitId: args.brandKitId } : {}), + }); + return { project: projectView(updated) }; + } + case "get": { + const found = await repos.getProject(spaceId, need(args.projectId, "projectId")); + return { + project: projectView(found.project), + plans: found.plans.map(planView), + renders: found.renders.map(renderView), + }; + } + case "list": { + const page = await repos.listProjects(spaceId, { cursor: null, limit: LIST_LIMIT }); + return { items: page.items.map(projectView), more: page.nextCursor !== null }; + } + case "auto_plan": { + const projectId = need(args.projectId, "projectId"); + await repos.getProjectRecord(spaceId, projectId); + await assertKnownProfiles(services, args.profileIds); + const plans = await services.marketing.autoPlan(actor, { + projectId, + ...(args.profileIds ? { profileIds: args.profileIds } : {}), + }); + return { plans: plans.map(planView) }; + } + } +} + +async function plan({ services, actor }: ToolContext, args: Args<"media_plan">) { + const { repos } = services.deps; + const projectId = need(args.projectId, "projectId"); + await repos.getProjectRecord(actor.spaceId, projectId); + if (args.edl === undefined) { + const plans = await repos.latestPlans(actor.spaceId, projectId); + return { + plans: plans.map((entry) => ({ ...planView(entry), edl: entry.edl })), + }; + } + const result = await services.marketing.submitPlan(actor, { + projectId, + edl: args.edl, + rationale: args.rationale ?? null, + createdBy: "agent", + }); + return result.ok ? { ok: true, plan: planView(result.plan) } : result; +} + +async function render({ services, actor }: ToolContext, args: Args<"media_render">) { + const { repos } = services.deps; + switch (args.action) { + case "start": { + const planId = need(args.planId, "planId"); + await repos.getPlan(actor.spaceId, planId); + const started = await services.render.start(actor, { + planId, + quality: args.quality ?? "draft", + }); + return { render: renderView(started) }; + } + case "status": + return { + render: renderView(await repos.getRender(actor.spaceId, need(args.renderId, "renderId"))), + }; + case "cancel": { + const renderId = need(args.renderId, "renderId"); + await repos.getRender(actor.spaceId, renderId); + return { render: renderView(await services.render.cancel(actor, renderId)) }; + } + } +} + +async function gateReport({ services, actor }: ToolContext, args: Args<"media_gate_report">) { + const found = await services.deps.repos.getRender(actor.spaceId, need(args.renderId, "renderId")); + const report = found.gateReport; + if (!report) return { renderId: found.id, status: found.status, report: null }; + return { + renderId: found.id, + status: found.status, + profileId: report.profileId, + passed: report.passed, + autoFixRound: report.autoFixRound, + blockers: report.issues.filter((issue) => issue.severity === "block").map(issueView), + warnings: report.issues.filter((issue) => issue.severity === "warn").map(issueView), + }; +} + +/** + * Reads, renames, sets anonymization, merges. Deliberately no consent path: + * the only consent write is the user's people.recordConsent RPC. + */ +async function people({ services, actor }: ToolContext, args: Args<"media_people">) { + const { repos } = services.deps; + const { spaceId } = actor; + switch (args.action) { + case "list": + return { people: (await repos.listPeople(spaceId)).map(personView) }; + case "update": { + const personId = need(args.personId, "personId"); + const displayName = args.displayName?.trim(); + if (args.displayName !== undefined && !displayName) { + throw new ToolInputError("displayName cannot be empty."); + } + if (!displayName && !args.anonymizeByDefault) { + throw new ToolInputError("Pass displayName or anonymizeByDefault."); + } + const updated = await repos.updatePerson(spaceId, personId, { + ...(displayName ? { displayName } : {}), + ...(args.anonymizeByDefault ? { anonymizeByDefault: args.anonymizeByDefault } : {}), + }); + return { person: personView(updated) }; + } + case "merge": { + const ids = [...new Set(need(args.personIds, "personIds"))]; + if (ids.length < 2) throw new ToolInputError("personIds needs at least two people."); + const involved = await Promise.all(ids.map((id) => repos.getPerson(spaceId, id))); + // Merging moves face tracks under the target's consent record; with any recorded + // consent involved that would extend consent, which only the user may do. + if ( + involved.some( + (person) => + person.consentRecordedAt !== null || + person.ageVerifiedAt !== null || + person.ageVerificationMethod !== "none", + ) + ) { + throw new ToolInputError( + "People with recorded consent or age proof can only be merged by the user in the app.", + ); + } + return { person: personView(await repos.mergePeople(spaceId, ids)) }; + } + } +} + +async function marks({ services, actor }: ToolContext, args: Args<"media_marks">) { + const { repos, mediaRoot } = services.deps; + const { spaceId } = actor; + switch (args.action) { + case "list": + return { + marks: ( + await repos.listMarks(spaceId, { + ...(args.personId ? { personId: args.personId } : {}), + ...(args.assetId ? { assetId: args.assetId } : {}), + }) + ).map(markView), + }; + case "create": { + const input = CreateMarkInput.parse({ + kind: need(args.kind, "kind"), + label: args.label ?? null, + personId: args.personId ?? null, + assetId: need(args.assetId, "assetId"), + prompt: need(args.prompt, "prompt"), + treatment: need(args.treatment, "treatment"), + }); + await repos.getAssetRecord(spaceId, input.assetId); + if (input.personId) await repos.getPerson(spaceId, input.personId); + return { mark: markView(await services.privacy.createMark(actor, input)) }; + } + case "update": { + const markId = need(args.markId, "markId"); + await repos.getMark(spaceId, markId); + if (args.personId) await repos.getPerson(spaceId, args.personId); + const updated = await services.privacy.updateMark(actor, { + markId, + ...(args.treatment ? { treatment: args.treatment } : {}), + ...(args.label !== undefined ? { label: args.label } : {}), + ...(args.personId !== undefined ? { personId: args.personId } : {}), + }); + return { mark: markView(updated) }; + } + case "remove": { + const { maskDir } = await repos.removeMark(spaceId, need(args.markId, "markId")); + if (maskDir) { + await rm(await resolveInside(mediaRoot, maskDir), { recursive: true, force: true }); + } + return { ok: true }; + } + } +} + +async function brandKit({ services, actor }: ToolContext, args: Args<"media_brand_kit">) { + const { repos, motion } = services.deps; + const { spaceId } = actor; + switch (args.action) { + case "list": + return { brandKits: (await repos.listBrandKits(spaceId)).map(brandKitView) }; + case "upsert": { + const kit = need(args.kit, "kit"); + await assertSpaceArtifacts(services, spaceId, kit); + const saved = await saveBrandKit(services.deps, actor, { + ...(args.id ? { id: args.id } : {}), + kit, + }); + return { id: saved.record.id, warnings: saved.warnings }; + } + case "templates": + return { + templates: (await motion.templates()).map((template) => ({ + id: template.id, + kind: template.kind, + name: template.name, + minDurationMs: template.minDurationMs, + maxDurationMs: template.maxDurationMs, + params: template.params, + })), + }; + case "intro_variants": { + const id = need(args.id, "id"); + await repos.getBrandKit(spaceId, id); + const variants = await services.marketing.introVariants(actor, { + brandKitId: id, + count: args.count ?? 3, + }); + return { + variants: variants.map((variant) => + compact({ + id: variant.id, + template: variant.template, + rationale: variant.rationale, + previewArtifactId: variant.previewArtifactId, + }), + ), + }; + } + } +} + +async function copy({ services, actor }: ToolContext, args: Args<"media_copy">) { + const renderId = need(args.renderId, "renderId"); + await services.deps.repos.getRender(actor.spaceId, renderId); + const language = args.language?.trim() || "en"; + if (language.length < 2) throw new ToolInputError("language needs at least two characters."); + return { copy: await services.marketing.copy(actor, { renderId, language }) }; +} + +async function thumbnails({ services, actor }: ToolContext, args: Args<"media_thumbnails">) { + const renderId = need(args.renderId, "renderId"); + await services.deps.repos.getRender(actor.spaceId, renderId); + const outputs = await services.marketing.thumbnails(actor, { renderId, count: args.count ?? 3 }); + return { thumbnails: outputs.map(outputView) }; +} + +/** + * Publications without uploads. A scheduled publication is handed to the + * configured MediaPublisher by the worker; the agent may only schedule when + * that publisher is the export bundle, so nothing reaches a platform without + * the user's approval. + */ +async function publish({ services, actor }: ToolContext, args: Args<"media_publish">) { + const { repos, publisher } = services.deps; + const { spaceId } = actor; + switch (args.action) { + case "create": { + const renderId = need(args.renderId, "renderId"); + await repos.getRender(spaceId, renderId); + if (args.scheduledAt && publisher.id !== BUNDLE_PUBLISHER_ID) { + throw new ToolInputError( + "Scheduling with an upload publisher needs the user's approval; schedule it in the app.", + ); + } + const created = await services.marketing.createPublication(actor, { + renderId, + destination: args.destination?.trim() || "bundle", + targetUrl: args.targetUrl ?? null, + utm: args.utm ?? null, + title: args.title ?? null, + caption: args.caption ?? null, + tags: args.tags ?? [], + thumbnailVariant: args.thumbnailVariant ?? null, + scheduledAt: args.scheduledAt ?? null, + }); + return { publication: publicationView(created) }; + } + case "bundle": { + const publicationId = need(args.publicationId, "publicationId"); + await repos.getPublication(spaceId, publicationId); + return { + publication: publicationView(await services.marketing.bundle(actor, publicationId)), + }; + } + case "list": { + const page = await repos.listPublications(spaceId, { + cursor: null, + limit: LIST_LIMIT, + ...(args.projectId ? { projectId: args.projectId } : {}), + }); + return { items: page.items.map(publicationView), more: page.nextCursor !== null }; + } + case "cancel": { + const publicationId = need(args.publicationId, "publicationId"); + await repos.getPublication(spaceId, publicationId); + return { + publication: publicationView( + await services.marketing.cancelPublication(actor, publicationId), + ), + }; + } + } +} + +async function metrics({ services, actor }: ToolContext, args: Args<"media_metrics">) { + switch (args.action) { + case "funnel": { + const rows = await services.deps.repos.funnel(actor.spaceId, args.projectId); + return { rows, totals: funnelTotals(rows) }; + } + case "record": + return { metric: await recordMetric(services.deps, actor, need(args.metric, "metric")) }; + } +} + +const HANDLERS: { + [N in MediaToolName]: (context: ToolContext, args: Args) => Promise; +} = { + media_search: search, + media_collections: collections, + media_project: project, + media_plan: plan, + media_render: render, + media_gate_report: gateReport, + media_people: people, + media_marks: marks, + media_brand_kit: brandKit, + media_copy: copy, + media_thumbnails: thumbnails, + media_publish: publish, + media_metrics: metrics, +}; + +const GUIDE_HINT = 'Call this tool with {"help": true} for the guide.'; + +/** Known failures become tool errors the model can act on; anything else propagates. */ +function toolError(error: unknown): ToolResult | null { + if (error instanceof IsolationError) { + return { error: "Not found in this space." }; + } + if (error instanceof ToolInputError) return { error: error.message, hint: GUIDE_HINT }; + if ( + error instanceof InvalidMediaInputError || + error instanceof MediaStateConflictError || + error instanceof UnsupportedMediaOperationError || + error instanceof PrivacyUnavailableError + ) { + return { error: error.message }; + } + if (error instanceof PathOutsideRootError) return { error: "That path is not allowed." }; + if (error && typeof error === "object" && "issues" in error && Array.isArray(error.issues)) { + return { + error: "Invalid arguments.", + issues: (error.issues as Array<{ path: PropertyKey[]; message: string }>).map((issue) => ({ + path: issue.path.map(String).join("."), + message: issue.message, + })), + hint: GUIDE_HINT, + }; + } + return null; +} + +/** + * Run one media_* tool for the run's actor. `services` is null when Media Studio + * is not configured; the tools are then not offered, and a stray call fails. + */ +export async function executeMediaTool( + services: MediaServices | null, + actor: MediaActor, + name: string, + rawArgs: Record, +): Promise { + if (!isMediaToolName(name)) return { error: `Unknown media tool ${name}.` }; + if (!services) return { error: MEDIA_NOT_CONFIGURED_TOOL_ERROR }; + if (rawArgs.help === true) return { guide: MEDIA_TOOL_GUIDES[name] }; + const parsed = MediaToolInputSchemas[name].safeParse(rawArgs); + if (!parsed.success) { + return toolError(parsed.error) ?? { error: "Invalid arguments.", hint: GUIDE_HINT }; + } + const handler = HANDLERS[name] as (context: ToolContext, args: unknown) => Promise; + try { + return await handler({ services, actor, rawArgs }, parsed.data); + } catch (error) { + const mapped = toolError(error); + if (mapped) return mapped; + throw error; + } +} diff --git a/packages/adapters/src/media-tools.test.ts b/packages/adapters/src/media-tools.test.ts new file mode 100644 index 0000000..8be732b --- /dev/null +++ b/packages/adapters/src/media-tools.test.ts @@ -0,0 +1,523 @@ +import type { MediaPerson, MediaRender } from "@rakazo/contracts"; +import { MEDIA_TOOL_NAMES, MediaToolInputSchemas } from "@rakazo/contracts"; +import { IsolationError } from "@rakazo/db"; +import { describe, expect, it, vi } from "vitest"; +import type { MediaServices } from "./media-context.js"; +import { filterMediaSkills, MEDIA_AGENT_SKILLS, MEDIA_HARD_LIMITS } from "./media-skills.js"; +import { + describeMediaToolActivity, + filterMediaTools, + isMediaToolName, + MEDIA_TOOL_GUIDES, + mediaAgentTools, +} from "./media-tools.js"; +import { executeMediaTool, MEDIA_NOT_CONFIGURED_TOOL_ERROR } from "./media-tools-execute.js"; + +const actor = { spaceId: "space-a", userId: "user-a" }; +const NOW = "2026-01-01T00:00:00.000Z"; + +function person(overrides: Partial = {}): MediaPerson { + return { + id: "person-1", + displayName: "Person 1", + isSelf: false, + confirmed: false, + anonymizeByDefault: "blur", + consentRecordedAt: null, + consentArtifactId: null, + ageVerifiedAt: null, + ageVerificationMethod: "none", + faceTrackCount: 2, + coverThumbAssetId: null, + ...overrides, + }; +} + +function renderRow(overrides: Partial = {}): MediaRender { + return { + id: "render-1", + planId: "plan-1", + projectId: "project-1", + profileId: "example-short-vertical", + quality: "final", + status: "done", + progress: 1, + outputs: [{ kind: "video", artifactId: null, fileName: "video.mp4", sizeBytes: 10 }], + gateReport: { + profileId: "example-short-vertical", + passed: false, + checkedAt: NOW, + sampledFrames: 10, + autoFixRound: 1, + issues: [ + { + check: "consent", + severity: "block", + code: "consent_missing", + message: "Person 1 has no consent", + atMs: 1200, + endMs: 2400, + personId: "person-1", + }, + { check: "text", severity: "warn", code: "caption_long", message: "Caption is long" }, + ], + }, + ffprobe: null, + error: null, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +/** + * Services whose repos record every call. Unlisted repo methods resolve to + * undefined so a test fails loudly on its assertions instead of on wiring. + */ +function fakeServices(options: { + repos?: Record unknown>; + marketing?: Record unknown>; + publisherId?: string; + artifactCount?: number; +}) { + const calls: Array<{ method: string; args: unknown[] }> = []; + const repos = new Proxy( + {}, + { + get(_target, method: string) { + return async (...args: unknown[]) => { + calls.push({ method, args }); + return options.repos?.[method]?.(...args); + }; + }, + }, + ); + const publish = vi.fn(); + const marketing = new Proxy( + {}, + { + get(_target, method: string) { + return async (...args: unknown[]) => { + calls.push({ method: `marketing.${method}`, args }); + return options.marketing?.[method]?.(...args); + }; + }, + }, + ); + const render = { + start: vi.fn(async (_actor: unknown, input: { quality: string }) => + renderRow({ status: "queued", progress: 0, quality: input.quality as "draft" }), + ), + cancel: vi.fn(async () => renderRow({ status: "canceled" })), + outputPath: vi.fn(), + }; + const privacy = { + createMark: vi.fn(), + updateMark: vi.fn(), + prepareSources: vi.fn(), + gate: vi.fn(), + removePerson: vi.fn(), + }; + const services = { + deps: { + repos, + prisma: { artifact: { count: vi.fn(async () => options.artifactCount ?? 0) } }, + publisher: { id: options.publisherId ?? "bundle", publish }, + profiles: async () => ({ profiles: new Map([["example-short-vertical", {}]]), errors: [] }), + motion: { templates: async () => [] }, + mediaRoot: "/nonexistent-media-root", + }, + marketing, + render, + privacy, + ingest: {}, + } as unknown as MediaServices; + return { services, calls, publish, render, privacy }; +} + +describe("media tool declarations", () => { + it("declares every frozen media tool with a help flag", () => { + expect(mediaAgentTools.map((tool) => tool.name)).toEqual([...MEDIA_TOOL_NAMES]); + for (const tool of mediaAgentTools) { + expect(tool.inputSchema.type).toBe("object"); + expect(tool.inputSchema.properties).toHaveProperty("help"); + expect(tool.description).toContain('{"help": true}'); + expect(MEDIA_TOOL_GUIDES[tool.name as keyof typeof MEDIA_TOOL_GUIDES]).toBeTruthy(); + } + }); + + it("offers media tools only when Media Studio is configured", () => { + const tools = [{ name: "read_file" }, ...mediaAgentTools]; + expect(filterMediaTools(tools, true)).toHaveLength(tools.length); + expect(filterMediaTools(tools, false)).toEqual([{ name: "read_file" }]); + expect(isMediaToolName("media_people")).toBe(true); + expect(isMediaToolName("render_plot")).toBe(false); + }); + + it("labels every media tool call and leaves other tools alone", () => { + for (const name of MEDIA_TOOL_NAMES) { + const label = describeMediaToolActivity(name, {}); + expect(label).toBeTruthy(); + expect(label).not.toContain("media_"); + } + expect(describeMediaToolActivity("media_render", { action: "start", quality: "final" })).toBe( + "Starting final render", + ); + expect(describeMediaToolActivity("media_search", { text: "beach" })).toBe( + "Searching media: beach", + ); + expect(describeMediaToolActivity("media_people", { help: true })).toBe( + "Reading the media guide", + ); + expect(describeMediaToolActivity("render_plot", {})).toBeUndefined(); + }); +}); + +describe("media skills", () => { + it("carry the section 0 limits verbatim", () => { + expect(MEDIA_AGENT_SKILLS.map((skill) => skill.name)).toEqual(["Video Producer", "Video Copy"]); + for (const skill of MEDIA_AGENT_SKILLS) { + expect(skill.body).toContain(MEDIA_HARD_LIMITS); + expect(skill.description.length).toBeLessThanOrEqual(72); + } + expect(MEDIA_HARD_LIMITS).toContain( + "Gesichtstausch mit dem Gesicht einer realen dritten Person, also Deepfakes.", + ); + expect(MEDIA_HARD_LIMITS).toContain( + "Export für Bezahl-Plattformen ohne dokumentierte Einwilligung und Altersnachweis aller erkannten Mitwirkenden.", + ); + }); + + it("hide built-in media skills when Media Studio is not configured", () => { + const skills = [ + { name: "Interrogate", source: "builtin" }, + { name: "Video Producer", source: "builtin" }, + { name: "Video Copy", source: "user" }, + ]; + expect(filterMediaSkills(skills, true)).toEqual(skills); + expect(filterMediaSkills(skills, false).map((skill) => skill.source)).toEqual([ + "builtin", + "user", + ]); + }); +}); + +describe("executeMediaTool", () => { + it("fails when Media Studio is not configured and returns guides without touching services", async () => { + expect(await executeMediaTool(null, actor, "media_search", {})).toEqual({ + error: MEDIA_NOT_CONFIGURED_TOOL_ERROR, + }); + const { services, calls } = fakeServices({}); + for (const name of MEDIA_TOOL_NAMES) { + const result = await executeMediaTool(services, actor, name, { help: true }); + expect(result).toEqual({ guide: MEDIA_TOOL_GUIDES[name] }); + } + expect(calls).toEqual([]); + expect(await executeMediaTool(services, actor, "render_plot", {})).toHaveProperty("error"); + }); + + it("returns structured argument errors", async () => { + const { services } = fakeServices({}); + const result = await executeMediaTool(services, actor, "media_render", { action: "explode" }); + expect(result).toMatchObject({ error: "Invalid arguments." }); + expect((result.issues as Array<{ path: string }>)[0]?.path).toBe("action"); + expect(await executeMediaTool(services, actor, "media_render", { action: "start" })).toEqual({ + error: "planId is required.", + hint: 'Call this tool with {"help": true} for the guide.', + }); + }); + + it("scopes every repo call to the run's space and hides foreign ids", async () => { + const { services, calls } = fakeServices({ + repos: { + searchAssets: () => ({ items: [], nextCursor: null }), + listPeople: () => [], + getRender: (spaceId) => { + if (spaceId !== actor.spaceId) throw new Error("wrong space"); + throw new IsolationError(); + }, + }, + }); + await executeMediaTool(services, actor, "media_search", { text: "beach" }); + await executeMediaTool(services, actor, "media_people", { action: "list" }); + expect( + await executeMediaTool(services, actor, "media_gate_report", { renderId: "render-x" }), + ).toEqual({ error: "Not found in this space." }); + expect(calls.length).toBeGreaterThan(0); + for (const call of calls) expect(call.args[0]).toBe(actor.spaceId); + expect(calls[0]?.args[1]).toMatchObject({ text: "beach", limit: 25 }); + }); + + it("searches inside matching theme collections", async () => { + const asset = (id: string, capturedAt: string) => ({ + id, + kind: "video", + capturedAt, + durationMs: 1000, + width: 1920, + height: 1080, + qualityScore: 0.812, + ingestState: "analyzed", + isFavorite: false, + }); + const { services, calls } = fakeServices({ + repos: { + listCollections: () => ({ + items: [ + { id: "theme-1", title: "Beach days" }, + { id: "theme-2", title: "Mountains" }, + ], + nextCursor: null, + }), + searchAssets: (_spaceId, query) => ({ + items: + (query as { collectionId: string }).collectionId === "theme-1" + ? [asset("a1", "2025-01-01T00:00:00.000Z"), asset("a2", "2025-02-01T00:00:00.000Z")] + : [], + nextCursor: null, + }), + }, + }); + const result = await executeMediaTool(services, actor, "media_search", { themeText: "beach" }); + expect(result).toEqual({ + themes: [{ id: "theme-1", title: "Beach days" }], + items: [ + { + id: "a2", + kind: "video", + capturedAt: "2025-02-01T00:00:00.000Z", + durationMs: 1000, + size: "1920x1080", + quality: 0.81, + state: "analyzed", + }, + expect.objectContaining({ id: "a1" }), + ], + nextCursor: null, + }); + expect(calls.filter((call) => call.method === "searchAssets")).toHaveLength(1); + }); + + it("media_people can never write consent", async () => { + const consentFields = { + consentRecordedAt: NOW, + consentArtifactId: "artifact-1", + ageVerifiedAt: NOW, + ageVerificationMethod: "document_on_file", + confirmed: true, + isSelf: true, + }; + const { services, calls } = fakeServices({ + repos: { + listPeople: () => [person()], + getPerson: (_spaceId, personId) => person({ id: personId as string }), + updatePerson: () => person({ displayName: "Alex" }), + mergePeople: () => person(), + }, + }); + // The schema the model sees has no consent fields at all. + const schema = mediaAgentTools.find((tool) => tool.name === "media_people")!.inputSchema; + const properties = Object.keys(schema.properties as Record); + expect(properties.some((key) => /consent|age|confirm|self/i.test(key))).toBe(false); + expect(MediaToolInputSchemas.media_people.parse({ action: "list", ...consentFields })).toEqual({ + action: "list", + }); + + for (const action of ["list", "update", "merge"] as const) { + await executeMediaTool(services, actor, "media_people", { + action, + personId: "person-1", + personIds: ["person-1", "person-2"], + displayName: "Alex", + ...consentFields, + }); + } + // Even with only consent fields, an update does nothing. + expect( + await executeMediaTool(services, actor, "media_people", { + action: "update", + personId: "person-1", + ...consentFields, + }), + ).toMatchObject({ error: "Pass displayName or anonymizeByDefault." }); + + expect(calls.some((call) => /consent|age/i.test(call.method))).toBe(false); + for (const call of calls.filter((entry) => entry.method === "updatePerson")) { + expect(call.args).toEqual([actor.spaceId, "person-1", { displayName: "Alex" }]); + } + expect(calls.some((call) => call.method === "mergePeople")).toBe(true); + }); + + it("refuses to merge people with recorded consent", async () => { + const { services, calls } = fakeServices({ + repos: { + getPerson: (_spaceId, personId) => + personId === "person-1" + ? person({ id: "person-1", consentRecordedAt: NOW, isSelf: true }) + : person({ id: personId as string }), + }, + }); + const result = await executeMediaTool(services, actor, "media_people", { + action: "merge", + personIds: ["person-1", "person-2"], + }); + expect(result.error).toMatch(/only be merged by the user/); + expect(calls.some((call) => call.method === "mergePeople")).toBe(false); + }); + + it("reports gate blockers with timestamps", async () => { + const { services } = fakeServices({ repos: { getRender: () => renderRow() } }); + expect( + await executeMediaTool(services, actor, "media_gate_report", { renderId: "render-1" }), + ).toEqual({ + renderId: "render-1", + status: "done", + profileId: "example-short-vertical", + passed: false, + autoFixRound: 1, + blockers: [ + { + check: "consent", + severity: "block", + code: "consent_missing", + message: "Person 1 has no consent", + atMs: 1200, + endMs: 2400, + personId: "person-1", + }, + ], + warnings: [ + { check: "text", severity: "warn", code: "caption_long", message: "Caption is long" }, + ], + }); + }); + + it("returns plan validation errors as structured codes and tags plans as agent-made", async () => { + const errors = [{ code: "overlap", path: "tracks.video[1]", message: "Clips overlap" }]; + const { services, calls } = fakeServices({ + repos: { getProjectRecord: () => ({ id: "project-1" }) }, + marketing: { submitPlan: () => ({ ok: false, errors }) }, + }); + expect( + await executeMediaTool(services, actor, "media_plan", { + projectId: "project-1", + edl: { edlVersion: 1 }, + }), + ).toEqual({ ok: false, errors }); + const submit = calls.find((call) => call.method === "marketing.submitPlan"); + expect(submit?.args).toEqual([ + actor, + { projectId: "project-1", edl: { edlVersion: 1 }, rationale: null, createdBy: "agent" }, + ]); + }); + + it("starts drafts by default and summarizes renders", async () => { + const { services, render } = fakeServices({ repos: { getPlan: () => ({ id: "plan-1" }) } }); + const result = await executeMediaTool(services, actor, "media_render", { + action: "start", + planId: "plan-1", + }); + expect(render.start).toHaveBeenCalledWith(actor, { planId: "plan-1", quality: "draft" }); + expect(result.render).toMatchObject({ status: "queued", quality: "draft", progress: 0 }); + expect(result.render).toMatchObject({ gate: { passed: false, blockers: 1, warnings: 1 } }); + }); + + it("media_publish bundles but never uploads", async () => { + const publication = { + id: "pub-1", + renderId: "render-1", + profileId: "example-short-vertical", + destination: "bundle", + status: "scheduled", + scheduledAt: NOW, + publishedAt: null, + externalUrl: null, + targetUrl: null, + trackingUrl: null, + utm: null, + title: null, + caption: null, + tags: [], + thumbnailVariant: null, + bundleArtifactId: null, + createdAt: NOW, + }; + const repos = { + getRender: () => renderRow(), + getPublication: () => publication, + listPublications: () => ({ items: [publication], nextCursor: null }), + }; + const marketing = { + createPublication: () => publication, + bundle: () => ({ ...publication, status: "bundled", bundleArtifactId: "artifact-9" }), + cancelPublication: () => ({ ...publication, status: "canceled" }), + }; + const bundleOnly = fakeServices({ repos, marketing }); + for (const args of [ + { action: "create", renderId: "render-1", scheduledAt: NOW }, + { action: "bundle", publicationId: "pub-1" }, + { action: "list" }, + { action: "cancel", publicationId: "pub-1" }, + ]) { + expect( + await executeMediaTool(bundleOnly.services, actor, "media_publish", args), + ).not.toHaveProperty("error"); + } + expect(bundleOnly.publish).not.toHaveBeenCalled(); + const create = bundleOnly.calls.find((call) => call.method === "marketing.createPublication"); + expect(create?.args[1]).toMatchObject({ destination: "bundle", scheduledAt: NOW }); + + const uploader = fakeServices({ repos, marketing, publisherId: "connector-upload" }); + const scheduled = await executeMediaTool(uploader.services, actor, "media_publish", { + action: "create", + renderId: "render-1", + scheduledAt: NOW, + }); + expect(scheduled.error).toMatch(/approval/); + expect(uploader.calls.some((call) => call.method === "marketing.createPublication")).toBe( + false, + ); + expect(uploader.publish).not.toHaveBeenCalled(); + }); + + it("rejects brand kits that reference another space's artifacts", async () => { + const kit = { + name: "Brand", + colors: { + primary: "#111111", + secondary: "#222222", + accent: "#333333", + text: "#444444", + background: "#555555", + }, + logoArtifactIds: ["artifact-foreign"], + }; + const { services } = fakeServices({ artifactCount: 0 }); + expect( + await executeMediaTool(services, actor, "media_brand_kit", { action: "upsert", kit }), + ).toEqual({ error: "Not found in this space." }); + }); + + it("totals the funnel per currency", async () => { + const row = { + publicationId: "pub-1", + projectId: "project-1", + profileId: "example-short-vertical", + variant: null, + thumbnailVariant: null, + views: 100, + clicks: 10, + conversions: 2, + revenueCents: 1998, + currency: "EUR", + }; + const { services } = fakeServices({ + repos: { funnel: () => [row, { ...row, publicationId: "pub-2", views: 50 }] }, + }); + expect( + await executeMediaTool(services, actor, "media_metrics", { action: "funnel" }), + ).toMatchObject({ + totals: [{ currency: "EUR", views: 150, clicks: 20, conversions: 4, revenueCents: 3996 }], + }); + }); +}); diff --git a/packages/adapters/src/media-tools.ts b/packages/adapters/src/media-tools.ts new file mode 100644 index 0000000..4f7cc1e --- /dev/null +++ b/packages/adapters/src/media-tools.ts @@ -0,0 +1,185 @@ +import type { ConnectorTool } from "@rakazo/adapter-kit"; +import type { MediaToolName } from "@rakazo/contracts"; +import { MEDIA_TOOL_NAMES, MediaToolInputSchemas } from "@rakazo/contracts"; +import { z } from "zod"; +import { sanitizeSensitiveText } from "./pi-model-config.js"; + +/* + * Media Studio agent tools (spec 11): model-facing declarations, the + * {"help": true} guides, and the live activity labels. Execution lives in + * media-tools-execute.ts. The tools are only offered when Media Studio is + * configured (filterMediaTools), and none of them can record consent or upload + * to a platform: consent is set by the signed-in user in the People view, and + * uploads to connected accounts go through connector tools and approval. + */ + +const MEDIA_TOOL_NAME_SET: ReadonlySet = new Set(MEDIA_TOOL_NAMES); + +export function isMediaToolName(name: string): name is MediaToolName { + return MEDIA_TOOL_NAME_SET.has(name); +} + +const HELP_HINT = 'Call with {"help": true} for the full guide.'; + +const DESCRIPTIONS: Record = { + media_search: `Search this space's photo and video library by text, date range, place, person, collection, theme, kind, or minimum quality. Returns compact asset rows (ids, kind, capture time, duration, quality). ${HELP_HINT}`, + media_collections: `List, open, rename, merge, or split sessions and themes in the media library. ${HELP_HINT}`, + media_project: `Create, update, open, or list media projects (goal, platform profiles, brand kit, collections), or build deterministic first-cut plans per profile with auto_plan. ${HELP_HINT}`, + media_plan: `Read a project's latest edit plans (EDL per profile) or submit a new or changed EDL. Validation errors come back as structured codes with paths. ${HELP_HINT}`, + media_render: `Start a draft or final render of an edit plan, check its status, or cancel it. ${HELP_HINT}`, + media_gate_report: `Read a render's platform gate report: blockers and warnings with timestamps. ${HELP_HINT}`, + media_people: `List detected people with their consent status, rename them, set their default anonymization, or merge duplicates. Consent can only be recorded by the user in the app. ${HELP_HINT}`, + media_marks: `List, mark, update, or remove identifying features (tattoos, scars, jewelry, plates, background details) and their treatment. ${HELP_HINT}`, + media_brand_kit: `List or save brand kits, list motion templates, or generate signature intro variants. ${HELP_HINT}`, + media_copy: `Generate platform copy (title, hook, caption, hashtags) for a render and validate it against the render's profile. ${HELP_HINT}`, + media_thumbnails: `Generate thumbnail variants for a finished render. ${HELP_HINT}`, + media_publish: `Create, schedule, bundle, list, or cancel publications of a final render. Produces the export bundle; never uploads to a platform. ${HELP_HINT}`, + media_metrics: `Read the results funnel (views, clicks, conversions, revenue) per publication, or record one observation. ${HELP_HINT}`, +}; + +/** Model-facing declarations; schemas come from the frozen contracts. */ +export const mediaAgentTools: ConnectorTool[] = MEDIA_TOOL_NAMES.map((name) => ({ + name, + description: DESCRIPTIONS[name], + inputSchema: z.toJSONSchema(MediaToolInputSchemas[name], { + io: "input", + unrepresentable: "any", + }) as Record, +})); + +/** Media tools exist only where Media Studio is configured. */ +export function filterMediaTools(tools: T[], configured: boolean): T[] { + return configured ? tools : tools.filter((tool) => !isMediaToolName(tool.name)); +} + +const ACTIVITY_LIMIT = 60; + +function short(value: unknown): string { + const text = sanitizeSensitiveText(String(value ?? "")) + .replaceAll(/\s+/g, " ") + .trim(); + return text.length > ACTIVITY_LIMIT ? `${text.slice(0, ACTIVITY_LIMIT)}…` : text; +} + +/** One live status line per media tool call; undefined for other tools. */ +export function describeMediaToolActivity( + toolName: string, + args: Record, +): string | undefined { + if (!isMediaToolName(toolName)) return undefined; + if (args.help === true) return "Reading the media guide"; + const action = args.action; + switch (toolName) { + case "media_search": { + const text = short(args.text ?? args.themeText); + return text ? `Searching media: ${text}` : "Searching media"; + } + case "media_collections": + if (action === "rename") return "Renaming a collection"; + if (action === "merge") return "Merging collections"; + if (action === "split") return "Splitting a collection"; + return "Browsing collections"; + case "media_project": + if (action === "create") return `Creating media project: ${short(args.title)}`; + if (action === "update") return "Updating media project"; + if (action === "auto_plan") return "Drafting edit plans"; + return "Opening media projects"; + case "media_plan": + return args.edl === undefined ? "Reading edit plans" : "Checking edit plan"; + case "media_render": + if (action === "start") + return args.quality === "final" ? "Starting final render" : "Starting draft render"; + if (action === "cancel") return "Canceling render"; + return "Checking render"; + case "media_gate_report": + return "Reading gate report"; + case "media_people": + if (action === "merge") return "Merging people"; + if (action === "update") return "Updating person"; + return "Listing people"; + case "media_marks": + if (action === "create") return "Marking a feature"; + if (action === "update") return "Updating a mark"; + if (action === "remove") return "Removing a mark"; + return "Listing marks"; + case "media_brand_kit": + if (action === "upsert") return "Saving brand kit"; + if (action === "intro_variants") return "Generating intro variants"; + if (action === "templates") return "Listing motion templates"; + return "Reading brand kits"; + case "media_copy": + return "Writing platform copy"; + case "media_thumbnails": + return "Generating thumbnails"; + case "media_publish": + if (action === "create") + return args.scheduledAt ? "Scheduling publication" : "Creating publication"; + if (action === "bundle") return "Building export bundle"; + if (action === "cancel") return "Canceling publication"; + return "Listing publications"; + case "media_metrics": + return action === "record" ? "Recording results" : "Reading results"; + } +} + +/** The {"help": true} guide per tool. Short, actionable, no repetition of the schema. */ +export const MEDIA_TOOL_GUIDES: Record = { + media_search: `media_search finds assets in this space's library. +- Filters combine: text (file names, albums, collection titles, transcripts, labels), from/to (ISO dates), near {lat, lon, radiusKm}, personId, collectionId, sourceId, kinds ["video","photo","live_photo"], minQuality 0..1, favoritesOnly, ingestStates. +- themeText matches theme collections by title and searches inside the matches. +- limit defaults to 25 here; pass the returned nextCursor as cursor for the next page. +- Results are ids and summaries only; use media_collections get or a project for more.`, + media_collections: `media_collections works with sessions (shoots grouped by time and place), themes, and manual collections. +- {"action":"list","kind":"session"|"theme"|"manual"} +- {"action":"get","collectionId"} returns the collection and its first assets. +- {"action":"rename","collectionId","title"} +- {"action":"merge","collectionIds":[two or more]} +- {"action":"split","collectionId","atAssetId"} starts a new collection at that asset.`, + media_project: `media_project manages projects. +- {"action":"create","title","goal","profileIds":[...],"collectionIds"?,"assetIds"?,"brandKitId"?}; goal is one of teaser, full_release, trailer, highlight, story, custom. +- {"action":"update","projectId",...fields to change} +- {"action":"get","projectId"} returns latest plans and renders. +- {"action":"list"} +- {"action":"auto_plan","projectId","profileIds"?} builds a deterministic first cut per profile; refine it with media_plan. +Profile ids come from the platform profiles the user has (unknown ids are rejected).`, + media_plan: `media_plan reads or submits edit decision lists (EDL, edlVersion 1). +- {"projectId"} returns the latest plan per profile with its full EDL. +- {"projectId","edl":{...},"rationale"?} validates and stores a new plan version for edl.output.profileId. +Validation failures return {"ok":false,"errors":[{code,path,message}]}; fix exactly those paths and resubmit. Codes include range_out_of_asset, overlap, duration_below_min, duration_above_max, paid_only_segment, overlay_outside_safe_zone, unknown_asset, unknown_mark, music_license, profile_mismatch. +Never use segments marked paid_only in teasers. For short vertical profiles put the hook before the intro.`, + media_render: `media_render runs the deterministic render engine. +- {"action":"start","planId","quality":"draft"|"final"} (default draft). Render drafts first; render final only after the draft passes the gate. +- {"action":"status","renderId"} while status is queued, rendering, or gating; check again later instead of waiting in a loop. +- {"action":"cancel","renderId"} +A blocked render failed the gate: read media_gate_report, change the plan, and render again.`, + media_gate_report: `media_gate_report {"renderId"} lists blockers and warnings with atMs/endMs timestamps, the check (technical, content, anonymization, consent, text, disclosure, brand), and the person or mark involved. +Fix blockers only with compliant changes: cut or trim the flagged range, add redactions, shorten text, add disclosure. Never try to get around a classifier or an age gate. +Consent blockers can only be resolved by the user recording consent and age proof in the People view; tell them which person needs it.`, + media_people: `media_people reads people and their consent status, and sets how they are anonymized. +- {"action":"list"}: consent and ageVerified show what the user has recorded. +- {"action":"update","personId","displayName"?,"anonymizeByDefault"?}; modes: none, blur, pixelate, solid, sticker, partial_eyes, partial_upper, partial_lower, reframe_out, synthetic_identity. +- {"action":"merge","personIds":[two or more]} only for people without recorded consent. +You cannot record or change consent or age proof. When an export needs it, ask the user to record it in the People view.`, + media_marks: `media_marks handles identifying features. +- {"action":"list","personId"?,"assetId"?} +- {"action":"create","kind","assetId","prompt":{"tMs","box"?:{x,y,w,h},"point"?:{x,y}},"treatment","personId"?,"label"?}; coordinates are normalized 0..1. kind: tattoo, birthmark, scar, jewelry, background_detail, license_plate, custom. treatment: inpaint, blur, cover, reframe_out, none. +- {"action":"update","markId","treatment"?,"label"?,"personId"?} +- {"action":"remove","markId"} +A low qualityScore comes with suggestedFallback; prefer it.`, + media_brand_kit: `media_brand_kit manages brand identity. +- {"action":"list"} returns every kit with its full definition. +- {"action":"upsert","id"?,"kit":{...}} saves a kit (name, colors as #RRGGBB, fonts, logos, captionPresets, musicBeds, intro, outro, endCard, tone, audience). Artifacts must belong to this space. +- {"action":"templates"} lists motion templates with their parameters. +- {"action":"intro_variants","id","count"?} generates signature intro variants (default 3).`, + media_copy: `media_copy {"renderId","language"?} generates title, hook, caption, hashtags, and optional description, chapters, and alt text for the render's profile, then validates them against the profile's banned patterns, lengths, and link policy. Issues come back in "issues"; rewrite and keep the claims true. Default language is en.`, + media_thumbnails: `media_thumbnails {"renderId","count"?} generates thumbnail variants (default 3) from a finished render. Variants pass the same gate as the render.`, + media_publish: `media_publish prepares publications of a final render that passed the gate. +- {"action":"create","renderId","targetUrl"?,"utm"?,"title"?,"caption"?,"tags"?,"thumbnailVariant"?,"scheduledAt"?,"destination"?}; UTM defaults to profile, format, project, and variant. +- {"action":"bundle","publicationId"} builds the export bundle (video, posters, subtitles, caption, metadata, UPLOAD-CHECKLIST.md). +- {"action":"list","projectId"?}, {"action":"cancel","publicationId"} +This tool never uploads. To upload to a connected account, use that connector's tool; it asks the user for approval. Paid-video platforms only ever get the bundle.`, + media_metrics: `media_metrics reads and records results. +- {"action":"funnel","projectId"?} returns views, clicks, conversions, and revenue per publication with totals per currency. +- {"action":"record","metric":{"publicationId","observedAt","views","clicks","conversions","revenueCents","currency"}} records one manual observation. +CSV imports run in the app's Results view.`, +};