import { mkdir, readFile, rm } from "node:fs/promises"; import path from "node:path"; import type { AgentInputImage } from "@rakazo/adapter-kit"; import type { BrandKit, ContentCategory, Edl, EdlError, MediaEditPlan, MediaProjectGoal, MediaSegment, PlatformProfile, TeaserWeights, } from "@rakazo/contracts"; import { TEASER_LEARNING_MIN_SAMPLES } from "@rakazo/contracts"; import type { MediaProjectRecord, MediaRepos } from "@rakazo/db"; import { InvalidMediaInputError, MediaStateConflictError } from "@rakazo/db"; import { getLogger } from "@rakazo/logging"; import type { EdlContext, ScoredSegmentInput, SegmentFeatures } from "@rakazo/media"; import { autoPlanEdl, defaultTeaserWeights, introMs, scoreSegment, speechHookScore, validateEdl, } from "@rakazo/media"; import { resolveInside } from "@rakazo/media/node"; import { z } from "zod"; import type { MediaActor, MediaServiceDeps } from "./media-context.js"; /* * Edit plans for a project: the deterministic first cut per profile, an * optional model ranking of the teaser candidates, and validation of plans the * agent or user submits. * * Model judge (spec 8): only the top JUDGE_TOP_K candidates by deterministic * score go to the model, and only those whose stored classifier scores already * pass the target profile thresholds and that show no face of a protected * person (or an unassigned face next to one). The model receives one still * frame per candidate, taken from the proxy, plus the transcript excerpt. * Face crops, embeddings and person data never leave the host. The model's * order re-weights the candidates: rank r of n scales every feature by * 1 - r / (2n); every usable segment it did not rank keeps half its weight * (paid-only segments stay as they are: they only mark the payoff). Without a * model, consent, or a usable reply the deterministic order stands. */ export const JUDGE_TOP_K = 12; const JUDGE_FRAME_WIDTH = 320; const TRANSCRIPT_EXCERPT_CHARS = 300; const RATIONALE_MAX_CHARS = 2000; const RANKING_GOALS: ReadonlySet = new Set([ "teaser", "trailer", "highlight", "story", ]); const JudgeReplySchema = z.object({ ranking: z .array(z.object({ segmentId: z.string().min(1).max(100), reason: z.string().max(300) })) .max(50), }); type AssetRow = Awaited>[number]; type FaceTrackRow = Awaited>[number]; type PersonRow = Awaited>[number]; type ClassifierFrames = Array<{ tMs: number; scores: Partial> }>; export interface PlanContext { project: MediaProjectRecord; kit: BrandKit | null; assets: Map; segments: MediaSegment[]; faceTracks: FaceTrackRow[]; /** Every person of the space; plans may reference any of them. */ people: PersonRow[]; markIds: Set; marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; profiles: Map; } export async function loadPlanContext( deps: MediaServiceDeps, actor: MediaActor, projectId: string, ): Promise { const { repos } = deps; const { spaceId } = actor; const project = await repos.getProjectRecord(spaceId, projectId); const fromCollections = await repos.collectionAssetIds(spaceId, project.collectionIds); const assetIds = [...new Set([...project.assetIds, ...fromCollections])]; const [assetRows, segments, faceTracks, people, marks, kit, profileSet] = await Promise.all([ repos.listAssetRecords(spaceId, assetIds), repos.listSegments(spaceId, assetIds), repos.faceTrackRecords(spaceId, assetIds), repos.listPersonRecords(spaceId), repos.listMarks(spaceId), project.brandKitId ? repos.getBrandKit(spaceId, project.brandKitId) : Promise.resolve(null), deps.profiles(), ]); return { project, kit: kit?.kit ?? null, assets: new Map(assetRows.map((asset) => [asset.id, asset])), segments, faceTracks, people, markIds: new Set(marks.map((mark) => mark.id)), marks: marks.map((mark) => ({ id: mark.id, treatment: mark.treatment, assetId: mark.assetId, personId: mark.personId, })), profiles: new Map([...profileSet.profiles.entries()].map(([id, entry]) => [id, entry.profile])), }; } function kitArtifactIds(kit: BrandKit | null): string[] { if (!kit) return []; return [ ...kit.fonts.map((font) => font.artifactId), ...kit.logoArtifactIds, ...(kit.lutArtifactId ? [kit.lutArtifactId] : []), ...(kit.watermark?.artifactId ? [kit.watermark.artifactId] : []), ...kit.musicBeds.map((bed) => bed.artifactId), ]; } export function buildEdlContext( deps: Pick, ctx: PlanContext, profile: PlatformProfile, ): EdlContext { return { profile, brandKit: ctx.kit, assets: new Map( [...ctx.assets.values()].map((asset) => [ asset.id, { kind: asset.kind as "video" | "photo" | "live_photo", durationMs: asset.durationMs, width: asset.width, height: asset.height, }, ]), ), artifactIds: new Set(kitArtifactIds(ctx.kit)), music: new Map( (ctx.kit?.musicBeds ?? []).map((bed) => [ bed.artifactId, { commercialUse: bed.commercialUse }, ]), ), markIds: ctx.markIds, personIds: new Set(ctx.people.map((person) => person.id)), faceTrackIds: new Set(ctx.faceTracks.map((track) => track.id)), templateIds: new Set(deps.motion.templates().map((template) => template.id)), paidOnly: ctx.segments .filter((segment) => segment.paidOnly) .map((segment) => ({ assetId: segment.assetId, startMs: segment.startMs, endMs: segment.endMs, })), goal: ctx.project.goal, }; } function segmentFeatures(segment: MediaSegment): SegmentFeatures { const features: SegmentFeatures = { ...segment.scores }; if (features.speechHook === undefined && segment.transcriptExcerpt) { features.speechHook = speechHookScore(segment.transcriptExcerpt); } return features; } function scoredInputs(ctx: PlanContext): ScoredSegmentInput[] { return ctx.segments.map((segment) => ({ id: segment.id, assetId: segment.assetId, startMs: segment.startMs, endMs: segment.endMs, capturedAt: ctx.assets.get(segment.assetId)?.capturedAt ?? null, features: segmentFeatures(segment), paidOnly: segment.paidOnly, transcript: segment.transcriptExcerpt, })); } /** Scale features so the model's order counts as much as the deterministic score. */ export function applyRanking( segments: ScoredSegmentInput[], order: string[], ): ScoredSegmentInput[] { const rank = new Map(order.map((id, index) => [id, index])); return segments.map((segment) => { // Paid-only segments are never used; they only mark the payoff, so they keep their score. if (segment.paidOnly) return segment; const r = rank.get(segment.id); const factor = r === undefined ? 0.5 : 1 - r / (2 * order.length); const features = Object.fromEntries( Object.entries(segment.features).map(([feature, value]) => [feature, (value ?? 0) * factor]), ) as SegmentFeatures; return { ...segment, features }; }); } async function classifierFrames( deps: MediaServiceDeps, spaceId: string, assetIds: Set, ): Promise> { const analyses = await deps.repos.latestAnalysesByKind(spaceId, "classifier"); const out = new Map(); for (const [assetId, row] of analyses) { if (!assetIds.has(assetId)) continue; let data: unknown = row.data; if ((data === null || data === undefined) && row.dataPath) { try { data = JSON.parse( await readFile(await resolveInside(deps.mediaRoot, row.dataPath), "utf8"), ); } catch { continue; } } const frames = (data as { frames?: unknown } | null)?.frames; if (!Array.isArray(frames)) continue; out.set( assetId, frames.filter( (frame): frame is ClassifierFrames[number] => typeof frame?.tMs === "number" && typeof frame.scores === "object" && frame.scores !== null, ), ); } return out; } /** Fail closed: a segment without classifier frames in its range does not pass. */ function classifierPasses( frames: ClassifierFrames | undefined, segment: { startMs: number; endMs: number }, profile: PlatformProfile, ): boolean { const inRange = (frames ?? []).filter( (frame) => frame.tMs >= segment.startMs && frame.tMs <= segment.endMs, ); if (inRange.length === 0) return false; return inRange.every((frame) => Object.entries(profile.content.thresholds).every(([category, threshold]) => { const score = frame.scores[category as ContentCategory]; return score === undefined || threshold === undefined || score <= threshold; }), ); } function protectedPersonIds(ctx: PlanContext): Set { return new Set( ctx.people.filter((person) => person.anonymizeByDefault !== "none").map((p) => p.id), ); } /** * Whether a segment may show something the user protects: a face of a * protected person (or an unassigned face while anyone is protected), or a * treated identifying mark of its asset or of a person visible in it. Such * segments never go to the model, even with consent. */ export function showsProtectedDetail( ctx: Pick, protectedIds: Set, segment: { assetId: string; startMs: number; endMs: number }, ): boolean { const tracks = ctx.faceTracks.filter( (track) => track.assetId === segment.assetId && track.startMs < segment.endMs && track.endMs > segment.startMs, ); if ( protectedIds.size > 0 && tracks.some((track) => track.personId === null || protectedIds.has(track.personId)) ) { return true; } const visible = new Set(tracks.flatMap((track) => (track.personId ? [track.personId] : []))); return ctx.marks.some( (mark) => mark.treatment !== "none" && (mark.assetId === segment.assetId || (mark.personId !== null && visible.has(mark.personId))), ); } interface Judgement { order: string[]; reasons: Map; } async function judgeCandidates( deps: MediaServiceDeps, actor: MediaActor, ctx: PlanContext, profile: PlatformProfile, segments: ScoredSegmentInput[], weights: TeaserWeights, ): Promise { const top = segments .filter((segment) => !segment.paidOnly && segment.endMs > segment.startMs) .map((segment) => ({ segment, score: scoreSegment(segment.features, weights) })) .sort((a, b) => b.score - a.score || a.segment.id.localeCompare(b.segment.id)) .slice(0, JUDGE_TOP_K); if (top.length < 2) return null; const classifier = await classifierFrames( deps, actor.spaceId, new Set(top.map((c) => c.segment.assetId)), ); const protectedIds = protectedPersonIds(ctx); const eligible = top.filter( ({ segment }) => classifierPasses(classifier.get(segment.assetId), segment, profile) && !showsProtectedDetail(ctx, protectedIds, segment), ); if (eligible.length < 2) return null; const workRelative = path.join(actor.spaceId, "tmp", `judge-${crypto.randomUUID()}`); const workDir = await resolveInside(deps.mediaRoot, workRelative); await mkdir(workDir, { recursive: true }); try { const images: AgentInputImage[] = []; const sent: typeof eligible = []; for (const candidate of eligible) { const asset = ctx.assets.get(candidate.segment.assetId); const isVideo = asset?.kind === "video"; const source = isVideo ? asset?.proxyPath : (asset?.thumbPath ?? asset?.proxyPath); if (!source) continue; const outPath = path.join(workDir, `${images.length}.png`); try { await deps.renderer.extractFrame({ path: await resolveInside(deps.mediaRoot, source), tMs: isVideo ? Math.round((candidate.segment.startMs + candidate.segment.endMs) / 2) : 0, outPath, width: JUDGE_FRAME_WIDTH, }); images.push({ name: `${candidate.segment.id}.png`, mimeType: "image/png", data: new Uint8Array(await readFile(outPath)), }); sent.push(candidate); } catch (error) { getLogger().warn("media judge frame failed", { reason: error instanceof Error ? error.message : String(error), }); } } if (sent.length < 2) return null; const lines = sent.map(({ segment, score }, index) => { const transcript = (segment.transcript ?? "").replace(/\s+/g, " ").trim(); return [ `${index + 1}. id=${segment.id} image=${segment.id}.png`, `duration=${segment.endMs - segment.startMs}ms score=${score.toFixed(3)}`, transcript ? `transcript="${transcript.slice(0, TRANSCRIPT_EXCERPT_CHARS)}"` : "", ] .filter(Boolean) .join(" "); }); const reply = await deps.model.generate({ spaceId: actor.spaceId, userId: actor.userId, instructions: [ "You rank candidate segments for a short promotional video.", `Goal: ${ctx.project.goal}. Platform: ${profile.name}. The first ${profile.duration.hookWindowMs} ms must hook the viewer.`, "Judge only what the frames and transcripts show. Prefer clear, engaging openers that keep the payoff hidden.", 'Reply with one JSON object: {"ranking":[{"segmentId":"...","reason":"..."}]}, best first, using only the given ids.', ].join("\n"), prompt: `Candidates:\n${lines.join("\n")}`, images, schema: JudgeReplySchema, }); if (!reply) return null; const ids = new Set(sent.map(({ segment }) => segment.id)); const order: string[] = []; const reasons = new Map(); for (const entry of reply.ranking) { if (!ids.has(entry.segmentId) || reasons.has(entry.segmentId)) continue; order.push(entry.segmentId); reasons.set(entry.segmentId, entry.reason.replace(/\s+/g, " ").trim()); } return order.length > 0 ? { order, reasons } : null; } finally { await rm(workDir, { recursive: true, force: true }); } } function rationaleFor(input: { edl: Edl; profile: PlatformProfile; ctx: PlanContext; learned: boolean; judgement: Judgement | null; }): string { const { edl, profile, ctx } = input; const lines = [ `Automatic ${ctx.project.goal} plan for ${profile.name}.`, input.learned ? "Weights learned from this space's published results." : `Default ${ctx.project.goal} weights.`, ]; const first = [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs)[0]; if (first) { lines.push( `Hook: clip ${first.id}${first.segmentId ? ` (segment ${first.segmentId})` : ""} starts at ${introMs(edl) + first.atMs} ms, inside the ${profile.duration.hookWindowMs} ms hook window.`, ); } if (edl.intro) lines.push("Brand intro opens the video."); else if (ctx.kit?.intro) lines.push("Brand intro left out so the hook comes first."); const paidOnly = new Set(ctx.segments.filter((s) => s.paidOnly).map((s) => s.id)); if ( paidOnly.size > 0 && !edl.tracks.video.some((c) => c.segmentId && paidOnly.has(c.segmentId)) ) { lines.push("Paid-only segments excluded."); } if (input.judgement) { const reasons = input.judgement.order .slice(0, 3) .map((id) => `${id}: ${input.judgement?.reasons.get(id) ?? ""}`.trim()); lines.push( `Model ranking applied (${input.judgement.order.length} ranked): ${reasons.join("; ")}`, ); } return lines.join("\n").slice(0, RATIONALE_MAX_CHARS); } /** * Teasers end on a call to action that follows the profile's link policy * (spec 8). The tracking link itself goes into the caption or bio, never into * the picture: it only exists once the render is published. */ export function defaultCtaText(goal: MediaProjectGoal, profile: PlatformProfile): string | null { if (goal !== "teaser") return null; if (profile.text.linkPolicy === "allowed") return "Link below"; if (profile.text.linkPolicy === "bio_only") return "Link in bio"; return "Full video on my profile"; } function requireProfile(ctx: PlanContext, profileId: string): PlatformProfile { if (!ctx.project.profileIds.includes(profileId)) { throw new InvalidMediaInputError(`Profile ${profileId} is not part of this project`); } const profile = ctx.profiles.get(profileId); if (!profile) throw new InvalidMediaInputError(`Unknown platform profile ${profileId}`); return profile; } /** First cut per profile, validated and stored as a new plan version by the agent. */ export async function autoPlan( deps: MediaServiceDeps, actor: MediaActor, input: { projectId: string; profileIds?: string[] }, ): Promise { const ctx = await loadPlanContext(deps, actor, input.projectId); const profileIds = [...new Set(input.profileIds ?? ctx.project.profileIds)]; const profiles = profileIds.map((id) => requireProfile(ctx, id)); const model = await deps.repos.getTeaserModel(actor.spaceId); const learned = Boolean(model && model.sampleCount >= TEASER_LEARNING_MIN_SAMPLES); const weights = learned && model ? model.weights : defaultTeaserWeights(ctx.project.goal); const base = scoredInputs(ctx); const assets = new Map( [...ctx.assets.values()].map((asset) => [ asset.id, { durationMs: asset.durationMs, width: asset.width, height: asset.height, hasSpeech: ctx.segments.some( (segment) => segment.assetId === asset.id && Boolean(segment.transcriptExcerpt?.trim()), ), }, ]), ); const projectPeople = new Set( ctx.faceTracks.flatMap((track) => (track.personId ? [track.personId] : [])), ); const people = ctx.people .filter((person) => projectPeople.has(person.id)) .map((person) => ({ id: person.id, anonymizeByDefault: person.anonymizeByDefault })); const endCardCta = ctx.kit?.endCard?.params.cta; const plans: MediaEditPlan[] = []; for (const profile of profiles) { const judgement = RANKING_GOALS.has(ctx.project.goal) ? await judgeCandidates(deps, actor, ctx, profile, base, weights) : null; const segments = judgement ? applyRanking(base, judgement.order) : base; let edl: Edl; try { edl = autoPlanEdl({ goal: ctx.project.goal, profile, brandKit: ctx.kit, segments, weights, assets, people, marks: ctx.marks, ctaText: typeof endCardCta === "string" ? endCardCta : defaultCtaText(ctx.project.goal, profile), }); } catch (error) { throw new MediaStateConflictError( `No automatic plan for ${profile.id}: ${error instanceof Error ? error.message : String(error)}`, ); } const validation = validateEdl(edl, buildEdlContext(deps, ctx, profile)); if (!validation.ok) { throw new InvalidMediaInputError( `Automatic plan for ${profile.id} is invalid: ${validation.errors .map((error) => `${error.path} ${error.message}`) .join("; ")}`, ); } plans.push( await deps.repos.createPlanVersion(actor.spaceId, { projectId: ctx.project.id, profileId: profile.id, edl: validation.edl, createdBy: "agent", rationale: rationaleFor({ edl: validation.edl, profile, ctx, learned, judgement }), }), ); } return plans; } export async function submitPlan( deps: MediaServiceDeps, actor: MediaActor, input: { projectId: string; edl: unknown; rationale: string | null; createdBy: "agent" | "user" }, ): Promise<{ ok: true; plan: MediaEditPlan } | { ok: false; errors: EdlError[] }> { const ctx = await loadPlanContext(deps, actor, input.projectId); const profileId = (input.edl as { output?: { profileId?: unknown } } | null)?.output?.profileId; const profile = typeof profileId === "string" && ctx.project.profileIds.includes(profileId) ? ctx.profiles.get(profileId) : undefined; if (!profile) { return { ok: false, errors: [ { code: "profile_mismatch", path: "output.profileId", message: `plan must target one of the project's profiles: ${ctx.project.profileIds.join(", ")}`, }, ], }; } const validation = validateEdl(input.edl, buildEdlContext(deps, ctx, profile)); if (!validation.ok) return { ok: false, errors: validation.errors }; const plan = await deps.repos.createPlanVersion(actor.spaceId, { projectId: ctx.project.id, profileId: profile.id, edl: validation.edl, createdBy: input.createdBy, rationale: input.rationale, }); return { ok: true, plan }; }