evOo-local/packages/adapters/src/media-marketing-plan.ts

579 lines
20 KiB
TypeScript

import { mkdir, readFile, rm } from "node:fs/promises";
import path from "node:path";
import type { AgentInputImage } from "@rakazo/adapter-kit";
import type {
BrandKit,
ContentCategory,
Edl,
EdlError,
MediaEditPlan,
MediaProjectGoal,
MediaSegment,
PlatformProfile,
TeaserWeights,
} from "@rakazo/contracts";
import { TEASER_LEARNING_MIN_SAMPLES } from "@rakazo/contracts";
import type { MediaProjectRecord, MediaRepos } from "@rakazo/db";
import { InvalidMediaInputError, MediaStateConflictError } from "@rakazo/db";
import { getLogger } from "@rakazo/logging";
import type { EdlContext, ScoredSegmentInput, SegmentFeatures } from "@rakazo/media";
import {
autoPlanEdl,
defaultTeaserWeights,
introMs,
scoreSegment,
speechHookScore,
validateEdl,
} from "@rakazo/media";
import { resolveInside } from "@rakazo/media/node";
import { z } from "zod";
import type { MediaActor, MediaServiceDeps } from "./media-context.js";
/*
* Edit plans for a project: the deterministic first cut per profile, an
* optional model ranking of the teaser candidates, and validation of plans the
* agent or user submits.
*
* Model judge (spec 8): only the top JUDGE_TOP_K candidates by deterministic
* score go to the model, and only those whose stored classifier scores already
* pass the target profile thresholds and that show no face of a protected
* person (or an unassigned face next to one). The model receives one still
* frame per candidate, taken from the proxy, plus the transcript excerpt.
* Face crops, embeddings and person data never leave the host. The model's
* order re-weights the candidates: rank r of n scales every feature by
* 1 - r / (2n); every usable segment it did not rank keeps half its weight
* (paid-only segments stay as they are: they only mark the payoff). Without a
* model, consent, or a usable reply the deterministic order stands.
*/
export const JUDGE_TOP_K = 12;
const JUDGE_FRAME_WIDTH = 320;
const TRANSCRIPT_EXCERPT_CHARS = 300;
const RATIONALE_MAX_CHARS = 2000;
const RANKING_GOALS: ReadonlySet<MediaProjectGoal> = new Set([
"teaser",
"trailer",
"highlight",
"story",
]);
const JudgeReplySchema = z.object({
ranking: z
.array(z.object({ segmentId: z.string().min(1).max(100), reason: z.string().max(300) }))
.max(50),
});
type AssetRow = Awaited<ReturnType<MediaRepos["listAssetRecords"]>>[number];
type FaceTrackRow = Awaited<ReturnType<MediaRepos["faceTrackRecords"]>>[number];
type PersonRow = Awaited<ReturnType<MediaRepos["listPersonRecords"]>>[number];
type ClassifierFrames = Array<{ tMs: number; scores: Partial<Record<ContentCategory, number>> }>;
export interface PlanContext {
project: MediaProjectRecord;
kit: BrandKit | null;
assets: Map<string, AssetRow>;
segments: MediaSegment[];
faceTracks: FaceTrackRow[];
/** Every person of the space; plans may reference any of them. */
people: PersonRow[];
markIds: Set<string>;
marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>;
profiles: Map<string, PlatformProfile>;
}
export async function loadPlanContext(
deps: MediaServiceDeps,
actor: MediaActor,
projectId: string,
): Promise<PlanContext> {
const { repos } = deps;
const { spaceId } = actor;
const project = await repos.getProjectRecord(spaceId, projectId);
const fromCollections = await repos.collectionAssetIds(spaceId, project.collectionIds);
const assetIds = [...new Set([...project.assetIds, ...fromCollections])];
const [assetRows, segments, faceTracks, people, marks, kit, profileSet] = await Promise.all([
repos.listAssetRecords(spaceId, assetIds),
repos.listSegments(spaceId, assetIds),
repos.faceTrackRecords(spaceId, assetIds),
repos.listPersonRecords(spaceId),
repos.listMarks(spaceId),
project.brandKitId ? repos.getBrandKit(spaceId, project.brandKitId) : Promise.resolve(null),
deps.profiles(),
]);
return {
project,
kit: kit?.kit ?? null,
assets: new Map(assetRows.map((asset) => [asset.id, asset])),
segments,
faceTracks,
people,
markIds: new Set(marks.map((mark) => mark.id)),
marks: marks.map((mark) => ({
id: mark.id,
treatment: mark.treatment,
assetId: mark.assetId,
personId: mark.personId,
})),
profiles: new Map([...profileSet.profiles.entries()].map(([id, entry]) => [id, entry.profile])),
};
}
function kitArtifactIds(kit: BrandKit | null): string[] {
if (!kit) return [];
return [
...kit.fonts.map((font) => font.artifactId),
...kit.logoArtifactIds,
...(kit.lutArtifactId ? [kit.lutArtifactId] : []),
...(kit.watermark?.artifactId ? [kit.watermark.artifactId] : []),
...kit.musicBeds.map((bed) => bed.artifactId),
];
}
export function buildEdlContext(
deps: Pick<MediaServiceDeps, "motion">,
ctx: PlanContext,
profile: PlatformProfile,
): EdlContext {
return {
profile,
brandKit: ctx.kit,
assets: new Map(
[...ctx.assets.values()].map((asset) => [
asset.id,
{
kind: asset.kind as "video" | "photo" | "live_photo",
durationMs: asset.durationMs,
width: asset.width,
height: asset.height,
},
]),
),
artifactIds: new Set(kitArtifactIds(ctx.kit)),
music: new Map(
(ctx.kit?.musicBeds ?? []).map((bed) => [
bed.artifactId,
{ commercialUse: bed.commercialUse },
]),
),
markIds: ctx.markIds,
personIds: new Set(ctx.people.map((person) => person.id)),
faceTrackIds: new Set(ctx.faceTracks.map((track) => track.id)),
templateIds: new Set(deps.motion.templates().map((template) => template.id)),
paidOnly: ctx.segments
.filter((segment) => segment.paidOnly)
.map((segment) => ({
assetId: segment.assetId,
startMs: segment.startMs,
endMs: segment.endMs,
})),
goal: ctx.project.goal,
};
}
function segmentFeatures(segment: MediaSegment): SegmentFeatures {
const features: SegmentFeatures = { ...segment.scores };
if (features.speechHook === undefined && segment.transcriptExcerpt) {
features.speechHook = speechHookScore(segment.transcriptExcerpt);
}
return features;
}
function scoredInputs(ctx: PlanContext): ScoredSegmentInput[] {
return ctx.segments.map((segment) => ({
id: segment.id,
assetId: segment.assetId,
startMs: segment.startMs,
endMs: segment.endMs,
capturedAt: ctx.assets.get(segment.assetId)?.capturedAt ?? null,
features: segmentFeatures(segment),
paidOnly: segment.paidOnly,
transcript: segment.transcriptExcerpt,
}));
}
/** Scale features so the model's order counts as much as the deterministic score. */
export function applyRanking(
segments: ScoredSegmentInput[],
order: string[],
): ScoredSegmentInput[] {
const rank = new Map(order.map((id, index) => [id, index]));
return segments.map((segment) => {
// Paid-only segments are never used; they only mark the payoff, so they keep their score.
if (segment.paidOnly) return segment;
const r = rank.get(segment.id);
const factor = r === undefined ? 0.5 : 1 - r / (2 * order.length);
const features = Object.fromEntries(
Object.entries(segment.features).map(([feature, value]) => [feature, (value ?? 0) * factor]),
) as SegmentFeatures;
return { ...segment, features };
});
}
async function classifierFrames(
deps: MediaServiceDeps,
spaceId: string,
assetIds: Set<string>,
): Promise<Map<string, ClassifierFrames>> {
const analyses = await deps.repos.latestAnalysesByKind(spaceId, "classifier");
const out = new Map<string, ClassifierFrames>();
for (const [assetId, row] of analyses) {
if (!assetIds.has(assetId)) continue;
let data: unknown = row.data;
if ((data === null || data === undefined) && row.dataPath) {
try {
data = JSON.parse(
await readFile(await resolveInside(deps.mediaRoot, row.dataPath), "utf8"),
);
} catch {
continue;
}
}
const frames = (data as { frames?: unknown } | null)?.frames;
if (!Array.isArray(frames)) continue;
out.set(
assetId,
frames.filter(
(frame): frame is ClassifierFrames[number] =>
typeof frame?.tMs === "number" &&
typeof frame.scores === "object" &&
frame.scores !== null,
),
);
}
return out;
}
/** Fail closed: a segment without classifier frames in its range does not pass. */
function classifierPasses(
frames: ClassifierFrames | undefined,
segment: { startMs: number; endMs: number },
profile: PlatformProfile,
): boolean {
const inRange = (frames ?? []).filter(
(frame) => frame.tMs >= segment.startMs && frame.tMs <= segment.endMs,
);
if (inRange.length === 0) return false;
return inRange.every((frame) =>
Object.entries(profile.content.thresholds).every(([category, threshold]) => {
const score = frame.scores[category as ContentCategory];
return score === undefined || threshold === undefined || score <= threshold;
}),
);
}
function protectedPersonIds(ctx: PlanContext): Set<string> {
return new Set(
ctx.people.filter((person) => person.anonymizeByDefault !== "none").map((p) => p.id),
);
}
/**
* Whether a segment may show something the user protects: a face of a
* protected person (or an unassigned face while anyone is protected), or a
* treated identifying mark of its asset or of a person visible in it. Such
* segments never go to the model, even with consent.
*/
export function showsProtectedDetail(
ctx: Pick<PlanContext, "faceTracks" | "marks">,
protectedIds: Set<string>,
segment: { assetId: string; startMs: number; endMs: number },
): boolean {
const tracks = ctx.faceTracks.filter(
(track) =>
track.assetId === segment.assetId &&
track.startMs < segment.endMs &&
track.endMs > segment.startMs,
);
if (
protectedIds.size > 0 &&
tracks.some((track) => track.personId === null || protectedIds.has(track.personId))
) {
return true;
}
const visible = new Set(tracks.flatMap((track) => (track.personId ? [track.personId] : [])));
return ctx.marks.some(
(mark) =>
mark.treatment !== "none" &&
(mark.assetId === segment.assetId || (mark.personId !== null && visible.has(mark.personId))),
);
}
interface Judgement {
order: string[];
reasons: Map<string, string>;
}
async function judgeCandidates(
deps: MediaServiceDeps,
actor: MediaActor,
ctx: PlanContext,
profile: PlatformProfile,
segments: ScoredSegmentInput[],
weights: TeaserWeights,
): Promise<Judgement | null> {
const top = segments
.filter((segment) => !segment.paidOnly && segment.endMs > segment.startMs)
.map((segment) => ({ segment, score: scoreSegment(segment.features, weights) }))
.sort((a, b) => b.score - a.score || a.segment.id.localeCompare(b.segment.id))
.slice(0, JUDGE_TOP_K);
if (top.length < 2) return null;
const classifier = await classifierFrames(
deps,
actor.spaceId,
new Set(top.map((c) => c.segment.assetId)),
);
const protectedIds = protectedPersonIds(ctx);
const eligible = top.filter(
({ segment }) =>
classifierPasses(classifier.get(segment.assetId), segment, profile) &&
!showsProtectedDetail(ctx, protectedIds, segment),
);
if (eligible.length < 2) return null;
const workRelative = path.join(actor.spaceId, "tmp", `judge-${crypto.randomUUID()}`);
const workDir = await resolveInside(deps.mediaRoot, workRelative);
await mkdir(workDir, { recursive: true });
try {
const images: AgentInputImage[] = [];
const sent: typeof eligible = [];
for (const candidate of eligible) {
const asset = ctx.assets.get(candidate.segment.assetId);
const isVideo = asset?.kind === "video";
const source = isVideo ? asset?.proxyPath : (asset?.thumbPath ?? asset?.proxyPath);
if (!source) continue;
const outPath = path.join(workDir, `${images.length}.png`);
try {
await deps.renderer.extractFrame({
path: await resolveInside(deps.mediaRoot, source),
tMs: isVideo ? Math.round((candidate.segment.startMs + candidate.segment.endMs) / 2) : 0,
outPath,
width: JUDGE_FRAME_WIDTH,
});
images.push({
name: `${candidate.segment.id}.png`,
mimeType: "image/png",
data: new Uint8Array(await readFile(outPath)),
});
sent.push(candidate);
} catch (error) {
getLogger().warn("media judge frame failed", {
reason: error instanceof Error ? error.message : String(error),
});
}
}
if (sent.length < 2) return null;
const lines = sent.map(({ segment, score }, index) => {
const transcript = (segment.transcript ?? "").replace(/\s+/g, " ").trim();
return [
`${index + 1}. id=${segment.id} image=${segment.id}.png`,
`duration=${segment.endMs - segment.startMs}ms score=${score.toFixed(3)}`,
transcript ? `transcript="${transcript.slice(0, TRANSCRIPT_EXCERPT_CHARS)}"` : "",
]
.filter(Boolean)
.join(" ");
});
const reply = await deps.model.generate({
spaceId: actor.spaceId,
userId: actor.userId,
instructions: [
"You rank candidate segments for a short promotional video.",
`Goal: ${ctx.project.goal}. Platform: ${profile.name}. The first ${profile.duration.hookWindowMs} ms must hook the viewer.`,
"Judge only what the frames and transcripts show. Prefer clear, engaging openers that keep the payoff hidden.",
'Reply with one JSON object: {"ranking":[{"segmentId":"...","reason":"..."}]}, best first, using only the given ids.',
].join("\n"),
prompt: `Candidates:\n${lines.join("\n")}`,
images,
schema: JudgeReplySchema,
});
if (!reply) return null;
const ids = new Set(sent.map(({ segment }) => segment.id));
const order: string[] = [];
const reasons = new Map<string, string>();
for (const entry of reply.ranking) {
if (!ids.has(entry.segmentId) || reasons.has(entry.segmentId)) continue;
order.push(entry.segmentId);
reasons.set(entry.segmentId, entry.reason.replace(/\s+/g, " ").trim());
}
return order.length > 0 ? { order, reasons } : null;
} finally {
await rm(workDir, { recursive: true, force: true });
}
}
function rationaleFor(input: {
edl: Edl;
profile: PlatformProfile;
ctx: PlanContext;
learned: boolean;
judgement: Judgement | null;
}): string {
const { edl, profile, ctx } = input;
const lines = [
`Automatic ${ctx.project.goal} plan for ${profile.name}.`,
input.learned
? "Weights learned from this space's published results."
: `Default ${ctx.project.goal} weights.`,
];
const first = [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs)[0];
if (first) {
lines.push(
`Hook: clip ${first.id}${first.segmentId ? ` (segment ${first.segmentId})` : ""} starts at ${introMs(edl) + first.atMs} ms, inside the ${profile.duration.hookWindowMs} ms hook window.`,
);
}
if (edl.intro) lines.push("Brand intro opens the video.");
else if (ctx.kit?.intro) lines.push("Brand intro left out so the hook comes first.");
const paidOnly = new Set(ctx.segments.filter((s) => s.paidOnly).map((s) => s.id));
if (
paidOnly.size > 0 &&
!edl.tracks.video.some((c) => c.segmentId && paidOnly.has(c.segmentId))
) {
lines.push("Paid-only segments excluded.");
}
if (input.judgement) {
const reasons = input.judgement.order
.slice(0, 3)
.map((id) => `${id}: ${input.judgement?.reasons.get(id) ?? ""}`.trim());
lines.push(
`Model ranking applied (${input.judgement.order.length} ranked): ${reasons.join("; ")}`,
);
}
return lines.join("\n").slice(0, RATIONALE_MAX_CHARS);
}
/**
* Teasers end on a call to action that follows the profile's link policy
* (spec 8). The tracking link itself goes into the caption or bio, never into
* the picture: it only exists once the render is published.
*/
export function defaultCtaText(goal: MediaProjectGoal, profile: PlatformProfile): string | null {
if (goal !== "teaser") return null;
if (profile.text.linkPolicy === "allowed") return "Link below";
if (profile.text.linkPolicy === "bio_only") return "Link in bio";
return "Full video on my profile";
}
function requireProfile(ctx: PlanContext, profileId: string): PlatformProfile {
if (!ctx.project.profileIds.includes(profileId)) {
throw new InvalidMediaInputError(`Profile ${profileId} is not part of this project`);
}
const profile = ctx.profiles.get(profileId);
if (!profile) throw new InvalidMediaInputError(`Unknown platform profile ${profileId}`);
return profile;
}
/** First cut per profile, validated and stored as a new plan version by the agent. */
export async function autoPlan(
deps: MediaServiceDeps,
actor: MediaActor,
input: { projectId: string; profileIds?: string[] },
): Promise<MediaEditPlan[]> {
const ctx = await loadPlanContext(deps, actor, input.projectId);
const profileIds = [...new Set(input.profileIds ?? ctx.project.profileIds)];
const profiles = profileIds.map((id) => requireProfile(ctx, id));
const model = await deps.repos.getTeaserModel(actor.spaceId);
const learned = Boolean(model && model.sampleCount >= TEASER_LEARNING_MIN_SAMPLES);
const weights = learned && model ? model.weights : defaultTeaserWeights(ctx.project.goal);
const base = scoredInputs(ctx);
const assets = new Map(
[...ctx.assets.values()].map((asset) => [
asset.id,
{
durationMs: asset.durationMs,
width: asset.width,
height: asset.height,
hasSpeech: ctx.segments.some(
(segment) => segment.assetId === asset.id && Boolean(segment.transcriptExcerpt?.trim()),
),
},
]),
);
const projectPeople = new Set(
ctx.faceTracks.flatMap((track) => (track.personId ? [track.personId] : [])),
);
const people = ctx.people
.filter((person) => projectPeople.has(person.id))
.map((person) => ({ id: person.id, anonymizeByDefault: person.anonymizeByDefault }));
const endCardCta = ctx.kit?.endCard?.params.cta;
const plans: MediaEditPlan[] = [];
for (const profile of profiles) {
const judgement = RANKING_GOALS.has(ctx.project.goal)
? await judgeCandidates(deps, actor, ctx, profile, base, weights)
: null;
const segments = judgement ? applyRanking(base, judgement.order) : base;
let edl: Edl;
try {
edl = autoPlanEdl({
goal: ctx.project.goal,
profile,
brandKit: ctx.kit,
segments,
weights,
assets,
people,
marks: ctx.marks,
ctaText:
typeof endCardCta === "string" ? endCardCta : defaultCtaText(ctx.project.goal, profile),
});
} catch (error) {
throw new MediaStateConflictError(
`No automatic plan for ${profile.id}: ${error instanceof Error ? error.message : String(error)}`,
);
}
const validation = validateEdl(edl, buildEdlContext(deps, ctx, profile));
if (!validation.ok) {
throw new InvalidMediaInputError(
`Automatic plan for ${profile.id} is invalid: ${validation.errors
.map((error) => `${error.path} ${error.message}`)
.join("; ")}`,
);
}
plans.push(
await deps.repos.createPlanVersion(actor.spaceId, {
projectId: ctx.project.id,
profileId: profile.id,
edl: validation.edl,
createdBy: "agent",
rationale: rationaleFor({ edl: validation.edl, profile, ctx, learned, judgement }),
}),
);
}
return plans;
}
export async function submitPlan(
deps: MediaServiceDeps,
actor: MediaActor,
input: { projectId: string; edl: unknown; rationale: string | null; createdBy: "agent" | "user" },
): Promise<{ ok: true; plan: MediaEditPlan } | { ok: false; errors: EdlError[] }> {
const ctx = await loadPlanContext(deps, actor, input.projectId);
const profileId = (input.edl as { output?: { profileId?: unknown } } | null)?.output?.profileId;
const profile =
typeof profileId === "string" && ctx.project.profileIds.includes(profileId)
? ctx.profiles.get(profileId)
: undefined;
if (!profile) {
return {
ok: false,
errors: [
{
code: "profile_mismatch",
path: "output.profileId",
message: `plan must target one of the project's profiles: ${ctx.project.profileIds.join(", ")}`,
},
],
};
}
const validation = validateEdl(input.edl, buildEdlContext(deps, ctx, profile));
if (!validation.ok) return { ok: false, errors: validation.errors };
const plan = await deps.repos.createPlanVersion(actor.spaceId, {
projectId: ctx.project.id,
profileId: profile.id,
edl: validation.edl,
createdBy: input.createdBy,
rationale: input.rationale,
});
return { ok: true, plan };
}