import type { AiAlteration, AnonymizeMode, BrandKit, Edl, MediaProjectGoal, PlatformProfile, TeaserFeature, TeaserWeights, } from "@rakazo/contracts"; import { AnonymizeModeSchema, EdlSchema, TEASER_FEATURES, TEASER_LEARNING_MIN_SAMPLES, } from "@rakazo/contracts"; import { introFits } from "./marketing.js"; export type SegmentFeatures = Partial>; const GOAL_WEIGHTS: Record> = { teaser: { motion: 0.14, sharpness: 0.06, faces: 0.08, gaze: 0.08, audioPeak: 0.14, speechHook: 0.24, novelty: 0.12, beatProximity: 0.06, aesthetics: 0.08, }, trailer: { motion: 0.16, sharpness: 0.08, faces: 0.08, gaze: 0.06, audioPeak: 0.14, speechHook: 0.12, novelty: 0.12, beatProximity: 0.12, aesthetics: 0.12, }, highlight: { motion: 0.2, sharpness: 0.08, faces: 0.12, gaze: 0.06, audioPeak: 0.18, speechHook: 0.08, novelty: 0.1, beatProximity: 0.08, aesthetics: 0.1, }, story: { motion: 0.08, sharpness: 0.08, faces: 0.16, gaze: 0.12, audioPeak: 0.06, speechHook: 0.22, novelty: 0.12, beatProximity: 0.04, aesthetics: 0.12, }, full_release: { motion: 0.1, sharpness: 0.14, faces: 0.1, gaze: 0.08, audioPeak: 0.08, speechHook: 0.1, novelty: 0.12, beatProximity: 0.06, aesthetics: 0.22, }, custom: { motion: 0.12, sharpness: 0.1, faces: 0.1, gaze: 0.1, audioPeak: 0.12, speechHook: 0.14, novelty: 0.12, beatProximity: 0.08, aesthetics: 0.12, }, }; /** Default per-goal weights over every teaser feature; each set sums to 1. */ export function defaultTeaserWeights(goal: MediaProjectGoal): TeaserWeights { return { ...GOAL_WEIGHTS[goal] }; } export interface ScoredSegmentInput { id: string; assetId: string; startMs: number; endMs: number; capturedAt: Date | null; features: SegmentFeatures; paidOnly: boolean; transcript: string | null; } const clamp01 = (value: number) => (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0); /** Weighted mean of 0..1 features; missing features count as 0. */ export function scoreSegment(features: SegmentFeatures, weights: TeaserWeights): number { let total = 0; let norm = 0; for (const feature of TEASER_FEATURES) { const weight = weights[feature] ?? 0; norm += Math.abs(weight); total += weight * clamp01(features[feature] ?? 0); } return norm === 0 ? 0 : total / norm; } const HOOK_LEXICON = { en: { questionWords: /^(what|why|how|who|when|where|which|did|do|does|can|could|would|should|is|are|have|ever)\b/, announcements: /\b(wait|watch|look|listen|check this|you won'?t believe|here'?s|secret|how to|never|finally|the moment|let me show|guess|imagine|stop|nobody|everyone|biggest|best|worst|first time|new|today)\b/, address: /\b(you|your)\b/, numbers: /\b(one|two|three|five|ten|hundred|thousand|million|first|second|third|top)\b/, }, de: { questionWords: /^(was|warum|wieso|weshalb|wie|wer|wann|wo|welche[rsmn]?|hast|habt|kannst|könnt|willst|würdest|weißt|ist|sind)\b/, announcements: /\b(warte|wartet|schau|schaut|guck|hör zu|pass auf|achtung|du wirst nicht glauben|geheimnis|so geht|niemals|endlich|gleich|hier ist|zeig ich dir|stell dir vor|rate mal|niemand|alle|größte|beste|schlimmste|zum ersten mal|neu|heute)\b/, address: /\b(du|dich|dir|dein|deine|ihr|euch|euer)\b/, numbers: /\b(eins|zwei|drei|fünf|zehn|hundert|tausend|million|erste[rsn]?|zweite[rsn]?|top)\b/, }, } as const; /** 0..1 speech-hook strength from a transcript excerpt (questions, exclamations, numbers, announcements). */ export function speechHookScore(text: string, language?: string): number { const normalized = text.trim().toLocaleLowerCase(language); if (!normalized) return 0; const code = language?.slice(0, 2).toLowerCase(); const lexicons = code === "de" ? [HOOK_LEXICON.de] : code === "en" ? [HOOK_LEXICON.en] : Object.values(HOOK_LEXICON); let score = 0; if (normalized.includes("?")) score += 0.35; if (normalized.includes("!")) score += 0.25; const hasNumber = /\d/.test(normalized) || lexicons.some((lexicon) => lexicon.numbers.test(normalized)); if (hasNumber) score += 0.2; if (lexicons.some((lexicon) => lexicon.announcements.test(normalized))) score += 0.3; if (lexicons.some((lexicon) => lexicon.questionWords.test(normalized))) score += 0.15; if (lexicons.some((lexicon) => lexicon.address.test(normalized))) score += 0.1; return Math.min(1, Math.round(score * 1000) / 1000); } function segmentFeatures(segment: ScoredSegmentInput): SegmentFeatures { if (segment.features.speechHook !== undefined || !segment.transcript) return segment.features; return { ...segment.features, speechHook: speechHookScore(segment.transcript) }; } /** Strength of a segment as an opener: speech hooks, audio peaks, motion, eye contact. */ function hookStrength(features: SegmentFeatures): number { return ( 0.45 * clamp01(features.speechHook ?? 0) + 0.25 * clamp01(features.audioPeak ?? 0) + 0.15 * clamp01(features.motion ?? 0) + 0.15 * clamp01(features.gaze ?? 0) ); } export interface TeaserSelection { clips: Array<{ segmentId: string; assetId: string; inMs: number; outMs: number }>; hookSegmentId: string | null; /** Why each clip was chosen, for plan rationale. */ notes: string[]; } const MIN_CLIP_MS = 1000; /** * Pick clips for a goal and profile: a hook inside hookWindowMs, rising * tension, cut before the payoff (open loop), never paid-only segments, * total within the profile's duration bounds. * * The primary asset is the one holding the best segment overall; its payoff * is its best segment (paid-only included, since that is usually the payoff). * Nothing from the primary asset at or after the payoff is used. The hook * opens the teaser, the rest follows in ascending score, and the lead-in to * the payoff closes it, leaving `endCardMs` for the end card. */ export function selectTeaserSegments(input: { segments: ScoredSegmentInput[]; profile: PlatformProfile; goal: MediaProjectGoal; targetMs: number; weights: TeaserWeights; endCardMs: number; }): TeaserSelection { const { profile } = input; const notes: string[] = []; const scored = input.segments .filter((s) => s.endMs > s.startMs) .map((segment) => { const features = segmentFeatures(segment); return { segment, features, score: scoreSegment(features, input.weights) }; }) .sort( (a, b) => b.score - a.score || a.segment.assetId.localeCompare(b.segment.assetId) || a.segment.startMs - b.segment.startMs || a.segment.id.localeCompare(b.segment.id), ); const maxMs = Math.max( 0, Math.min(input.targetMs, profile.duration.hardMaxMs) - Math.max(0, input.endCardMs), ); const minMs = Math.max(0, profile.duration.hardMinMs - Math.max(0, input.endCardMs)); const usable = scored.filter((s) => !s.segment.paidOnly); if (usable.length === 0 || maxMs < MIN_CLIP_MS) { return { clips: [], hookSegmentId: null, notes: ["no usable segments"] }; } const primaryAsset = scored[0]!.segment.assetId; const payoff = scored.find((s) => s.segment.assetId === primaryAsset)!; const pool = usable.filter( (s) => s.segment.assetId !== primaryAsset || (s.segment.id !== payoff.segment.id && s.segment.endMs <= payoff.segment.startMs), ); const openLoop = pool.length < usable.length || payoff.segment.paidOnly; if (pool.length === 0) { return { clips: [], hookSegmentId: null, notes: ["only the payoff is usable"] }; } const hook = [...pool].sort( (a, b) => hookStrength(b.features) - hookStrength(a.features) || b.score - a.score || a.segment.id.localeCompare(b.segment.id), )[0]!; const hookWindow = profile.duration.hookWindowMs; const clipCap = Math.max(MIN_CLIP_MS, Math.min(5000, Math.round(maxMs / 4))); const lengthOf = (s: ScoredSegmentInput, cap: number) => Math.min(s.endMs - s.startMs, cap); // Lead-in: the last primary-asset segment before the payoff closes the teaser. const leadIn = openLoop ? pool .filter((s) => s.segment.assetId === primaryAsset && s !== hook) .sort((a, b) => b.segment.endMs - a.segment.endMs)[0] : undefined; const chosen: typeof pool = [hook]; let total = lengthOf(hook.segment, Math.max(clipCap, Math.min(hookWindow, maxMs))); const reserve = leadIn ? lengthOf(leadIn.segment, clipCap) : 0; for (const candidate of pool) { if (candidate === hook || candidate === leadIn) continue; const length = lengthOf(candidate.segment, clipCap); if (length < Math.min(MIN_CLIP_MS, candidate.segment.endMs - candidate.segment.startMs)) continue; if (total + length + reserve > maxMs) continue; chosen.push(candidate); total += length; } if (leadIn && total + reserve <= maxMs) { chosen.push(leadIn); total += reserve; } const middle = chosen .slice(1) .filter((s) => s !== leadIn) .sort((a, b) => a.score - b.score || a.segment.id.localeCompare(b.segment.id)); const ordered = [hook, ...middle, ...(chosen.includes(leadIn!) ? [leadIn!] : [])]; // Lengths: start from the cap, then grow toward the minimum duration if needed. const lengths = ordered.map((s, i) => lengthOf(s.segment, i === 0 ? Math.max(clipCap, Math.min(hookWindow, maxMs)) : clipCap), ); let sum = lengths.reduce((a, b) => a + b, 0); for (let i = 0; i < ordered.length && sum < minMs; i++) { const full = ordered[i]!.segment.endMs - ordered[i]!.segment.startMs; const grow = Math.min(full - lengths[i]!, minMs - sum, maxMs - sum); if (grow > 0) { lengths[i]! += grow; sum += grow; } } while (sum > maxMs && lengths.length > 0) { const over = sum - maxMs; const last = lengths.length - 1; if (lengths[last]! - over >= MIN_CLIP_MS) { lengths[last]! -= over; sum -= over; } else { sum -= lengths.pop()!; ordered.pop(); } } const clips = ordered.map((s, i) => { const length = lengths[i]!; // The lead-in ends where the payoff would begin; other clips open at their start. const isLeadIn = s === leadIn; const outMs = isLeadIn ? s.segment.endMs : s.segment.startMs + length; return { segmentId: s.segment.id, assetId: s.segment.assetId, inMs: outMs - length, outMs, }; }); notes.push( `hook: segment ${hook.segment.id} opens inside the ${hookWindow} ms hook window`, ...middle.map((s) => `build: segment ${s.segment.id} (score ${s.score.toFixed(2)})`), ); if (leadIn && ordered.includes(leadIn)) { notes.push(`open loop: segment ${leadIn.segment.id} cuts before payoff ${payoff.segment.id}`); } else if (openLoop) { notes.push(`open loop: payoff ${payoff.segment.id} is left out`); } if (input.segments.some((s) => s.paidOnly)) notes.push("paid-only segments excluded"); if (sum < minMs) notes.push(`only ${sum} ms of usable material; profile minimum is ${minMs} ms`); return { clips, hookSegmentId: hook.segment.id, notes }; } const LEARNING_RATE = 0.5; const LEARNING_ITERATIONS = 400; const L2 = 0.05; const SHRINKAGE = 50; /** * Regularized logistic regression of conversion on features; null below * TEASER_LEARNING_MIN_SAMPLES samples with views. * * Each sample contributes `conversions` successes out of `views` trials, * weighted so every publication counts equally. Full-batch gradient descent * from zero with a fixed iteration count keeps it deterministic. Positive * coefficients of features that vary in the data redistribute those features' * prior mass; features without variation keep their prior. The result is * shrunk toward the prior by n / (n + 50) so a small sample cannot overturn it. */ export function fitTeaserWeights( samples: Array<{ features: SegmentFeatures; views: number; conversions: number }>, prior: TeaserWeights, ): TeaserWeights | null { const usable = samples.filter((s) => s.views > 0 && s.conversions >= 0); if (usable.length < TEASER_LEARNING_MIN_SAMPLES) return null; const k = TEASER_FEATURES.length; const xs = usable.map((s) => TEASER_FEATURES.map((f) => clamp01(s.features[f] ?? 0))); const ys = usable.map((s) => Math.min(1, s.conversions / s.views)); const beta = new Array(k).fill(0); let intercept = 0; for (let iteration = 0; iteration < LEARNING_ITERATIONS; iteration++) { const gradient = new Array(k).fill(0); let gradientIntercept = 0; for (let i = 0; i < xs.length; i++) { const x = xs[i]!; let z = intercept; for (let j = 0; j < k; j++) z += beta[j]! * x[j]!; const error = 1 / (1 + Math.exp(-z)) - ys[i]!; gradientIntercept += error; for (let j = 0; j < k; j++) gradient[j]! += error * x[j]!; } intercept -= (LEARNING_RATE * gradientIntercept) / xs.length; for (let j = 0; j < k; j++) { beta[j]! -= LEARNING_RATE * (gradient[j]! / xs.length + L2 * beta[j]!); } } // Only features that vary in the data are re-weighted; they share their prior mass. const varies = TEASER_FEATURES.map((_, j) => xs.some((x) => x[j] !== xs[0]![j])); const mass = TEASER_FEATURES.reduce( (sum, f, j) => sum + (varies[j] ? Math.max(0, prior[f] ?? 0) : 0), 0, ); const positive = beta.map((b, j) => (varies[j] ? Math.max(0, b) : 0)); const positiveTotal = positive.reduce((a, b) => a + b, 0); const alpha = usable.length / (usable.length + SHRINKAGE); return Object.fromEntries( TEASER_FEATURES.map((feature, j) => { const base = prior[feature] ?? 0; const learned = varies[j] && positiveTotal > 0 ? (positive[j]! / positiveTotal) * mass : base; const value = (1 - alpha) * base + alpha * learned; return [feature, Math.round(value * 1e6) / 1e6]; }), ) as TeaserWeights; } const CTA_CARD_MS = 2500; function pickTarget(profile: PlatformProfile, goal: MediaProjectGoal, availableMs: number): number { const { hardMinMs, hardMaxMs, recommended } = profile.duration; const options = [...recommended] .filter((r) => r >= hardMinMs && r <= hardMaxMs) .sort((a, b) => a - b); let target: number; if (goal === "full_release" || goal === "custom") { target = Math.min(hardMaxMs, availableMs); } else if (options.length > 0) { target = options[Math.floor((options.length - 1) / 2)]!; } else { target = Math.min(hardMaxMs, 30_000); } return Math.max(hardMinMs, Math.min(hardMaxMs, target)); } /** * Deterministic first-cut EDL for a project and profile; the agent refines it. * * Teaser-style goals use selectTeaserSegments; full releases and custom * projects take segments in order (paid-only ones only for non-social * profiles). Reframing is "auto" whenever the source aspect differs from the * output, captions follow the profile when the material has speech, the * brand intro only plays where introFits allows it, and a teaser ends on the * brand end card (or a CTA text card). Person and mark defaults become * redactions, and the AI-alteration metadata follows from them. */ export function autoPlanEdl(input: { goal: MediaProjectGoal; profile: PlatformProfile; brandKit: BrandKit | null; segments: ScoredSegmentInput[]; weights: TeaserWeights; assets: Map< string, { durationMs: number | null; width: number | null; height: number | null; hasSpeech: boolean } >; /** Person anonymization defaults to carry into redactions. */ people: Array<{ id: string; anonymizeByDefault: string }>; marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; ctaText: string | null; }): Edl { const { goal, profile, brandKit: kit } = input; const W = profile.video.width; const H = profile.video.height; const segments = input.segments.filter((s) => { const asset = input.assets.get(s.assetId); return ( asset && s.endMs > s.startMs && (asset.durationMs === null || s.endMs <= asset.durationMs) ); }); const available = segments.reduce((sum, s) => sum + (s.endMs - s.startMs), 0); const targetMs = pickTarget(profile, goal, available); const intro = kit?.intro && introFits(profile, kit.intro.durationMs) ? kit.intro : undefined; let outro = goal === "teaser" ? undefined : kit?.outro; const ctaText = input.ctaText?.trim() || null; let ctaCard = false; if (goal === "teaser") { if (kit?.endCard) { outro = { ...kit.endCard, params: { ...kit.endCard.params, ...(ctaText ? { cta: ctaText.slice(0, 200) } : {}) }, }; } else if (ctaText) { ctaCard = true; } } const bookendsMs = (intro?.durationMs ?? 0) + (outro?.durationMs ?? 0); const budget = Math.max(MIN_CLIP_MS, targetMs - bookendsMs); let picks: Array<{ segmentId: string; assetId: string; inMs: number; outMs: number }>; if (goal === "full_release" || goal === "custom") { const allowPaid = profile.kind !== "free_social"; const ordered = segments .filter((s) => allowPaid || !s.paidOnly) .sort( (a, b) => (a.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) - (b.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) || a.assetId.localeCompare(b.assetId) || a.startMs - b.startMs, ); picks = []; let used = 0; for (const s of ordered) { const length = Math.min(s.endMs - s.startMs, budget - used); if (length < MIN_CLIP_MS) break; picks.push({ segmentId: s.id, assetId: s.assetId, inMs: s.startMs, outMs: s.startMs + length, }); used += length; } } else { picks = selectTeaserSegments({ segments, profile, goal, targetMs: budget, weights: input.weights, endCardMs: 0, }).clips; } if (picks.length === 0) throw new Error("no usable segments for this plan"); let atMs = 0; const lut = Boolean(kit?.lutArtifactId); const video = picks.map((pick, index) => { const asset = input.assets.get(pick.assetId)!; const sameAspect = !asset.width || !asset.height || Math.abs(asset.width / asset.height / (W / H) - 1) <= 0.01; const clip = { id: `c${index + 1}`, assetId: pick.assetId, inMs: pick.inMs, outMs: pick.outMs, atMs, speed: 1, reframe: sameAspect ? ("none" as const) : ("auto" as const), ...(lut ? { color: { useBrandLut: true } } : {}), segmentId: pick.segmentId, }; atMs += pick.outMs - pick.inMs; return clip; }); const mainMs = atMs; const introLength = intro?.durationMs ?? 0; const totalMs = introLength + mainMs + (outro?.durationMs ?? 0); const usedAssets = new Set(picks.map((p) => p.assetId)); const hasSpeech = [...usedAssets].some((id) => input.assets.get(id)?.hasSpeech); const preset = kit?.captionPresets[0]; const captions = (profile.captions.recommended || profile.captions.required) && hasSpeech ? { mode: profile.export.subtitles ? ("both" as const) : ("burned" as const), ...(preset ? { presetId: preset.id } : {}), maxCharsPerLine: W < H ? 24 : 32, wordHighlight: preset?.template === "karaoke", position: "lower" as const, } : undefined; const zones = profile.safeZones; const overlays: Array> = []; if (kit?.watermark) { overlays.push({ id: "watermark", kind: "watermark", atMs: 0, durationMs: totalMs }); } if (ctaCard && ctaText) { const cardMs = Math.min(CTA_CARD_MS, mainMs); const left = zones.leftPct / 100 + 0.02; const right = zones.rightPct / 100 + 0.02; overlays.push({ id: "cta", kind: "cta_end_card", atMs: introLength + mainMs - cardMs, durationMs: cardMs, text: ctaText.slice(0, 200), box: { x: left, y: 0.42, w: Math.max(0.1, 1 - left - right), h: 0.12 }, }); } const protectedPeople = input.people.flatMap((person) => { const mode = AnonymizeModeSchema.safeParse(person.anonymizeByDefault); return mode.success && mode.data !== "none" ? [{ id: person.id, mode: mode.data }] : []; }); const faces: Array<{ id: string; target: { personId: string } | { unassignedFaces: true }; mode: Exclude; }> = protectedPeople.map((person, index) => ({ id: `face-${index + 1}`, target: { personId: person.id }, mode: person.mode, })); if (faces.length > 0) { // Unknown faces next to a protected person are blurred too; the gate blocks them otherwise. faces.push({ id: "face-unassigned", target: { unassignedFaces: true }, mode: "blur" }); } const peopleIds = new Set(input.people.map((p) => p.id)); const marks = input.marks.flatMap((mark) => { if (mark.treatment === "none") return []; const relevant = (mark.assetId !== null && usedAssets.has(mark.assetId)) || (mark.personId !== null && peopleIds.has(mark.personId)) || (mark.assetId === null && mark.personId === null); if (!relevant) return []; const treatment = mark.treatment as "inpaint" | "blur" | "cover" | "reframe_out"; return [{ markId: mark.id, treatment }]; }); const aiAlterations: AiAlteration[] = []; if (faces.some((f) => f.mode === "synthetic_identity")) aiAlterations.push("synthetic_identity"); if (marks.some((m) => m.treatment === "inpaint")) aiAlterations.push("inpaint"); return EdlSchema.parse({ edlVersion: 1, output: { profileId: profile.id, width: W, height: H, fps: profile.video.fps, durationTargetMs: targetMs, durationHardMinMs: profile.duration.hardMinMs, durationHardMaxMs: profile.duration.hardMaxMs, }, tracks: { video, audio: [] }, transitions: [], ...(captions ? { captions } : {}), overlays, ...(intro ? { intro } : {}), ...(outro ? { outro } : {}), redactions: { faces, marks, regions: [] }, metadata: { tags: [], aiAltered: aiAlterations.length > 0, aiAlterations }, }); } const THUMBNAIL_WEIGHTS = { sharpness: 0.4, face: 0.25, thirds: 0.2, contrast: 0.15 }; /** * Best thumbnail frame times, spread out: candidates are ranked by sharpness, * face presence, rule of thirds and contrast, and each pick must be at least * a tenth of the candidate span (min 1 s) away from earlier picks while * enough candidates remain. */ export function rankThumbnailFrames( candidates: Array<{ tMs: number; sharpness: number; face: number; thirds: number; contrast: number; }>, count: number, ): number[] { if (count <= 0 || candidates.length === 0) return []; const ranked = candidates .map((c) => ({ tMs: c.tMs, score: THUMBNAIL_WEIGHTS.sharpness * clamp01(c.sharpness) + THUMBNAIL_WEIGHTS.face * clamp01(c.face) + THUMBNAIL_WEIGHTS.thirds * clamp01(c.thirds) + THUMBNAIL_WEIGHTS.contrast * clamp01(c.contrast), })) .sort((a, b) => b.score - a.score || a.tMs - b.tMs); const times = candidates.map((c) => c.tMs); const spacing = Math.max(1000, (Math.max(...times) - Math.min(...times)) / 10); const picked: number[] = []; for (const candidate of ranked) { if (picked.length >= count) break; if (picked.every((t) => Math.abs(t - candidate.tMs) >= spacing)) picked.push(candidate.tMs); } for (const candidate of ranked) { if (picked.length >= count) break; if (!picked.includes(candidate.tMs)) picked.push(candidate.tMs); } return picked; }