From 3f9a601dba3d84af1965fe785217ff01caed48a8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:24:53 +0000 Subject: [PATCH] wip(media): w2-marketing partial work (handover, unverified) Stopped mid-implementation for the handover to the project thread. Not typechecked or tested; do not merge as is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj --- packages/adapters/src/media-marketing-copy.ts | 164 ++++++ .../adapters/src/media-marketing-files.ts | 124 ++++ packages/adapters/src/media-marketing-plan.ts | 532 ++++++++++++++++++ .../src/media-marketing-thumbnails.ts | 350 ++++++++++++ packages/adapters/src/media-metrics.ts | 133 +++++ packages/adapters/src/media-publish.ts | 357 ++++++++++++ packages/adapters/src/media-zip.ts | 143 +++++ 7 files changed, 1803 insertions(+) create mode 100644 packages/adapters/src/media-marketing-copy.ts create mode 100644 packages/adapters/src/media-marketing-files.ts create mode 100644 packages/adapters/src/media-marketing-plan.ts create mode 100644 packages/adapters/src/media-marketing-thumbnails.ts create mode 100644 packages/adapters/src/media-metrics.ts create mode 100644 packages/adapters/src/media-publish.ts create mode 100644 packages/adapters/src/media-zip.ts diff --git a/packages/adapters/src/media-marketing-copy.ts b/packages/adapters/src/media-marketing-copy.ts new file mode 100644 index 0000000..f8ed067 --- /dev/null +++ b/packages/adapters/src/media-marketing-copy.ts @@ -0,0 +1,164 @@ +import type { Edl, MarketingCopy, MediaSegment, PlatformProfile } from "@rakazo/contracts"; +import { MarketingCopySchema } from "@rakazo/contracts"; +import { InvalidMediaInputError, MediaStateConflictError } from "@rakazo/db"; +import { checkCopy, edlDurationMs, isShortForm, normalizeHashtags } from "@rakazo/media"; +import { z } from "zod"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * Platform copy for a render (spec 10.2). The model writes from the plan's + * metadata and the transcript of the clips actually used, under the + * profile's limits; it is told to state nothing the material does not show. + * Without a model (or consent) the copy comes from the plan metadata and the + * project title only. Either way hashtags are normalized and capped, long + * fields are cut at a word boundary to the profile limits, and the result is + * checked like the gate's text check; findings are returned as `issues`. + */ + +const TRANSCRIPT_CHARS = 1500; +const CHAPTER_MIN_PROGRAM_MS = 60_000; + +const CopyReplySchema = z.object({ + title: z.string().max(200), + hook: z.string().max(200), + caption: z.string().max(5000), + hashtags: z.array(z.string().max(60)).max(50), + description: z.string().max(5000).optional(), + chapters: z + .array(z.object({ atMs: z.number().int().min(0), title: z.string().max(100) })) + .max(100) + .optional(), + altText: z.string().max(500).optional(), +}); +type CopyReply = z.infer; + +const LANGUAGE = /^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})?$/; + +/** Cut to `max` characters, at the last word boundary when there is one. */ +export function truncateText(value: string, max: number): string { + const chars = [...value.trim()]; + if (chars.length <= max) return value.trim(); + const cut = chars.slice(0, Math.max(0, max)).join(""); + const space = cut.lastIndexOf(" "); + return (space > max * 0.6 ? cut.slice(0, space) : cut).trim(); +} + +function transcriptOf(edl: Edl, segments: MediaSegment[]): string { + const byId = new Map(segments.map((segment) => [segment.id, segment])); + const parts: string[] = []; + for (const clip of [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs)) { + const linked = clip.segmentId ? byId.get(clip.segmentId) : undefined; + const overlapping = linked + ? [linked] + : segments.filter( + (s) => s.assetId === clip.assetId && s.startMs < clip.outMs && s.endMs > clip.inMs, + ); + for (const segment of overlapping) { + const text = segment.transcriptExcerpt?.replace(/\s+/g, " ").trim(); + if (text && !parts.includes(text)) parts.push(text); + } + } + return parts.join(" ").slice(0, TRANSCRIPT_CHARS); +} + +function wantsChapters(profile: PlatformProfile, edl: Edl): boolean { + return !isShortForm(profile) && edlDurationMs(edl) >= CHAPTER_MIN_PROGRAM_MS; +} + +function fallbackCopy(edl: Edl, projectTitle: string, transcript: string): CopyReply { + const title = edl.metadata.title?.trim() || projectTitle; + const sentence = transcript.split(/(?<=[.!?])\s+/)[0]?.trim() ?? ""; + return { + title, + hook: sentence && sentence.length <= 120 ? sentence : title, + caption: edl.metadata.description?.trim() || title, + hashtags: edl.metadata.tags, + ...(edl.metadata.description ? { description: edl.metadata.description } : {}), + }; +} + +function instructions(profile: PlatformProfile, language: string, chapters: boolean): string { + const { text } = profile; + const link = + text.linkPolicy === "allowed" + ? "Links are allowed." + : text.linkPolicy === "bio_only" + ? 'Never put a link in the text; say "link in bio" if a call to action needs one.' + : "Never mention or include links."; + return [ + `You write social video copy in the language "${language}" for ${profile.name}.`, + "Use only facts from the given title, description, tags and transcript. Never invent claims, prices, results, quotes, guarantees or urgency.", + "No clickbait that misrepresents the video, no engagement bait, no promises the video does not keep.", + `Limits: title ${text.titleMaxChars ?? 100} characters, hook 120 characters, caption ${text.captionMaxChars} characters, at most ${text.hashtagLimit} hashtags without the # sign.`, + link, + chapters + ? "Add chapters (atMs from 0, at least 10 s apart) only where the transcript shows topic changes." + : "Do not add chapters.", + "altText describes what is visible for screen readers, factually.", + 'Reply with one JSON object: {"title","hook","caption","hashtags":[],"description","chapters":[{"atMs","title"}],"altText"}.', + ].join("\n"); +} + +export async function generateCopy( + deps: MediaServiceDeps, + actor: MediaActor, + input: { renderId: string; language: string }, +): Promise { + const language = input.language.trim(); + if (!LANGUAGE.test(language)) throw new InvalidMediaInputError("Invalid language code"); + const { spaceId } = actor; + const render = await deps.repos.getRender(spaceId, input.renderId); + const [plan, profileSet, project] = await Promise.all([ + deps.repos.getPlan(spaceId, render.planId), + deps.profiles(), + deps.repos.getProjectRecord(spaceId, render.projectId), + ]); + const profile = profileSet.profiles.get(render.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${render.profileId} is not available`); + const edl = plan.edl; + const segments = await deps.repos.listSegments(spaceId, [ + ...new Set(edl.tracks.video.map((clip) => clip.assetId)), + ]); + const transcript = transcriptOf(edl, segments); + const chapters = wantsChapters(profile, edl); + const durationMs = edlDurationMs(edl); + + const reply = await deps.model.generate({ + spaceId, + userId: actor.userId, + instructions: instructions(profile, language, chapters), + prompt: JSON.stringify({ + projectTitle: project.title, + title: edl.metadata.title ?? null, + description: edl.metadata.description ?? null, + tags: edl.metadata.tags, + durationMs, + transcript: transcript || null, + }), + schema: CopyReplySchema, + }); + const draft = reply ?? fallbackCopy(edl, project.title, transcript); + + const title = truncateText(draft.title, Math.min(200, profile.text.titleMaxChars ?? 200)); + const caption = truncateText(draft.caption, Math.min(5000, profile.text.captionMaxChars)); + const hashtags = normalizeHashtags(draft.hashtags, profile.text.hashtagLimit).map((tag) => + tag.slice(0, 60), + ); + const chapterList = chapters + ? (draft.chapters ?? []) + .filter((chapter) => chapter.atMs < durationMs && chapter.title.trim()) + .sort((a, b) => a.atMs - b.atMs) + : []; + return MarketingCopySchema.parse({ + profileId: profile.id, + language, + title, + hook: truncateText(draft.hook, 200), + caption, + hashtags, + ...(draft.description ? { description: truncateText(draft.description, 5000) } : {}), + ...(chapterList.length > 0 ? { chapters: chapterList } : {}), + ...(draft.altText?.trim() ? { altText: truncateText(draft.altText, 500) } : {}), + issues: checkCopy(profile, { title, caption, tags: hashtags }), + }); +} diff --git a/packages/adapters/src/media-marketing-files.ts b/packages/adapters/src/media-marketing-files.ts new file mode 100644 index 0000000..534c9c2 --- /dev/null +++ b/packages/adapters/src/media-marketing-files.ts @@ -0,0 +1,124 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { AdapterContext } from "@rakazo/adapter-kit"; +import type { MediaRenderOutput } from "@rakazo/contracts"; +import { MediaRenderOutputSchema } from "@rakazo/contracts"; +import { IsolationError } from "@rakazo/db"; +import { resolveInside } from "@rakazo/media/node"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * File and artifact plumbing shared by the marketing services. Media artifacts + * (thumbnails, intro posters, small bundles) are space artifacts without a bot, + * owned by the acting user, like chat attachments. + */ + +/** A stored render output: the public fields plus its path relative to the media root. */ +export type StoredRenderOutput = MediaRenderOutput & { path?: string }; + +export function mediaAdapterContext(actor: MediaActor, operationId: string): AdapterContext { + return { operationId, traceId: operationId, spaceId: actor.spaceId, userId: actor.userId }; +} + +export async function storeMediaArtifact( + deps: Pick, + actor: MediaActor, + input: { name: string; mimeType: string; bytes: Uint8Array }, +): Promise { + const context = mediaAdapterContext(actor, `media-artifact:${input.name}`); + const stored = await deps.artifacts.put(input, context); + const row = await deps.prisma.artifact + .create({ + data: { + spaceId: actor.spaceId, + userId: actor.userId, + name: input.name, + mimeType: input.mimeType, + size: input.bytes.byteLength, + hash: createHash("sha256").update(input.bytes).digest("hex"), + storageKey: stored.id, + }, + select: { id: true }, + }) + .catch(async (error) => { + await deps.artifacts.remove(stored.id, context).catch(() => undefined); + throw error; + }); + return row.id; +} + +/** + * Copy a space artifact (brand font, logo) into `dir` so a local renderer can + * read it. The file name is the artifact id plus its sanitized extension. + */ +export async function materializeArtifact( + deps: Pick, + actor: MediaActor, + artifactId: string, + dir: string, +): Promise { + const row = await deps.prisma.artifact.findFirst({ + where: { id: artifactId, spaceId: actor.spaceId }, + select: { id: true, name: true, storageKey: true }, + }); + if (!row) throw new IsolationError(); + const bytes = await deps.artifacts.get( + row.storageKey, + mediaAdapterContext(actor, `media-artifact-get:${row.id}`), + ); + const ext = path.extname(row.name).toLowerCase(); + const safeId = row.id.replace(/[^A-Za-z0-9_-]/g, ""); + const file = path.join(dir, `${safeId}${/^\.[a-z0-9]{1,5}$/.test(ext) ? ext : ""}`); + await mkdir(dir, { recursive: true }); + await writeFile(file, bytes); + return file; +} + +/** Outputs of a render row, dropping anything that does not parse. */ +export function storedRenderOutputs(outputs: unknown): StoredRenderOutput[] { + if (!Array.isArray(outputs)) return []; + return outputs.flatMap((raw) => { + const parsed = MediaRenderOutputSchema.safeParse(raw); + if (!parsed.success) return []; + const internal = (raw as { path?: unknown }).path; + return [{ ...parsed.data, ...(typeof internal === "string" ? { path: internal } : {}) }]; + }); +} + +export function publicRenderOutput(output: StoredRenderOutput): MediaRenderOutput { + return { + kind: output.kind, + artifactId: output.artifactId, + fileName: output.fileName, + sizeBytes: output.sizeBytes, + }; +} + +/** + * Absolute path of a render output: its stored relative path, or + * `/renders//` when the renderer stored none. + */ +export function renderOutputPath( + deps: Pick, + spaceId: string, + renderId: string, + output: StoredRenderOutput, +): Promise { + return resolveInside( + deps.mediaRoot, + output.path ?? path.join(spaceId, "renders", renderId, path.basename(output.fileName)), + ); +} + +/** Lowercase ASCII slug for UTM values and file names; empty when nothing is left. */ +export function slugify(value: string, max = 100): string { + return value + .normalize("NFKD") + .replace(/[̀-ͯ]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, max) + .replace(/-+$/g, ""); +} diff --git a/packages/adapters/src/media-marketing-plan.ts b/packages/adapters/src/media-marketing-plan.ts new file mode 100644 index 0000000..ff9377e --- /dev/null +++ b/packages/adapters/src/media-marketing-plan.ts @@ -0,0 +1,532 @@ +import { mkdir, readFile, rm } from "node:fs/promises"; +import path from "node:path"; +import type { AgentInputImage } from "@rakazo/adapter-kit"; +import type { + BrandKit, + ContentCategory, + Edl, + EdlError, + MediaEditPlan, + MediaProjectGoal, + MediaSegment, + PlatformProfile, + TeaserWeights, +} from "@rakazo/contracts"; +import { TEASER_LEARNING_MIN_SAMPLES } from "@rakazo/contracts"; +import type { MediaProjectRecord, MediaRepos } from "@rakazo/db"; +import { InvalidMediaInputError } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import type { EdlContext, ScoredSegmentInput, SegmentFeatures } from "@rakazo/media"; +import { + autoPlanEdl, + defaultTeaserWeights, + introMs, + scoreSegment, + speechHookScore, + validateEdl, +} from "@rakazo/media"; +import { resolveInside } from "@rakazo/media/node"; +import { z } from "zod"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * Edit plans for a project: the deterministic first cut per profile, an + * optional model ranking of the teaser candidates, and validation of plans the + * agent or user submits. + * + * Model judge (spec 8): only the top JUDGE_TOP_K candidates by deterministic + * score go to the model, and only those whose stored classifier scores already + * pass the target profile thresholds and that show no face of a protected + * person (or an unassigned face next to one). The model receives one still + * frame per candidate, taken from the proxy, plus the transcript excerpt. + * Face crops, embeddings and person data never leave the host. The model's + * order re-weights the candidates: rank r of n scales every feature by + * 1 - r / (2n); every usable segment it did not rank keeps half its weight + * (paid-only segments stay as they are: they only mark the payoff). Without a + * model, consent, or a usable reply the deterministic order stands. + */ + +export const JUDGE_TOP_K = 12; +const JUDGE_FRAME_WIDTH = 320; +const TRANSCRIPT_EXCERPT_CHARS = 300; +const RATIONALE_MAX_CHARS = 2000; + +const RANKING_GOALS: ReadonlySet = new Set([ + "teaser", + "trailer", + "highlight", + "story", +]); + +const JudgeReplySchema = z.object({ + ranking: z + .array(z.object({ segmentId: z.string().min(1).max(100), reason: z.string().max(300) })) + .max(50), +}); + +type AssetRow = Awaited>[number]; +type FaceTrackRow = Awaited>[number]; +type PersonRow = Awaited>[number]; +type ClassifierFrames = Array<{ tMs: number; scores: Partial> }>; + +export interface PlanContext { + project: MediaProjectRecord; + kit: BrandKit | null; + assets: Map; + segments: MediaSegment[]; + faceTracks: FaceTrackRow[]; + /** Every person of the space; plans may reference any of them. */ + people: PersonRow[]; + markIds: Set; + marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; + profiles: Map; +} + +export async function loadPlanContext( + deps: MediaServiceDeps, + actor: MediaActor, + projectId: string, +): Promise { + const { repos } = deps; + const { spaceId } = actor; + const project = await repos.getProjectRecord(spaceId, projectId); + const fromCollections = await repos.collectionAssetIds(spaceId, project.collectionIds); + const assetIds = [...new Set([...project.assetIds, ...fromCollections])]; + const [assetRows, segments, faceTracks, people, marks, kit, profileSet] = await Promise.all([ + repos.listAssetRecords(spaceId, assetIds), + repos.listSegments(spaceId, assetIds), + repos.faceTrackRecords(spaceId, assetIds), + repos.listPersonRecords(spaceId), + repos.listMarks(spaceId), + project.brandKitId ? repos.getBrandKit(spaceId, project.brandKitId) : Promise.resolve(null), + deps.profiles(), + ]); + return { + project, + kit: kit?.kit ?? null, + assets: new Map(assetRows.map((asset) => [asset.id, asset])), + segments, + faceTracks, + people, + markIds: new Set(marks.map((mark) => mark.id)), + marks: marks.map((mark) => ({ + id: mark.id, + treatment: mark.treatment, + assetId: mark.assetId, + personId: mark.personId, + })), + profiles: new Map( + [...profileSet.profiles.entries()].map(([id, entry]) => [id, entry.profile]), + ), + }; +} + +function kitArtifactIds(kit: BrandKit | null): string[] { + if (!kit) return []; + return [ + ...kit.fonts.map((font) => font.artifactId), + ...kit.logoArtifactIds, + ...(kit.lutArtifactId ? [kit.lutArtifactId] : []), + ...(kit.watermark?.artifactId ? [kit.watermark.artifactId] : []), + ...kit.musicBeds.map((bed) => bed.artifactId), + ]; +} + +export function buildEdlContext( + deps: Pick, + ctx: PlanContext, + profile: PlatformProfile, +): EdlContext { + return { + profile, + brandKit: ctx.kit, + assets: new Map( + [...ctx.assets.values()].map((asset) => [ + asset.id, + { + kind: asset.kind as "video" | "photo" | "live_photo", + durationMs: asset.durationMs, + width: asset.width, + height: asset.height, + }, + ]), + ), + artifactIds: new Set(kitArtifactIds(ctx.kit)), + music: new Map( + (ctx.kit?.musicBeds ?? []).map((bed) => [bed.artifactId, { commercialUse: bed.commercialUse }]), + ), + markIds: ctx.markIds, + personIds: new Set(ctx.people.map((person) => person.id)), + faceTrackIds: new Set(ctx.faceTracks.map((track) => track.id)), + templateIds: new Set(deps.motion.templates().map((template) => template.id)), + paidOnly: ctx.segments + .filter((segment) => segment.paidOnly) + .map((segment) => ({ + assetId: segment.assetId, + startMs: segment.startMs, + endMs: segment.endMs, + })), + goal: ctx.project.goal, + }; +} + +function segmentFeatures(segment: MediaSegment): SegmentFeatures { + const features: SegmentFeatures = { ...segment.scores }; + if (features.speechHook === undefined && segment.transcriptExcerpt) { + features.speechHook = speechHookScore(segment.transcriptExcerpt); + } + return features; +} + +function scoredInputs(ctx: PlanContext): ScoredSegmentInput[] { + return ctx.segments.map((segment) => ({ + id: segment.id, + assetId: segment.assetId, + startMs: segment.startMs, + endMs: segment.endMs, + capturedAt: ctx.assets.get(segment.assetId)?.capturedAt ?? null, + features: segmentFeatures(segment), + paidOnly: segment.paidOnly, + transcript: segment.transcriptExcerpt, + })); +} + +/** Scale features so the model's order counts as much as the deterministic score. */ +export function applyRanking( + segments: ScoredSegmentInput[], + order: string[], +): ScoredSegmentInput[] { + const rank = new Map(order.map((id, index) => [id, index])); + return segments.map((segment) => { + // Paid-only segments are never used; they only mark the payoff, so they keep their score. + if (segment.paidOnly) return segment; + const r = rank.get(segment.id); + const factor = r === undefined ? 0.5 : 1 - r / (2 * order.length); + const features = Object.fromEntries( + Object.entries(segment.features).map(([feature, value]) => [feature, (value ?? 0) * factor]), + ) as SegmentFeatures; + return { ...segment, features }; + }); +} + +async function classifierFrames( + deps: MediaServiceDeps, + spaceId: string, + assetIds: Set, +): Promise> { + const analyses = await deps.repos.latestAnalysesByKind(spaceId, "classifier"); + const out = new Map(); + for (const [assetId, row] of analyses) { + if (!assetIds.has(assetId)) continue; + let data: unknown = row.data; + if ((data === null || data === undefined) && row.dataPath) { + try { + data = JSON.parse(await readFile(await resolveInside(deps.mediaRoot, row.dataPath), "utf8")); + } catch { + continue; + } + } + const frames = (data as { frames?: unknown } | null)?.frames; + if (!Array.isArray(frames)) continue; + out.set( + assetId, + frames.filter( + (frame): frame is ClassifierFrames[number] => + typeof frame?.tMs === "number" && typeof frame.scores === "object" && frame.scores !== null, + ), + ); + } + return out; +} + +/** Fail closed: a segment without classifier frames in its range does not pass. */ +function classifierPasses( + frames: ClassifierFrames | undefined, + segment: { startMs: number; endMs: number }, + profile: PlatformProfile, +): boolean { + const inRange = (frames ?? []).filter( + (frame) => frame.tMs >= segment.startMs && frame.tMs <= segment.endMs, + ); + if (inRange.length === 0) return false; + return inRange.every((frame) => + Object.entries(profile.content.thresholds).every(([category, threshold]) => { + const score = frame.scores[category as ContentCategory]; + return score === undefined || threshold === undefined || score <= threshold; + }), + ); +} + +function protectedPersonIds(ctx: PlanContext): Set { + return new Set( + ctx.people.filter((person) => person.anonymizeByDefault !== "none").map((p) => p.id), + ); +} + +function showsProtectedFace( + ctx: PlanContext, + protectedIds: Set, + segment: { assetId: string; startMs: number; endMs: number }, +): boolean { + if (protectedIds.size === 0) return false; + return ctx.faceTracks.some( + (track) => + track.assetId === segment.assetId && + track.startMs < segment.endMs && + track.endMs > segment.startMs && + (track.personId === null || protectedIds.has(track.personId)), + ); +} + +interface Judgement { + order: string[]; + reasons: Map; +} + +async function judgeCandidates( + deps: MediaServiceDeps, + actor: MediaActor, + ctx: PlanContext, + profile: PlatformProfile, + segments: ScoredSegmentInput[], + weights: TeaserWeights, +): Promise { + const top = segments + .filter((segment) => !segment.paidOnly && segment.endMs > segment.startMs) + .map((segment) => ({ segment, score: scoreSegment(segment.features, weights) })) + .sort((a, b) => b.score - a.score || a.segment.id.localeCompare(b.segment.id)) + .slice(0, JUDGE_TOP_K); + if (top.length < 2) return null; + const classifier = await classifierFrames( + deps, + actor.spaceId, + new Set(top.map((c) => c.segment.assetId)), + ); + const protectedIds = protectedPersonIds(ctx); + const eligible = top.filter( + ({ segment }) => + classifierPasses(classifier.get(segment.assetId), segment, profile) && + !showsProtectedFace(ctx, protectedIds, segment), + ); + if (eligible.length < 2) return null; + + const workRelative = path.join(actor.spaceId, "tmp", `judge-${crypto.randomUUID()}`); + const workDir = await resolveInside(deps.mediaRoot, workRelative); + await mkdir(workDir, { recursive: true }); + try { + const images: AgentInputImage[] = []; + const sent: typeof eligible = []; + for (const candidate of eligible) { + const asset = ctx.assets.get(candidate.segment.assetId); + const isVideo = asset?.kind === "video"; + const source = isVideo ? asset?.proxyPath : (asset?.thumbPath ?? asset?.proxyPath); + if (!source) continue; + const outPath = path.join(workDir, `${images.length}.png`); + try { + await deps.renderer.extractFrame({ + path: await resolveInside(deps.mediaRoot, source), + tMs: isVideo + ? Math.round((candidate.segment.startMs + candidate.segment.endMs) / 2) + : 0, + outPath, + width: JUDGE_FRAME_WIDTH, + }); + images.push({ + name: `${candidate.segment.id}.png`, + mimeType: "image/png", + data: new Uint8Array(await readFile(outPath)), + }); + sent.push(candidate); + } catch (error) { + getLogger().warn("media judge frame failed", { + reason: error instanceof Error ? error.message : String(error), + }); + } + } + if (sent.length < 2) return null; + const lines = sent.map(({ segment, score }, index) => { + const transcript = (segment.transcript ?? "").replace(/\s+/g, " ").trim(); + return [ + `${index + 1}. id=${segment.id} image=${segment.id}.png`, + `duration=${segment.endMs - segment.startMs}ms score=${score.toFixed(3)}`, + transcript ? `transcript="${transcript.slice(0, TRANSCRIPT_EXCERPT_CHARS)}"` : "", + ] + .filter(Boolean) + .join(" "); + }); + const reply = await deps.model.generate({ + spaceId: actor.spaceId, + userId: actor.userId, + instructions: [ + "You rank candidate segments for a short promotional video.", + `Goal: ${ctx.project.goal}. Platform: ${profile.name}. The first ${profile.duration.hookWindowMs} ms must hook the viewer.`, + "Judge only what the frames and transcripts show. Prefer clear, engaging openers that keep the payoff hidden.", + 'Reply with one JSON object: {"ranking":[{"segmentId":"...","reason":"..."}]}, best first, using only the given ids.', + ].join("\n"), + prompt: `Candidates:\n${lines.join("\n")}`, + images, + schema: JudgeReplySchema, + }); + if (!reply) return null; + const ids = new Set(sent.map(({ segment }) => segment.id)); + const order: string[] = []; + const reasons = new Map(); + for (const entry of reply.ranking) { + if (!ids.has(entry.segmentId) || reasons.has(entry.segmentId)) continue; + order.push(entry.segmentId); + reasons.set(entry.segmentId, entry.reason.replace(/\s+/g, " ").trim()); + } + return order.length > 0 ? { order, reasons } : null; + } finally { + await rm(workDir, { recursive: true, force: true }); + } +} + +function rationaleFor(input: { + edl: Edl; + profile: PlatformProfile; + ctx: PlanContext; + learned: boolean; + judgement: Judgement | null; +}): string { + const { edl, profile, ctx } = input; + const lines = [ + `Automatic ${ctx.project.goal} plan for ${profile.name}.`, + input.learned + ? "Weights learned from this space's published results." + : `Default ${ctx.project.goal} weights.`, + ]; + const first = [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs)[0]; + if (first) { + lines.push( + `Hook: clip ${first.id}${first.segmentId ? ` (segment ${first.segmentId})` : ""} starts at ${introMs(edl) + first.atMs} ms, inside the ${profile.duration.hookWindowMs} ms hook window.`, + ); + } + if (edl.intro) lines.push("Brand intro opens the video."); + else if (ctx.kit?.intro) lines.push("Brand intro left out so the hook comes first."); + if (ctx.segments.some((segment) => segment.paidOnly)) lines.push("Paid-only segments excluded."); + if (input.judgement) { + const reasons = input.judgement.order + .slice(0, 3) + .map((id) => `${id}: ${input.judgement?.reasons.get(id) ?? ""}`.trim()); + lines.push(`Model ranking applied (${input.judgement.order.length} ranked): ${reasons.join("; ")}`); + } + return lines.join("\n").slice(0, RATIONALE_MAX_CHARS); +} + +function requireProfile(ctx: PlanContext, profileId: string): PlatformProfile { + if (!ctx.project.profileIds.includes(profileId)) { + throw new InvalidMediaInputError(`Profile ${profileId} is not part of this project`); + } + const profile = ctx.profiles.get(profileId); + if (!profile) throw new InvalidMediaInputError(`Unknown platform profile ${profileId}`); + return profile; +} + +/** First cut per profile, validated and stored as a new plan version by the agent. */ +export async function autoPlan( + deps: MediaServiceDeps, + actor: MediaActor, + input: { projectId: string; profileIds?: string[] }, +): Promise { + const ctx = await loadPlanContext(deps, actor, input.projectId); + const profileIds = [...new Set(input.profileIds ?? ctx.project.profileIds)]; + const profiles = profileIds.map((id) => requireProfile(ctx, id)); + const model = await deps.repos.getTeaserModel(actor.spaceId); + const learned = Boolean(model && model.sampleCount >= TEASER_LEARNING_MIN_SAMPLES); + const weights = + learned && model ? model.weights : defaultTeaserWeights(ctx.project.goal); + const base = scoredInputs(ctx); + const assets = new Map( + [...ctx.assets.values()].map((asset) => [ + asset.id, + { + durationMs: asset.durationMs, + width: asset.width, + height: asset.height, + hasSpeech: ctx.segments.some( + (segment) => segment.assetId === asset.id && Boolean(segment.transcriptExcerpt?.trim()), + ), + }, + ]), + ); + const projectPeople = new Set( + ctx.faceTracks.flatMap((track) => (track.personId ? [track.personId] : [])), + ); + const people = ctx.people + .filter((person) => projectPeople.has(person.id)) + .map((person) => ({ id: person.id, anonymizeByDefault: person.anonymizeByDefault })); + const endCardCta = ctx.kit?.endCard?.params.cta; + + const plans: MediaEditPlan[] = []; + for (const profile of profiles) { + const judgement = RANKING_GOALS.has(ctx.project.goal) + ? await judgeCandidates(deps, actor, ctx, profile, base, weights) + : null; + const segments = judgement ? applyRanking(base, judgement.order) : base; + const edl = autoPlanEdl({ + goal: ctx.project.goal, + profile, + brandKit: ctx.kit, + segments, + weights, + assets, + people, + marks: ctx.marks, + ctaText: typeof endCardCta === "string" ? endCardCta : null, + }); + const validation = validateEdl(edl, buildEdlContext(deps, ctx, profile)); + if (!validation.ok) { + throw new InvalidMediaInputError( + `Automatic plan for ${profile.id} is invalid: ${validation.errors + .map((error) => `${error.path} ${error.message}`) + .join("; ")}`, + ); + } + plans.push( + await deps.repos.createPlanVersion(actor.spaceId, { + projectId: ctx.project.id, + profileId: profile.id, + edl: validation.edl, + createdBy: "agent", + rationale: rationaleFor({ edl: validation.edl, profile, ctx, learned, judgement }), + }), + ); + } + return plans; +} + +export async function submitPlan( + deps: MediaServiceDeps, + actor: MediaActor, + input: { projectId: string; edl: unknown; rationale: string | null; createdBy: "agent" | "user" }, +): Promise<{ ok: true; plan: MediaEditPlan } | { ok: false; errors: EdlError[] }> { + const ctx = await loadPlanContext(deps, actor, input.projectId); + const profileId = (input.edl as { output?: { profileId?: unknown } } | null)?.output?.profileId; + const profile = + typeof profileId === "string" && ctx.project.profileIds.includes(profileId) + ? ctx.profiles.get(profileId) + : undefined; + if (!profile) { + return { + ok: false, + errors: [ + { + code: "profile_mismatch", + path: "output.profileId", + message: `plan must target one of the project's profiles: ${ctx.project.profileIds.join(", ")}`, + }, + ], + }; + } + const validation = validateEdl(input.edl, buildEdlContext(deps, ctx, profile)); + if (!validation.ok) return { ok: false, errors: validation.errors }; + const plan = await deps.repos.createPlanVersion(actor.spaceId, { + projectId: ctx.project.id, + profileId: profile.id, + edl: validation.edl, + createdBy: input.createdBy, + rationale: input.rationale, + }); + return { ok: true, plan }; +} diff --git a/packages/adapters/src/media-marketing-thumbnails.ts b/packages/adapters/src/media-marketing-thumbnails.ts new file mode 100644 index 0000000..213e9dd --- /dev/null +++ b/packages/adapters/src/media-marketing-thumbnails.ts @@ -0,0 +1,350 @@ +import { access, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import type { BrandKit, ContentCategory, Edl, MediaRenderOutput, PlatformProfile } from "@rakazo/contracts"; +import { MediaStateConflictError } from "@rakazo/db"; +import { + clipLengthMs, + escapeFilterText, + escapeFilterValue, + introMs, + locateOutputTime, + mainProgramMs, + rankThumbnailFrames, +} from "@rakazo/media"; +import { resolveInside } from "@rakazo/media/node"; +import sharp from "sharp"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; +import type { MediaToolPaths } from "./media-folder-source-probe.js"; +import { runMediaTool } from "./media-folder-source-probe.js"; +import type { StoredRenderOutput } from "./media-marketing-files.js"; +import { + materializeArtifact, + publicRenderOutput, + renderOutputPath, + storeMediaArtifact, + storedRenderOutputs, +} from "./media-marketing-files.js"; + +/* + * Thumbnail and poster variants for a finished render (spec 10.1). + * + * Candidates are 24 evenly spaced frames of the main program plus the frame + * just after every cut. Each is decoded small and scored: sharpness is the + * variance of a 4-neighbour Laplacian, contrast the luminance standard + * deviation (both relative to the best candidate), thirds the gradient energy + * near the rule-of-thirds points against the frame average, and face 1 when a + * stored face track of an unprotected person covers the source time. + * rankThumbnailFrames orders them; the best frames are extracted at full size + * as PNG (no EXIF or text chunks) with the plan title drawn in the brand + * colours inside the safe zone. Every variant passes the content thresholds of + * the profile and, when the project protects a person, must show no detectable + * face; failing frames are dropped and the next candidate takes their place. + */ + +export const THUMBNAIL_CANDIDATES = 24; +const MAX_VARIANTS = 10; +const ANALYSIS_WIDTH = 160; +const MAX_THUMB_WIDTH = 1920; +const CUT_OFFSET_MS = 200; +const TITLE_MAX_CHARS = 80; +const DEJAVU_BOLD = [ + "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", + "/usr/share/fonts/dejavu/DejaVuSans-Bold.ttf", + "/usr/share/fonts/TTF/DejaVuSans-Bold.ttf", +]; + +export function thumbnailVariantName(index: number): string { + return `thumb-${String.fromCharCode(97 + index)}`; +} + +export interface FrameMeasure { + /** Variance of the 4-neighbour Laplacian over the grey frame. */ + laplacianVariance: number; + /** Standard deviation of the grey levels. */ + contrast: number; + /** 0..1: gradient energy near the third intersections relative to the frame. */ + thirds: number; +} + +/** Sharpness, contrast and thirds of a packed 8-bit grey frame. */ +export function measureGrayFrame(data: Uint8Array, width: number, height: number): FrameMeasure { + let sum = 0; + let sumSq = 0; + for (let i = 0; i < width * height; i++) { + const value = data[i] ?? 0; + sum += value; + sumSq += value * value; + } + const pixels = Math.max(1, width * height); + const mean = sum / pixels; + const contrast = Math.sqrt(Math.max(0, sumSq / pixels - mean * mean)); + + let lapSum = 0; + let lapSq = 0; + let lapCount = 0; + let energyAll = 0; + let energyThirds = 0; + let thirdsCount = 0; + const radiusX = width / 8; + const radiusY = height / 8; + const points = [1 / 3, 2 / 3].flatMap((fx) => + [1 / 3, 2 / 3].map((fy) => [fx * width, fy * height] as const), + ); + for (let y = 1; y < height - 1; y++) { + for (let x = 1; x < width - 1; x++) { + const at = (dx: number, dy: number) => data[(y + dy) * width + x + dx] ?? 0; + const center = at(0, 0); + const lap = 4 * center - at(-1, 0) - at(1, 0) - at(0, -1) - at(0, 1); + lapSum += lap; + lapSq += lap * lap; + lapCount += 1; + const gradient = Math.abs(at(1, 0) - at(-1, 0)) + Math.abs(at(0, 1) - at(0, -1)); + energyAll += gradient; + if (points.some(([px, py]) => Math.abs(x - px) <= radiusX && Math.abs(y - py) <= radiusY)) { + energyThirds += gradient; + thirdsCount += 1; + } + } + } + const lapMean = lapCount ? lapSum / lapCount : 0; + const laplacianVariance = lapCount ? Math.max(0, lapSq / lapCount - lapMean * lapMean) : 0; + const meanAll = lapCount ? energyAll / lapCount : 0; + const meanThirds = thirdsCount ? energyThirds / thirdsCount : 0; + const thirds = meanAll > 0 ? Math.min(1, meanThirds / meanAll / 2) : 0; + return { laplacianVariance, contrast, thirds }; +} + +/** Candidate output times: evenly spaced plus just after each cut, inside the main program. */ +export function thumbnailCandidateTimes(edl: Edl, durationMs: number): number[] { + const start = introMs(edl); + const end = Math.min(durationMs, start + mainProgramMs(edl)); + if (end <= start) return []; + const span = end - start; + const times = new Set(); + for (let i = 0; i < THUMBNAIL_CANDIDATES; i++) { + times.add(Math.round(start + ((i + 0.5) * span) / THUMBNAIL_CANDIDATES)); + } + for (const clip of edl.tracks.video) { + const t = start + clip.atMs + Math.min(CUT_OFFSET_MS, clipLengthMs(clip) / 2); + if (t > start && t < end) times.add(Math.round(t)); + } + return [...times].sort((a, b) => a - b); +} + +async function existingFile(candidates: string[]): Promise { + for (const file of candidates) { + try { + await access(file); + return file; + } catch { + // try the next location + } + } + return null; +} + +function ffmpegColor(hex: string | undefined, fallback: string): string { + return hex && /^#[0-9a-fA-F]{6}$/.test(hex) ? `0x${hex.slice(1)}` : fallback; +} + +/** drawtext title band inside the top safe zone; every value is escaped. */ +export function titleOverlayFilter(input: { + text: string; + width: number; + height: number; + profile: PlatformProfile; + kit: BrandKit | null; + fontFile: string | null; +}): string { + const text = input.text.replace(/\s+/g, " ").trim().slice(0, TITLE_MAX_CHARS); + const fontSize = Math.max( + 12, + Math.round(Math.min(input.height / 14, (input.width * 0.9) / Math.max(1, text.length * 0.6))), + ); + const y = Math.round(input.height * (input.profile.safeZones.topPct / 100 + 0.04)); + const font = input.fontFile + ? `fontfile=${escapeFilterValue(input.fontFile)}` + : `font=${escapeFilterValue("DejaVu Sans")}`; + return [ + `drawtext=${font}`, + `text=${escapeFilterText(text)}`, + `fontsize=${fontSize}`, + `fontcolor=${ffmpegColor(input.kit?.colors.text, "white")}`, + "box=1", + `boxcolor=${ffmpegColor(input.kit?.colors.background, "black")}@0.6`, + `boxborderw=${Math.max(4, Math.round(fontSize / 3))}`, + "x=(w-text_w)/2", + `y=${y}`, + ].join(":"); +} + +function exceedsThresholds( + frames: Array<{ scores: Partial> }>, + profile: PlatformProfile, +): boolean { + return frames.some((frame) => + Object.entries(profile.content.thresholds).some(([category, threshold]) => { + const score = frame.scores[category as ContentCategory]; + return score !== undefined && threshold !== undefined && score > threshold; + }), + ); +} + +export async function createThumbnails( + deps: MediaServiceDeps, + actor: MediaActor, + input: { renderId: string; count: number }, + tools: MediaToolPaths = {}, +): Promise { + const { spaceId } = actor; + const row = await deps.repos.getRenderRecord(spaceId, input.renderId); + if (row.status !== "done") throw new MediaStateConflictError("Thumbnails need a finished render"); + const outputs = storedRenderOutputs(row.outputs); + const video = outputs.find((output) => output.kind === "video"); + if (!video) throw new MediaStateConflictError("Render has no video output"); + const videoPath = await renderOutputPath(deps, spaceId, row.id, video); + const [plan, profileSet, project] = await Promise.all([ + deps.repos.getPlan(spaceId, row.planId), + deps.profiles(), + deps.repos.getProjectRecord(spaceId, row.projectId), + ]); + const profile = profileSet.profiles.get(row.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${row.profileId} is not available`); + const edl = plan.edl; + const kit = project.brandKitId + ? (await deps.repos.getBrandKit(spaceId, project.brandKitId)).kit + : null; + const stored = row.ffprobe as { durationMs?: unknown } | null; + const durationMs = + typeof stored?.durationMs === "number" + ? stored.durationMs + : (await deps.renderer.probe(videoPath)).durationMs; + + const assetIds = [...new Set(edl.tracks.video.map((clip) => clip.assetId))]; + const tracks = await deps.repos.faceTrackRecords(spaceId, assetIds); + const personIds = [...new Set(tracks.flatMap((t) => (t.personId ? [t.personId] : [])))]; + const people = personIds.length ? await deps.repos.listPersonRecords(spaceId, personIds) : []; + const protectedIds = new Set( + people.filter((person) => person.anonymizeByDefault !== "none").map((person) => person.id), + ); + const protectsFaces = protectedIds.size > 0 || edl.redactions.faces.length > 0; + + const relativeDir = path.join(spaceId, "renders", row.id, "thumbs"); + const dir = await resolveInside(deps.mediaRoot, relativeDir); + const candidateDir = path.join(dir, "candidates"); + await rm(candidateDir, { recursive: true, force: true }); + await mkdir(candidateDir, { recursive: true }); + try { + const measured: Array<{ tMs: number } & FrameMeasure & { face: number }> = []; + for (const tMs of thumbnailCandidateTimes(edl, durationMs)) { + const file = path.join(candidateDir, `${tMs}.png`); + await deps.renderer.extractFrame({ path: videoPath, tMs, outPath: file, width: ANALYSIS_WIDTH }); + const { data, info } = await sharp(file).greyscale().raw().toBuffer({ resolveWithObject: true }); + const at = locateOutputTime(edl, tMs); + const face = + at && + tracks.some( + (track) => + track.assetId === at.assetId && + track.startMs <= at.sourceMs && + track.endMs >= at.sourceMs && + track.personId !== null && + !protectedIds.has(track.personId), + ) + ? 1 + : 0; + measured.push({ tMs, face, ...measureGrayFrame(data, info.width, info.height) }); + } + const maxSharp = Math.max(1e-9, ...measured.map((m) => m.laplacianVariance)); + const maxContrast = Math.max(1e-9, ...measured.map((m) => m.contrast)); + const order = rankThumbnailFrames( + measured.map((m) => ({ + tMs: m.tMs, + sharpness: m.laplacianVariance / maxSharp, + face: m.face, + thirds: m.thirds, + contrast: m.contrast / maxContrast, + })), + measured.length, + ); + + const title = edl.metadata.title?.trim() || null; + let fontFile: string | null = null; + if (title) { + const heading = kit?.fonts.find((font) => font.role === "heading") ?? kit?.fonts[0]; + fontFile = heading + ? await materializeArtifact(deps, actor, heading.artifactId, path.join(dir, "fonts")) + : await existingFile(DEJAVU_BOLD); + } + const width = Math.min(edl.output.width, MAX_THUMB_WIDTH); + const height = Math.round((width * edl.output.height) / edl.output.width / 2) * 2; + const wanted = Math.max(0, Math.min(MAX_VARIANTS, Math.floor(input.count))); + const results: StoredRenderOutput[] = []; + for (const tMs of order) { + if (results.length >= wanted) break; + const name = thumbnailVariantName(results.length); + const out = path.join(dir, `${name}.png`); + const frame = path.join(candidateDir, `full-${tMs}.png`); + await deps.renderer.extractFrame({ path: videoPath, tMs, outPath: frame, width }); + if (title) { + await runMediaTool(tools.ffmpegPath ?? "ffmpeg", [ + "-v", + "error", + "-y", + "-i", + frame, + "-vf", + titleOverlayFilter({ text: title, width, height, profile, kit, fontFile }), + "-frames:v", + "1", + "-map_metadata", + "-1", + "-bitexact", + "-c:v", + "png", + out, + ]); + } else { + await rename(frame, out); + } + if (deps.inference) { + const content = await deps.inference.classifyContent({ + path: out, + sampleFps: 1, + extraTimesMs: [0], + }); + let rejected = exceedsThresholds(content.frames, profile); + if (!rejected && protectsFaces) { + const faces = await deps.inference.detectFaces({ path: out, sampleFps: 1, timesMs: [0] }); + rejected = faces.frames.some((f) => f.faces.length > 0); + } + if (rejected) { + await rm(out, { force: true }); + continue; + } + } + const bytes = await readFile(out); + const artifactId = await storeMediaArtifact(deps, actor, { + name: `${name}.png`, + mimeType: "image/png", + bytes, + }); + results.push({ + kind: "thumbnail", + artifactId, + fileName: `${name}.png`, + sizeBytes: (await stat(out)).size, + path: path.join(relativeDir, `${name}.png`), + }); + } + await deps.repos.updateRender( + spaceId, + row.id, + { outputs: [...outputs.filter((output) => output.kind !== "thumbnail"), ...results] }, + ["done"], + ); + return results.map(publicRenderOutput); + } finally { + await rm(candidateDir, { recursive: true, force: true }); + } +} diff --git a/packages/adapters/src/media-metrics.ts b/packages/adapters/src/media-metrics.ts new file mode 100644 index 0000000..fec5389 --- /dev/null +++ b/packages/adapters/src/media-metrics.ts @@ -0,0 +1,133 @@ +import type { Edl, MediaSegment } from "@rakazo/contracts"; +import { TEASER_FEATURES, TEASER_LEARNING_MIN_SAMPLES } from "@rakazo/contracts"; +import type { NewMediaMetric, TeaserModelRecord } from "@rakazo/db"; +import { InvalidMediaInputError } from "@rakazo/db"; +import type { SegmentFeatures } from "@rakazo/media"; +import { + clipLengthMs, + defaultTeaserWeights, + fitTeaserWeights, + parseMetricsCsv, + speechHookScore, +} from "@rakazo/media"; +import type { MediaActor, MediaServiceDeps } from "./media-context.js"; + +/* + * Results import and teaser learning (spec 8 "Lernen", 10.6). + * + * A CSV import resolves rows through the saved mapping and records them as + * cumulative snapshots: re-importing a report replaces the same + * (publication, date, source) rows, so numbers never double. After every + * import the teaser weights are refit from all publications with views. Each + * publication contributes the clip-length-weighted mean features of the + * segments its plan used, with its latest views and conversions. Below + * TEASER_LEARNING_MIN_SAMPLES publications nothing is stored and plans keep + * the default weights; resetTeaserModel goes back to them. + */ + +export const METRICS_CSV_MAX_CHARS = 5_000_000; + +/** Clip-length-weighted mean of the features of the segments a plan used. */ +export function planFeatures(edl: Edl, segments: MediaSegment[]): SegmentFeatures | null { + const byId = new Map(segments.map((segment) => [segment.id, segment])); + const totals: Record = {}; + let weight = 0; + for (const clip of edl.tracks.video) { + const used = clip.segmentId + ? [byId.get(clip.segmentId)].filter((s): s is MediaSegment => Boolean(s)) + : segments.filter( + (s) => s.assetId === clip.assetId && s.startMs < clip.outMs && s.endMs > clip.inMs, + ); + if (used.length === 0) continue; + const length = clipLengthMs(clip) / used.length; + for (const segment of used) { + const features: SegmentFeatures = { ...segment.scores }; + if (features.speechHook === undefined && segment.transcriptExcerpt) { + features.speechHook = speechHookScore(segment.transcriptExcerpt); + } + for (const feature of TEASER_FEATURES) { + totals[feature] = (totals[feature] ?? 0) + (features[feature] ?? 0) * length; + } + weight += length; + } + } + if (weight <= 0) return null; + return Object.fromEntries( + TEASER_FEATURES.map((feature) => [feature, (totals[feature] ?? 0) / weight]), + ) as SegmentFeatures; +} + +/** Refit and store the space's teaser weights; null while there are too few results. */ +export async function refitTeaserModel( + deps: Pick, + spaceId: string, +): Promise { + const { repos } = deps; + const rows = (await repos.funnel(spaceId)).filter((row) => row.views > 0); + if (rows.length < TEASER_LEARNING_MIN_SAMPLES) return null; + const plans = new Map(); + const samples: Array<{ edl: Edl; views: number; conversions: number }> = []; + for (const row of rows) { + const publication = await repos.getPublication(spaceId, row.publicationId); + const render = await repos.getRender(spaceId, publication.renderId); + let edl = plans.get(render.planId); + if (!edl) { + edl = (await repos.getPlan(spaceId, render.planId)).edl; + plans.set(render.planId, edl); + } + samples.push({ edl, views: row.views, conversions: row.conversions }); + } + const assetIds = [ + ...new Set(samples.flatMap((s) => s.edl.tracks.video.map((clip) => clip.assetId))), + ]; + const segments = await repos.listSegments(spaceId, assetIds); + const usable = samples.flatMap((sample) => { + const features = planFeatures(sample.edl, segments); + return features + ? [{ features, views: sample.views, conversions: Math.min(sample.conversions, sample.views) }] + : []; + }); + const weights = fitTeaserWeights(usable, defaultTeaserWeights("teaser")); + if (!weights) return null; + return repos.putTeaserModel(spaceId, weights, usable.length); +} + +export async function importMetricsCsv( + deps: Pick, + actor: MediaActor, + input: { mappingId: string; csv: string }, +): Promise<{ imported: number; skipped: Array<{ row: number; reason: string }> }> { + if (input.csv.length > METRICS_CSV_MAX_CHARS) { + throw new InvalidMediaInputError("CSV file is too large"); + } + const { spaceId } = actor; + const mapping = await deps.repos.getMetricMapping(spaceId, input.mappingId); + const parsed = parseMetricsCsv(input.csv, mapping); + const keys = await deps.repos.resolvePublicationKeys( + spaceId, + mapping.publicationKey, + parsed.rows.map((row) => row.publicationKey), + ); + const skipped = [...parsed.skipped]; + const metrics: NewMediaMetric[] = []; + for (const row of parsed.rows) { + const publicationId = keys.get(row.publicationKey.trim()); + if (!publicationId) { + skipped.push({ row: row.row, reason: "publication not found" }); + continue; + } + metrics.push({ + publicationId, + observedAt: row.observedAt.toISOString(), + views: row.views, + clicks: row.clicks, + conversions: row.conversions, + revenueCents: row.revenueCents, + currency: row.currency, + source: "csv_import", + }); + } + const imported = await deps.repos.recordMetrics(spaceId, metrics); + if (imported > 0) await refitTeaserModel(deps, spaceId); + return { imported, skipped: skipped.sort((a, b) => a.row - b.row) }; +} diff --git a/packages/adapters/src/media-publish.ts b/packages/adapters/src/media-publish.ts new file mode 100644 index 0000000..76dafac --- /dev/null +++ b/packages/adapters/src/media-publish.ts @@ -0,0 +1,357 @@ +import { copyFile, link, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { MediaJobHandlers } from "@rakazo/adapter-kit"; +import { mediaPublishJob } from "@rakazo/adapter-kit"; +import type { MediaPublication, Utm } from "@rakazo/contracts"; +import { ATTACHMENT_MAX_BYTES, UtmSchema } from "@rakazo/contracts"; +import { InvalidMediaInputError, MediaStateConflictError } from "@rakazo/db"; +import { getLogger } from "@rakazo/logging"; +import { buildTrackingUrl, normalizeHashtags, uploadChecklist } from "@rakazo/media"; +import { resolveInside } from "@rakazo/media/node"; +import type { MediaActor, MediaMarketingService, MediaServiceDeps } from "./media-context.js"; +import type { StoredRenderOutput } from "./media-marketing-files.js"; +import { + renderOutputPath, + slugify, + storedRenderOutputs, + storeMediaArtifact, +} from "./media-marketing-files.js"; +import { thumbnailVariantName } from "./media-marketing-thumbnails.js"; +import type { ZipEntry } from "./media-zip.js"; +import { writeStoreZip } from "./media-zip.js"; + +/* + * Publications: tracking links, scheduling, and the export bundle (spec 10.3–10.5). + * + * A scheduled publication is a `media.publish` background job with + * `availableAt` (Graphile scheduling, as routines use), keyed per publication + * so rescheduling replaces it and canceling removes it. + * + * The bundle lives in `/bundles//` and is rebuilt from + * scratch on every call. Every file name is a constant from BUNDLE_FILES (or a + * thumbnail variant name), never user input, and the ZIP writer only accepts + * flat names, so extracting the archive cannot escape its folder. The ZIP + * becomes an artifact when it fits the attachment limit (ATTACHMENT_MAX_BYTES, + * the same limit chat artifacts have); larger bundles stay as `bundle.zip` in + * the folder for the media file route and bundleArtifactId stays null. + */ + +export const BUNDLE_FILES = { + video: "video.mp4", + posterPng: "poster.png", + posterJpg: "poster.jpg", + srt: "subtitles.srt", + vtt: "subtitles.vtt", + caption: "caption.txt", + metadata: "metadata.json", + checklist: "UPLOAD-CHECKLIST.md", + disclosure: "AI-DISCLOSURE.txt", + zip: "bundle.zip", +} as const; + +const MAX_THUMBNAIL_VARIANTS = 10; + +export function bundleRelativeDir(spaceId: string, publicationId: string): string { + return path.join(spaceId, "bundles", publicationId); +} + +async function publishEvent(deps: MediaServiceDeps, spaceId: string, pub: MediaPublication) { + await deps.publish(spaceId, { + type: "media.publication", + publicationId: pub.id, + status: pub.status, + }); +} + +/** "9x16"-style format of an output size. */ +function aspectLabel(width: number, height: number): string { + const gcd = (a: number, b: number): number => (b === 0 ? a : gcd(b, a % b)); + const divisor = gcd(width, height) || 1; + return `${width / divisor}x${height / divisor}`; +} + +export async function createPublication( + deps: MediaServiceDeps, + actor: MediaActor, + input: { + renderId: string; + destination: string; + targetUrl: string | null; + utm: Utm | null; + title: string | null; + caption: string | null; + tags: string[]; + thumbnailVariant: string | null; + scheduledAt: string | null; + }, +): Promise { + const { spaceId } = actor; + const render = await deps.repos.getRender(spaceId, input.renderId); + if (render.status !== "done") { + throw new MediaStateConflictError("Only a finished render can be published"); + } + const [plan, project] = await Promise.all([ + deps.repos.getPlan(spaceId, render.planId), + deps.repos.getProjectRecord(spaceId, render.projectId), + ]); + const scheduledAt = input.scheduledAt ? new Date(input.scheduledAt) : null; + if (scheduledAt && Number.isNaN(scheduledAt.getTime())) { + throw new InvalidMediaInputError("Invalid schedule time"); + } + const utm = UtmSchema.parse( + input.utm ?? { + source: render.profileId, + medium: `video-${aspectLabel(plan.edl.output.width, plan.edl.output.height)}`, + campaign: slugify(project.title) || project.id.slice(0, 100), + content: (input.thumbnailVariant ?? render.id).slice(0, 100), + }, + ); + let trackingUrl: string | null = null; + if (input.targetUrl) { + try { + trackingUrl = buildTrackingUrl(input.targetUrl, utm); + } catch (error) { + throw new InvalidMediaInputError(error instanceof Error ? error.message : "Invalid target"); + } + if (deps.shortener) { + try { + trackingUrl = await deps.shortener.shorten(trackingUrl); + } catch (error) { + getLogger().warn("media link shortener failed; keeping the full tracking link", { + shortener: deps.shortener.id, + reason: error instanceof Error ? error.message : String(error), + }); + } + } + } + const publication = await deps.repos.createPublication(spaceId, actor.userId, { + renderId: render.id, + destination: input.destination.trim() || "bundle", + targetUrl: input.targetUrl, + trackingUrl, + utm, + title: input.title, + caption: input.caption, + tags: input.tags, + thumbnailVariant: input.thumbnailVariant, + scheduledAt, + }); + if (scheduledAt) await deps.jobs.enqueue(mediaPublishJob(publication.id, scheduledAt)); + await publishEvent(deps, spaceId, publication); + return publication; +} + +export async function cancelPublication( + deps: MediaServiceDeps, + actor: MediaActor, + publicationId: string, +): Promise { + const publication = await deps.repos.cancelPublication(actor.spaceId, publicationId); + await deps.jobs.cancel(mediaPublishJob(publicationId).replaceKey as string); + await publishEvent(deps, actor.spaceId, publication); + return publication; +} + +async function placeFile(source: string, target: string): Promise { + try { + await link(source, target); + } catch { + await copyFile(source, target); + } +} + +function disclosureText(alterations: string[]): string { + return [ + `This video contains AI alterations: ${alterations.join(", ")}.`, + "Set the platform's label for AI-generated or altered content when you upload it.", + "", + ].join("\n"); +} + +export async function buildBundle( + deps: MediaServiceDeps, + actor: MediaActor, + publicationId: string, +): Promise { + const { spaceId } = actor; + const pub = await deps.repos.getPublication(spaceId, publicationId); + if (pub.status === "canceled" || pub.status === "published") { + throw new MediaStateConflictError(`Publication is already ${pub.status}`); + } + const row = await deps.repos.getRenderRecord(spaceId, pub.renderId); + const [plan, profileSet] = await Promise.all([ + deps.repos.getPlan(spaceId, row.planId), + deps.profiles(), + ]); + const profile = profileSet.profiles.get(pub.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${pub.profileId} is not available`); + const edl = plan.edl; + const outputs = storedRenderOutputs(row.outputs); + const video = outputs.find((output) => output.kind === "video"); + if (!video) throw new MediaStateConflictError("Render has no video output"); + + const dir = await resolveInside(deps.mediaRoot, bundleRelativeDir(spaceId, pub.id)); + await rm(dir, { recursive: true, force: true }); + await mkdir(dir, { recursive: true }); + const entries: ZipEntry[] = []; + const add = async (name: string, source: string) => { + const target = path.join(dir, name); + await placeFile(source, target); + entries.push({ name, path: target }); + }; + const write = async (name: string, content: string) => { + const target = path.join(dir, name); + await writeFile(target, content, "utf8"); + entries.push({ name, path: target }); + }; + const sourceOf = (output: StoredRenderOutput) => renderOutputPath(deps, spaceId, row.id, output); + + await add(BUNDLE_FILES.video, await sourceOf(video)); + + if (profile.export.poster) { + const variants = new Map( + Array.from({ length: MAX_THUMBNAIL_VARIANTS }, (_, i) => thumbnailVariantName(i)).map( + (name) => [name, outputs.find((o) => o.kind === "thumbnail" && o.fileName === `${name}.png`)], + ), + ); + const chosen = + (pub.thumbnailVariant ? variants.get(pub.thumbnailVariant) : undefined) ?? + outputs.find((o) => o.kind === "poster") ?? + outputs.find((o) => o.kind === "thumbnail"); + if (chosen) { + const jpeg = /\.jpe?g$/i.test(chosen.fileName); + await add(jpeg ? BUNDLE_FILES.posterJpg : BUNDLE_FILES.posterPng, await sourceOf(chosen)); + } + for (const [name, output] of variants) { + if (output) await add(`${name}.png`, await sourceOf(output)); + } + } + let hasSubtitles = false; + if (profile.export.subtitles) { + for (const [kind, name] of [ + ["subtitles_srt", BUNDLE_FILES.srt], + ["subtitles_vtt", BUNDLE_FILES.vtt], + ] as const) { + const output = outputs.find((o) => o.kind === kind); + if (!output) continue; + await add(name, await sourceOf(output)); + hasSubtitles = true; + } + } + const hashtags = normalizeHashtags(pub.tags, profile.text.hashtagLimit); + if (profile.export.captionText) { + const parts = [pub.caption?.trim() ?? "", hashtags.join(" ")]; + if (pub.trackingUrl && profile.text.linkPolicy === "allowed") parts.push(pub.trackingUrl); + await write(BUNDLE_FILES.caption, `${parts.filter(Boolean).join("\n\n")}\n`); + } + const aiAlterations = [...new Set(edl.metadata.aiAlterations)].sort(); + const aiAltered = edl.metadata.aiAltered || aiAlterations.length > 0; + await write( + BUNDLE_FILES.metadata, + `${JSON.stringify( + { + publicationId: pub.id, + renderId: row.id, + profile: { id: profile.id, name: profile.name, kind: profile.kind }, + title: pub.title, + caption: pub.caption, + tags: hashtags, + trackingUrl: pub.trackingUrl, + utm: pub.utm, + thumbnailVariant: pub.thumbnailVariant, + aiAltered, + aiAlterations, + ageRestricted: profile.disclosure.ageRestrictedFlag, + paidPartnershipLabel: profile.disclosure.paidPartnershipLabel, + }, + null, + 2, + )}\n`, + ); + if (profile.export.uploadChecklist) { + await write( + BUNDLE_FILES.checklist, + uploadChecklist({ + profile, + aiAltered, + hasCaptions: + hasSubtitles || edl.captions?.mode === "burned" || edl.captions?.mode === "both", + trackingUrl: pub.trackingUrl, + }), + ); + } + if (aiAltered) { + await write( + BUNDLE_FILES.disclosure, + disclosureText(aiAlterations.length > 0 ? aiAlterations : ["unspecified"]), + ); + } + + const zipPath = path.join(dir, BUNDLE_FILES.zip); + const zipBytes = await writeStoreZip(zipPath, entries); + let artifactId: string | null = null; + if (zipBytes <= ATTACHMENT_MAX_BYTES) { + artifactId = await storeMediaArtifact(deps, actor, { + name: `bundle-${slugify(profile.id, 40) || "export"}.zip`, + mimeType: "application/zip", + bytes: new Uint8Array(await readFile(zipPath)), + }); + } + const bundled = await deps.repos.markPublicationBundled(spaceId, pub.id, artifactId); + await publishEvent(deps, spaceId, bundled); + return bundled; +} + +/** + * Worker handler for `media.publish`: build the bundle at the scheduled time, + * hand it to the configured publisher, and record the outcome. A canceled or + * already published publication is left alone. Failures mark the publication + * failed and rethrow so the job queue retries; a retry rebuilds from scratch. + * + * No notification is sent here: MediaServiceDeps has no notification provider, + * so clients learn about the bundle through the realtime publication event. + */ +export function createMediaPublishHandler( + deps: MediaServiceDeps, + marketing: Pick, +): MediaJobHandlers["media.publish"] { + return async ({ publicationId }) => { + const spaceId = await deps.repos.mediaSpaceOf("publication", publicationId); + if (!spaceId) return; + const record = await deps.repos.getPublicationRecord(spaceId, publicationId); + if (record.status === "canceled" || record.status === "published") return; + const actor = { spaceId, userId: record.createdBy }; + try { + const bundled = await marketing.bundle(actor, publicationId); + const profile = (await deps.profiles()).profiles.get(bundled.profileId)?.profile; + if (!profile) throw new MediaStateConflictError(`Profile ${bundled.profileId} is not available`); + const result = await deps.publisher.publish({ + bundleDir: await resolveInside(deps.mediaRoot, bundleRelativeDir(spaceId, publicationId)), + profile, + publicationId, + title: bundled.title, + caption: bundled.caption, + tags: bundled.tags, + }); + if (result.status === "published") { + const published = await deps.repos.markPublicationPublished( + spaceId, + publicationId, + result.externalUrl, + deps.now(), + ); + await publishEvent(deps, spaceId, published); + } + } catch (error) { + const current = await deps.repos.getPublication(spaceId, publicationId); + if (current.status === "canceled") return; + if (current.status !== "failed" && current.status !== "published") { + const failed = await deps.repos + .markPublicationFailed(spaceId, publicationId) + .catch(() => null); + if (failed) await publishEvent(deps, spaceId, failed); + } + throw error; + } + }; +} diff --git a/packages/adapters/src/media-zip.ts b/packages/adapters/src/media-zip.ts new file mode 100644 index 0000000..6ac9b82 --- /dev/null +++ b/packages/adapters/src/media-zip.ts @@ -0,0 +1,143 @@ +import { createReadStream } from "node:fs"; +import { open, rm, stat } from "node:fs/promises"; +import { crc32 } from "node:zlib"; + +/* + * Minimal store-only (no compression) ZIP writer for export bundles. Videos + * are already compressed, so storing keeps it fast and dependency-free. + * + * Entry names are flat file names from the bundle's fixed allowlist: no + * directories, no "..", no absolute paths, so extracting can never write + * outside the target folder (zip slip). Timestamps are a fixed 1980-01-01 so + * the same inputs give the same archive. No ZIP64: an entry or archive past + * 4 GiB is rejected with ZipTooLargeError. + */ + +const ENTRY_NAME = /^[A-Za-z0-9][A-Za-z0-9_-]*(\.[A-Za-z0-9_-]+)*$/; +const MAX_ENTRY_NAME = 100; +const MAX_ZIP32 = 0xffff_fffe; +const MAX_ENTRIES = 0xffff; +const DOS_TIME = 0; +/** 1980-01-01 in DOS date format (year offset 0, month 1, day 1). */ +const DOS_DATE = (1 << 5) | 1; + +export class ZipTooLargeError extends Error { + constructor() { + super("bundle is larger than a ZIP archive without ZIP64 can hold (4 GiB)"); + this.name = "ZipTooLargeError"; + } +} + +export interface ZipEntry { + /** Flat file name inside the archive. */ + name: string; + /** Absolute path of the file to store. */ + path: string; +} + +/** Whether a name is a safe flat archive entry (letters, digits, "-", "_", inner dots). */ +export function isSafeZipEntryName(name: string): boolean { + return name.length <= MAX_ENTRY_NAME && ENTRY_NAME.test(name); +} + +async function fileCrc(file: string, signal?: AbortSignal): Promise { + let crc = 0; + for await (const chunk of createReadStream(file, { signal })) { + crc = crc32(chunk as Buffer, crc); + } + return crc >>> 0; +} + +/** Write a store-only ZIP of `entries` to `outPath`, replacing it. Returns the archive size. */ +export async function writeStoreZip( + outPath: string, + entries: ZipEntry[], + signal?: AbortSignal, +): Promise { + if (entries.length > MAX_ENTRIES) throw new ZipTooLargeError(); + const seen = new Set(); + for (const entry of entries) { + if (!isSafeZipEntryName(entry.name)) throw new Error(`unsafe zip entry name: ${entry.name}`); + if (seen.has(entry.name)) throw new Error(`duplicate zip entry: ${entry.name}`); + seen.add(entry.name); + } + const handle = await open(outPath, "w", 0o600); + let offset = 0; + const write = async (bytes: Uint8Array) => { + signal?.throwIfAborted(); + await handle.write(bytes); + offset += bytes.byteLength; + }; + const central: Buffer[] = []; + try { + for (const entry of entries) { + const size = (await stat(entry.path)).size; + const name = Buffer.from(entry.name, "ascii"); + if (size > MAX_ZIP32 || offset + 30 + name.length + size > MAX_ZIP32) { + throw new ZipTooLargeError(); + } + const crc = await fileCrc(entry.path, signal); + const headerOffset = offset; + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(10, 4); // version needed: 1.0, stored + local.writeUInt16LE(0, 6); // flags + local.writeUInt16LE(0, 8); // method: store + local.writeUInt16LE(DOS_TIME, 10); + local.writeUInt16LE(DOS_DATE, 12); + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(size, 18); + local.writeUInt32LE(size, 22); + local.writeUInt16LE(name.length, 26); + local.writeUInt16LE(0, 28); + await write(local); + await write(name); + let copied = 0; + for await (const chunk of createReadStream(entry.path, { signal })) { + await write(chunk as Buffer); + copied += (chunk as Buffer).byteLength; + } + if (copied !== size) throw new Error(`${entry.name} changed while archiving`); + + const record = Buffer.alloc(46); + record.writeUInt32LE(0x02014b50, 0); + record.writeUInt16LE((3 << 8) | 20, 4); // made by: Unix, 2.0 + record.writeUInt16LE(10, 6); + record.writeUInt16LE(0, 8); + record.writeUInt16LE(0, 10); + record.writeUInt16LE(DOS_TIME, 12); + record.writeUInt16LE(DOS_DATE, 14); + record.writeUInt32LE(crc, 16); + record.writeUInt32LE(size, 20); + record.writeUInt32LE(size, 24); + record.writeUInt16LE(name.length, 28); + record.writeUInt16LE(0, 30); // extra + record.writeUInt16LE(0, 32); // comment + record.writeUInt16LE(0, 34); // disk + record.writeUInt16LE(0, 36); // internal attributes + record.writeUInt32LE((0o100644 << 16) >>> 0, 38); // regular file, rw-r--r-- + record.writeUInt32LE(headerOffset, 42); + central.push(record, name); + } + const directory = Buffer.concat(central); + const directoryOffset = offset; + if (directoryOffset + directory.length + 22 > MAX_ZIP32) throw new ZipTooLargeError(); + await write(directory); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(0, 4); + end.writeUInt16LE(0, 6); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(directoryOffset, 16); + end.writeUInt16LE(0, 20); + await write(end); + await handle.close(); + return offset; + } catch (error) { + await handle.close().catch(() => undefined); + await rm(outPath, { force: true }); + throw error; + } +}