3ebe0fe47b
Fail closed when anonymization cannot prepare every clip, retry a failing hardware encoder in software, verify each ffmpeg output exists, and fail renders whose video or poster still carries source metadata. Cover parsing, encoder detection, two-pass loudness, abort and cleanup, the render job (auto-fix, cancel, outputs) on memory and Postgres, and a synthetic 16:9 to 9:16 render plus a region blur check with ffprobe. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NLSCTnhge4umMnemGXWr1w
911 lines
30 KiB
TypeScript
911 lines
30 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import type { AdapterContext, MediaJobHandlers, MediaRenderProgress } from "@rakazo/adapter-kit";
|
|
import { mediaRenderJob, UnsupportedMediaOperationError } from "@rakazo/adapter-kit";
|
|
import type {
|
|
AiAlteration,
|
|
BrandKit,
|
|
Edl,
|
|
GateReport,
|
|
MediaRender,
|
|
MediaRenderOutput,
|
|
MediaRenderQuality,
|
|
PlatformProfile,
|
|
RenderPlan,
|
|
} from "@rakazo/contracts";
|
|
import { MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
|
|
import type { MediaRenderPatch } from "@rakazo/db";
|
|
import { InvalidMediaInputError } from "@rakazo/db";
|
|
import { getLogger } from "@rakazo/logging";
|
|
import type { CaptionWord } from "@rakazo/media";
|
|
import { compileRenderPlan, findMetadataLeaks, planAutoFix } from "@rakazo/media";
|
|
import { resolveInside } from "@rakazo/media/node";
|
|
import type {
|
|
MediaActor,
|
|
MediaPreparedSources,
|
|
MediaPrivacyService,
|
|
MediaRenderService,
|
|
MediaServiceDeps,
|
|
} from "./media-context.js";
|
|
import { MediaToolMissingError } from "./media-folder-source-probe.js";
|
|
|
|
/*
|
|
* Render service (API side) and render job (worker side).
|
|
*
|
|
* A render runs: claim → prepare redacted sources (privacy slice) → pre-render
|
|
* motion clips → compile → ffmpeg → probe → gate. A blocked gate gets up to
|
|
* MAX_AUTO_FIX_ROUNDS compliance-only repairs (planAutoFix); each repair is
|
|
* stored as a new plan version by the agent and rendered again under the same
|
|
* render id. Only a passed gate moves outputs out of the work directory and
|
|
* marks the render done; every other outcome removes the render's files.
|
|
*
|
|
* Layout: `<spaceId>/renders/<renderId>/{video.mp4,poster.jpg,captions.srt,
|
|
* captions.vtt}` with intermediates in `work/` beside them.
|
|
*
|
|
* Cancel: the API marks the render canceled and aborts it when this process
|
|
* runs it. A worker in another process notices through a failed status update
|
|
* (updateRender returns null once canceled) or the periodic status check.
|
|
*/
|
|
|
|
/** Renders running in this process, so a cancel can kill ffmpeg immediately. */
|
|
export type RunningMediaRenders = Map<string, AbortController>;
|
|
|
|
const runningRenders: RunningMediaRenders = new Map();
|
|
|
|
const PROGRESS_INTERVAL_MS = 1000;
|
|
const CANCEL_POLL_MS = 2000;
|
|
/** Share of the progress bar spent rendering; probing and gating take the rest. */
|
|
const RENDER_SHARE = 0.85;
|
|
|
|
const OUTPUT_FILES = {
|
|
video: "video.mp4",
|
|
poster: "poster.jpg",
|
|
subtitles_srt: "captions.srt",
|
|
subtitles_vtt: "captions.vtt",
|
|
} as const;
|
|
|
|
const OUTPUT_MIME: Record<keyof typeof OUTPUT_FILES, string> = {
|
|
video: "video/mp4",
|
|
poster: "image/jpeg",
|
|
subtitles_srt: "application/x-subrip",
|
|
subtitles_vtt: "text/vtt",
|
|
};
|
|
|
|
const EXTENSION_BY_MIME: Record<string, string> = {
|
|
"image/png": ".png",
|
|
"image/jpeg": ".jpg",
|
|
"image/svg+xml": ".svg",
|
|
"image/webp": ".webp",
|
|
"audio/mpeg": ".mp3",
|
|
"audio/wav": ".wav",
|
|
"audio/x-wav": ".wav",
|
|
"audio/aac": ".aac",
|
|
"audio/mp4": ".m4a",
|
|
"audio/ogg": ".ogg",
|
|
"audio/flac": ".flac",
|
|
"font/ttf": ".ttf",
|
|
"font/otf": ".otf",
|
|
"font/collection": ".ttc",
|
|
};
|
|
|
|
/** A failure whose message is safe to show the user (no paths, no tool output). */
|
|
export class MediaRenderFailure extends Error {
|
|
constructor(message: string) {
|
|
super(message);
|
|
this.name = "MediaRenderFailure";
|
|
}
|
|
}
|
|
|
|
class RenderCanceled extends Error {
|
|
constructor() {
|
|
super("render canceled");
|
|
this.name = "RenderCanceled";
|
|
}
|
|
}
|
|
|
|
function userMessage(error: unknown): string {
|
|
if (error instanceof MediaRenderFailure) return error.message;
|
|
if (error instanceof UnsupportedMediaOperationError) {
|
|
return `${error.operation} is not available on this server.`;
|
|
}
|
|
if (error instanceof MediaToolMissingError) return "ffmpeg is not installed on the media worker.";
|
|
return "Rendering failed. Check the plan and try again.";
|
|
}
|
|
|
|
function renderEvent(render: MediaRender) {
|
|
return {
|
|
type: "media.render" as const,
|
|
renderId: render.id,
|
|
projectId: render.projectId,
|
|
status: render.status,
|
|
progress: render.progress,
|
|
};
|
|
}
|
|
|
|
async function publishRender(deps: MediaServiceDeps, spaceId: string, render: MediaRender) {
|
|
try {
|
|
await deps.publish(spaceId, renderEvent(render));
|
|
} catch (error) {
|
|
getLogger().warn("media render event not published", {
|
|
renderId: render.id,
|
|
reason: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Service
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export function createMediaRenderService(
|
|
deps: MediaServiceDeps,
|
|
options: { running?: RunningMediaRenders } = {},
|
|
): MediaRenderService {
|
|
const running = options.running ?? runningRenders;
|
|
return {
|
|
async start(actor, input) {
|
|
const plan = await deps.repos.getPlan(actor.spaceId, input.planId);
|
|
const { profiles } = await deps.profiles();
|
|
if (!profiles.has(plan.profileId)) {
|
|
throw new InvalidMediaInputError("The plan's platform profile is not available");
|
|
}
|
|
const render = await deps.repos.createRender(actor.spaceId, {
|
|
planId: plan.id,
|
|
quality: input.quality,
|
|
createdBy: actor.userId,
|
|
});
|
|
await deps.jobs.enqueue(mediaRenderJob(render.id));
|
|
await publishRender(deps, actor.spaceId, render);
|
|
return render;
|
|
},
|
|
|
|
async cancel(actor, renderId) {
|
|
const render = await deps.repos.cancelRender(actor.spaceId, renderId);
|
|
running.get(renderId)?.abort();
|
|
await deps.jobs.cancel(mediaRenderJob(renderId).replaceKey!).catch((error: unknown) => {
|
|
getLogger().warn("media render job not dequeued", {
|
|
renderId,
|
|
reason: error instanceof Error ? error.message : String(error),
|
|
});
|
|
});
|
|
await publishRender(deps, actor.spaceId, render);
|
|
return render;
|
|
},
|
|
|
|
async outputPath(actor, renderId, fileName) {
|
|
const record = await deps.repos.getRenderRecord(actor.spaceId, renderId);
|
|
if (record.status !== "done") return null;
|
|
if (path.basename(fileName) !== fileName || fileName.startsWith(".")) return null;
|
|
const outputs = Array.isArray(record.outputs) ? record.outputs : [];
|
|
const recorded = outputs.some(
|
|
(output) =>
|
|
output !== null &&
|
|
typeof output === "object" &&
|
|
!Array.isArray(output) &&
|
|
output.fileName === fileName,
|
|
);
|
|
if (!recorded) return null;
|
|
// Derive the location instead of trusting a stored path.
|
|
const file = await resolveInside(
|
|
deps.mediaRoot,
|
|
`${actor.spaceId}/renders/${renderId}/${fileName}`,
|
|
);
|
|
const found = await stat(file).catch(() => null);
|
|
return found?.isFile() ? file : null;
|
|
},
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Job handler
|
|
// ---------------------------------------------------------------------------
|
|
|
|
interface BrandFiles {
|
|
/** Font artifact id → file. */
|
|
fonts: Map<string, string>;
|
|
fontList: Array<{ family: string; path: string }>;
|
|
logos: string[];
|
|
watermark: string | null;
|
|
lut: string | null;
|
|
}
|
|
|
|
function safeExtension(name: string, mimeType: string): string {
|
|
const fromName = path.extname(name).toLowerCase();
|
|
if (/^\.[a-z0-9]{1,5}$/.test(fromName)) return fromName;
|
|
return EXTENSION_BY_MIME[mimeType.toLowerCase()] ?? ".bin";
|
|
}
|
|
|
|
const even = (value: number) => Math.max(2, Math.floor(value / 2) * 2);
|
|
|
|
function finiteNumber(value: unknown): value is number {
|
|
return typeof value === "number" && Number.isFinite(value);
|
|
}
|
|
|
|
/** Transcript words from a stored TranscribeResult. */
|
|
export function transcriptWords(data: unknown): CaptionWord[] {
|
|
const words = (data as { words?: unknown } | null)?.words;
|
|
if (!Array.isArray(words)) return [];
|
|
return words.flatMap((word) => {
|
|
const w = word as { startMs?: unknown; endMs?: unknown; text?: unknown };
|
|
if (!finiteNumber(w.startMs) || !finiteNumber(w.endMs) || typeof w.text !== "string") {
|
|
return [];
|
|
}
|
|
return [{ startMs: w.startMs, endMs: w.endMs, text: w.text }];
|
|
});
|
|
}
|
|
|
|
type Subject = { tMs: number; cx: number; cy: number; strength: number };
|
|
|
|
/** Reframe subjects from a stored saliency result, or face centres from detections. */
|
|
export function subjectPoints(saliency: unknown, faces: unknown): Subject[] {
|
|
const points = (saliency as { points?: unknown } | null)?.points;
|
|
if (Array.isArray(points)) {
|
|
const parsed = points.flatMap((point) => {
|
|
const p = point as Partial<Subject>;
|
|
return finiteNumber(p.tMs) &&
|
|
finiteNumber(p.cx) &&
|
|
finiteNumber(p.cy) &&
|
|
finiteNumber(p.strength)
|
|
? [{ tMs: p.tMs, cx: p.cx, cy: p.cy, strength: p.strength }]
|
|
: [];
|
|
});
|
|
if (parsed.length > 0) return parsed;
|
|
}
|
|
const frames = (faces as { frames?: unknown } | null)?.frames;
|
|
if (!Array.isArray(frames)) return [];
|
|
return frames.flatMap((frame) => {
|
|
const f = frame as { tMs?: unknown; faces?: unknown };
|
|
if (!finiteNumber(f.tMs) || !Array.isArray(f.faces)) return [];
|
|
const tMs = f.tMs;
|
|
return f.faces.flatMap((face) => {
|
|
const box = (face as { box?: { x?: unknown; y?: unknown; w?: unknown; h?: unknown } }).box;
|
|
const confidence = (face as { confidence?: unknown }).confidence;
|
|
if (!box || !finiteNumber(box.x) || !finiteNumber(box.y)) return [];
|
|
if (!finiteNumber(box.w) || !finiteNumber(box.h)) return [];
|
|
return [
|
|
{
|
|
tMs,
|
|
cx: box.x + box.w / 2,
|
|
cy: box.y + box.h / 2,
|
|
strength: finiteNumber(confidence) ? confidence : 1,
|
|
},
|
|
];
|
|
});
|
|
});
|
|
}
|
|
|
|
export interface MediaRenderHandlerOptions {
|
|
running?: RunningMediaRenders;
|
|
/** How often a running render checks whether another process canceled it. */
|
|
cancelPollMs?: number;
|
|
}
|
|
|
|
/**
|
|
* Worker handler for `media.render`. Never throws for a render failure: the
|
|
* render record carries the outcome, so a job retry has nothing to redo.
|
|
*/
|
|
export function createMediaRenderHandler(
|
|
deps: MediaServiceDeps,
|
|
privacy: MediaPrivacyService,
|
|
options: MediaRenderHandlerOptions = {},
|
|
): MediaJobHandlers["media.render"] {
|
|
const running = options.running ?? runningRenders;
|
|
const pollMs = options.cancelPollMs ?? CANCEL_POLL_MS;
|
|
const { repos } = deps;
|
|
|
|
return async ({ renderId }) => {
|
|
const spaceId = await repos.mediaSpaceOf("render", renderId);
|
|
if (!spaceId) return;
|
|
const claimed = await repos.claimRender(spaceId, renderId);
|
|
if (!claimed) return;
|
|
await publishRender(deps, spaceId, claimed);
|
|
|
|
const controller = new AbortController();
|
|
running.set(renderId, controller);
|
|
const poll = setInterval(() => {
|
|
repos
|
|
.getRender(spaceId, renderId)
|
|
.then((render) => {
|
|
if (render.status === "canceled") controller.abort();
|
|
})
|
|
.catch(() => undefined);
|
|
}, pollMs);
|
|
poll.unref?.();
|
|
const createdArtifacts: Array<{ id: string; storageKey: string }> = [];
|
|
let renderDir: string | null = null;
|
|
let userId = "";
|
|
|
|
try {
|
|
renderDir = await resolveInside(deps.mediaRoot, `${spaceId}/renders/${renderId}`);
|
|
const record = await repos.getRenderRecord(spaceId, renderId);
|
|
userId = record.createdBy;
|
|
await runRender({
|
|
deps,
|
|
privacy,
|
|
spaceId,
|
|
renderId,
|
|
renderDir,
|
|
record,
|
|
signal: controller.signal,
|
|
abort: () => controller.abort(),
|
|
createdArtifacts,
|
|
});
|
|
} catch (error) {
|
|
if (renderDir) await rm(renderDir, { recursive: true, force: true }).catch(() => undefined);
|
|
await removeArtifacts(deps, spaceId, userId, renderId, createdArtifacts);
|
|
const current = await repos.getRender(spaceId, renderId).catch(() => null);
|
|
if (current?.status === "canceled") {
|
|
await publishRender(deps, spaceId, current);
|
|
return;
|
|
}
|
|
getLogger().error("media render failed", {
|
|
renderId,
|
|
reason: error instanceof Error ? error.message : String(error),
|
|
});
|
|
const failed = await repos
|
|
.updateRender(spaceId, renderId, { status: "failed", error: userMessage(error) })
|
|
.catch(() => null);
|
|
if (failed) await publishRender(deps, spaceId, failed);
|
|
} finally {
|
|
clearInterval(poll);
|
|
running.delete(renderId);
|
|
}
|
|
};
|
|
}
|
|
|
|
function artifactContext(spaceId: string, userId: string, renderId: string): AdapterContext {
|
|
return {
|
|
operationId: `media-render:${renderId}`,
|
|
traceId: `media-render:${renderId}`,
|
|
spaceId,
|
|
userId,
|
|
signal: new AbortController().signal,
|
|
};
|
|
}
|
|
|
|
async function removeArtifacts(
|
|
deps: MediaServiceDeps,
|
|
spaceId: string,
|
|
userId: string,
|
|
renderId: string,
|
|
created: Array<{ id: string; storageKey: string }>,
|
|
) {
|
|
for (const artifact of created) {
|
|
await deps.prisma.artifact
|
|
.deleteMany({ where: { id: artifact.id, spaceId } })
|
|
.catch(() => undefined);
|
|
await deps.artifacts
|
|
.remove(artifact.storageKey, artifactContext(spaceId, userId, renderId))
|
|
.catch(() => undefined);
|
|
}
|
|
}
|
|
|
|
type RenderRecord = Awaited<ReturnType<MediaServiceDeps["repos"]["getRenderRecord"]>>;
|
|
|
|
interface RunInput {
|
|
deps: MediaServiceDeps;
|
|
privacy: MediaPrivacyService;
|
|
spaceId: string;
|
|
renderId: string;
|
|
renderDir: string;
|
|
record: RenderRecord;
|
|
signal: AbortSignal;
|
|
abort: () => void;
|
|
createdArtifacts: Array<{ id: string; storageKey: string }>;
|
|
}
|
|
|
|
async function runRender(input: RunInput): Promise<void> {
|
|
const { deps, privacy, spaceId, renderId, renderDir, record, signal } = input;
|
|
const { repos } = deps;
|
|
const actor: MediaActor = { spaceId, userId: record.createdBy };
|
|
const context = artifactContext(spaceId, actor.userId, renderId);
|
|
|
|
/** Apply a patch while the render is active; a null result means it was canceled. */
|
|
const update = async (patch: MediaRenderPatch) => {
|
|
const updated = await repos.updateRender(spaceId, renderId, patch);
|
|
if (!updated) {
|
|
input.abort();
|
|
throw new RenderCanceled();
|
|
}
|
|
return updated;
|
|
};
|
|
const checkCanceled = () => {
|
|
if (signal.aborted) throw new RenderCanceled();
|
|
};
|
|
|
|
let plan = await repos.getPlan(spaceId, record.planId);
|
|
const project = await repos.getProjectRecord(spaceId, record.projectId);
|
|
const entry = (await deps.profiles()).profiles.get(record.profileId);
|
|
if (!entry) throw new MediaRenderFailure("The platform profile is no longer available.");
|
|
const profile = entry.profile;
|
|
const kit = project.brandKitId
|
|
? (await repos.getBrandKit(spaceId, project.brandKitId)).kit
|
|
: null;
|
|
const quality = record.quality as MediaRenderQuality;
|
|
|
|
// A retried job starts from a clean work directory.
|
|
const workDir = path.join(renderDir, "work");
|
|
await rm(renderDir, { recursive: true, force: true });
|
|
await mkdir(workDir, { recursive: true });
|
|
|
|
const fetchArtifact = async (artifactId: string, dir: string, baseName: string) => {
|
|
const row = await deps.prisma.artifact.findFirst({
|
|
where: { id: artifactId, spaceId },
|
|
select: { storageKey: true, name: true, mimeType: true },
|
|
});
|
|
if (!row) return null;
|
|
const bytes = await deps.artifacts.get(row.storageKey, context);
|
|
await mkdir(dir, { recursive: true });
|
|
const file = path.join(dir, `${baseName}${safeExtension(row.name, row.mimeType)}`);
|
|
await writeFile(file, bytes);
|
|
return file;
|
|
};
|
|
const brand = await loadBrandFiles(kit, path.join(workDir, "brand"), fetchArtifact);
|
|
const audioFiles = new Map<string, string>();
|
|
|
|
const alternatives = await alternativeSegments(deps, spaceId, project, plan.edl);
|
|
let round = record.autoFixRound;
|
|
let edl = plan.edl;
|
|
|
|
for (;;) {
|
|
checkCanceled();
|
|
const roundDir = path.join(workDir, `round-${round}`);
|
|
await mkdir(roundDir, { recursive: true });
|
|
|
|
// Fail closed: no redacted sources, no render.
|
|
let prepared: MediaPreparedSources;
|
|
try {
|
|
prepared = await privacy.prepareSources({
|
|
actor,
|
|
projectId: project.id,
|
|
edl,
|
|
quality,
|
|
workDir: roundDir,
|
|
signal,
|
|
});
|
|
} catch (error) {
|
|
checkCanceled();
|
|
getLogger().warn("media render sources not prepared", {
|
|
renderId,
|
|
reason: error instanceof Error ? error.message : String(error),
|
|
});
|
|
throw new MediaRenderFailure(
|
|
error instanceof UnsupportedMediaOperationError
|
|
? `Anonymization needs ${error.operation}, which is not available. Nothing was rendered.`
|
|
: "Anonymization could not be applied. Nothing was rendered.",
|
|
);
|
|
}
|
|
checkCanceled();
|
|
for (const clip of edl.tracks.video) {
|
|
if (!prepared.sources.has(clip.assetId)) {
|
|
throw new MediaRenderFailure(
|
|
"Anonymization did not prepare every clip. Nothing was rendered.",
|
|
);
|
|
}
|
|
}
|
|
|
|
for (const clip of edl.tracks.audio) {
|
|
if (audioFiles.has(clip.artifactId)) continue;
|
|
const file = await fetchArtifact(
|
|
clip.artifactId,
|
|
path.join(workDir, "audio"),
|
|
`audio-${audioFiles.size + 1}`,
|
|
);
|
|
if (!file) throw new MediaRenderFailure("An audio file of the plan is no longer available.");
|
|
audioFiles.set(clip.artifactId, file);
|
|
}
|
|
|
|
const motionClips = await renderMotionClips(deps, edl, kit, brand, roundDir, signal);
|
|
const { transcripts, subjects } = await analysisInputs(deps, spaceId, edl);
|
|
const compile = (videoEncoder: string) =>
|
|
compileRenderPlan({
|
|
edl,
|
|
profile,
|
|
brandKit: kit,
|
|
quality,
|
|
workDir: roundDir,
|
|
sources: prepared.sources,
|
|
transcripts,
|
|
subjects,
|
|
motionClips,
|
|
files: {
|
|
lutPath: brand.lut,
|
|
watermarkPath: brand.watermark,
|
|
fonts: brand.fonts,
|
|
audio: audioFiles,
|
|
logoPath: brand.logos[0] ?? null,
|
|
},
|
|
videoEncoder,
|
|
});
|
|
const renderPlan = await renderWithFallback(input, profile, quality, compile, update);
|
|
|
|
const video = renderPlan.outputs.find((output) => output.kind === "video");
|
|
if (!video) throw new Error("render plan has no video output");
|
|
const ffprobe = await deps.renderer.probe(video.path, { measureLoudness: true, signal });
|
|
// A render whose metadata could not be removed is failed, not gated.
|
|
if (findMetadataLeaks(ffprobe.tags).length > 0) {
|
|
throw new MediaRenderFailure("The export still carried source metadata.");
|
|
}
|
|
for (const output of renderPlan.outputs) {
|
|
if (output.kind !== "poster") continue;
|
|
const poster = await deps.renderer.probe(output.path, { signal });
|
|
if (findMetadataLeaks(poster.tags).length > 0) {
|
|
throw new MediaRenderFailure("The poster still carried source metadata.");
|
|
}
|
|
}
|
|
const gating = await update({ status: "gating", progress: 0.9, ffprobe });
|
|
await publishRender(deps, spaceId, gating);
|
|
|
|
const aiAlterations = [
|
|
...new Set<AiAlteration>([...edl.metadata.aiAlterations, ...prepared.aiAlterations]),
|
|
];
|
|
const rendered: Edl = {
|
|
...edl,
|
|
metadata: {
|
|
...edl.metadata,
|
|
aiAlterations,
|
|
aiAltered: edl.metadata.aiAltered || aiAlterations.length > 0,
|
|
},
|
|
};
|
|
const report = await privacy.gate({
|
|
actor,
|
|
projectId: project.id,
|
|
edl: rendered,
|
|
profile,
|
|
outputPath: video.path,
|
|
ffprobe,
|
|
copy: {
|
|
title: edl.metadata.title ?? null,
|
|
caption: edl.metadata.description ?? null,
|
|
tags: edl.metadata.tags,
|
|
},
|
|
// The publish bundle always carries the AI disclosure file and checklist
|
|
// entry when the profile requires a label and the output is altered.
|
|
bundleHasDisclosure: profile.disclosure.aiAlteredLabelRequired && aiAlterations.length > 0,
|
|
autoFixRound: round,
|
|
signal,
|
|
});
|
|
checkCanceled();
|
|
|
|
if (report.passed) {
|
|
await finish(input, renderPlan, report, plan.id, update, context);
|
|
return;
|
|
}
|
|
|
|
const fix =
|
|
round < MAX_AUTO_FIX_ROUNDS
|
|
? planAutoFix(edl, { ...report, autoFixRound: round }, alternatives)
|
|
: null;
|
|
if (!fix) {
|
|
await rm(renderDir, { recursive: true, force: true });
|
|
const blocked = await update({ status: "blocked", progress: 1, gateReport: report });
|
|
await publishRender(deps, spaceId, blocked);
|
|
return;
|
|
}
|
|
const codes = [
|
|
...new Set(report.issues.filter((i) => i.severity === "block").map((i) => i.code)),
|
|
];
|
|
plan = await repos.createPlanVersion(spaceId, {
|
|
projectId: project.id,
|
|
profileId: plan.profileId,
|
|
edl: fix.edl,
|
|
createdBy: "agent",
|
|
rationale: `Auto-fix ${round + 1}/${MAX_AUTO_FIX_ROUNDS} after the gate blocked (${codes.join(
|
|
", ",
|
|
)}): ${fix.changes.join("; ")}`,
|
|
});
|
|
round += 1;
|
|
edl = plan.edl;
|
|
await rm(roundDir, { recursive: true, force: true });
|
|
const retry = await update({
|
|
status: "rendering",
|
|
progress: 0,
|
|
autoFixRound: round,
|
|
gateReport: report,
|
|
ffprobe: null,
|
|
});
|
|
await publishRender(deps, spaceId, retry);
|
|
}
|
|
}
|
|
|
|
async function loadBrandFiles(
|
|
kit: BrandKit | null,
|
|
dir: string,
|
|
fetchArtifact: (artifactId: string, dir: string, baseName: string) => Promise<string | null>,
|
|
): Promise<BrandFiles> {
|
|
const files: BrandFiles = {
|
|
fonts: new Map(),
|
|
fontList: [],
|
|
logos: [],
|
|
watermark: null,
|
|
lut: null,
|
|
};
|
|
if (!kit) return files;
|
|
// Missing files are skipped here; the compiler fails only if the plan needs one.
|
|
const missing = (what: string) => getLogger().warn("brand kit file unavailable", { what });
|
|
for (const [index, font] of kit.fonts.entries()) {
|
|
const file = await fetchArtifact(font.artifactId, dir, `font-${index + 1}`);
|
|
if (!file) {
|
|
missing("font");
|
|
continue;
|
|
}
|
|
files.fonts.set(font.artifactId, file);
|
|
files.fontList.push({ family: font.family, path: file });
|
|
}
|
|
for (const [index, logo] of kit.logoArtifactIds.entries()) {
|
|
const file = await fetchArtifact(logo, dir, `logo-${index + 1}`);
|
|
if (file) files.logos.push(file);
|
|
else missing("logo");
|
|
}
|
|
const watermarkId = kit.watermark?.artifactId;
|
|
if (watermarkId) {
|
|
files.watermark = await fetchArtifact(watermarkId, dir, "watermark");
|
|
if (!files.watermark) missing("watermark");
|
|
} else if (kit.watermark) {
|
|
files.watermark = files.logos[0] ?? null;
|
|
}
|
|
if (kit.lutArtifactId) {
|
|
files.lut = await fetchArtifact(kit.lutArtifactId, dir, "look");
|
|
if (!files.lut) missing("lut");
|
|
}
|
|
return files;
|
|
}
|
|
|
|
async function renderMotionClips(
|
|
deps: MediaServiceDeps,
|
|
edl: Edl,
|
|
kit: BrandKit | null,
|
|
brand: BrandFiles,
|
|
dir: string,
|
|
signal: AbortSignal,
|
|
): Promise<Map<string, string>> {
|
|
const W = edl.output.width;
|
|
const H = edl.output.height;
|
|
const jobs: Array<{ key: string; template: NonNullable<Edl["intro"]>; w: number; h: number }> =
|
|
[];
|
|
if (edl.intro && edl.intro.durationMs > 0)
|
|
jobs.push({ key: "intro", template: edl.intro, w: W, h: H });
|
|
if (edl.outro && edl.outro.durationMs > 0)
|
|
jobs.push({ key: "outro", template: edl.outro, w: W, h: H });
|
|
for (const overlay of edl.overlays) {
|
|
if (!overlay.motion) continue;
|
|
const box = overlay.box;
|
|
jobs.push({
|
|
key: overlay.id,
|
|
template: overlay.motion,
|
|
w: box ? even(box.w * W) : W,
|
|
h: box ? even(box.h * H) : H,
|
|
});
|
|
}
|
|
const clips = new Map<string, string>();
|
|
if (jobs.length === 0) return clips;
|
|
if (!kit) throw new MediaRenderFailure("Motion templates need a brand kit on the project.");
|
|
for (const job of jobs) {
|
|
// Keys are SafeId values ([a-z0-9_-]), so they are safe file names.
|
|
const result = await deps.motion.render({
|
|
template: job.template,
|
|
brandKit: kit,
|
|
assets: { fonts: brand.fontList, logoPaths: brand.logos },
|
|
width: job.w,
|
|
height: job.h,
|
|
fps: edl.output.fps,
|
|
outPath: path.join(dir, `motion-${job.key}.mov`),
|
|
signal,
|
|
});
|
|
clips.set(job.key, result.path);
|
|
}
|
|
return clips;
|
|
}
|
|
|
|
async function analysisInputs(deps: MediaServiceDeps, spaceId: string, edl: Edl) {
|
|
const transcripts = new Map<string, CaptionWord[]>();
|
|
const subjects = new Map<string, Subject[]>();
|
|
const assetIds = [...new Set(edl.tracks.video.map((clip) => clip.assetId))];
|
|
for (const assetId of assetIds) {
|
|
const [transcript, saliency, faces] = await Promise.all([
|
|
deps.repos.latestAnalysis(spaceId, assetId, "transcript"),
|
|
deps.repos.latestAnalysis(spaceId, assetId, "saliency"),
|
|
deps.repos.latestAnalysis(spaceId, assetId, "faces"),
|
|
]);
|
|
const words = transcriptWords(transcript?.data ?? null);
|
|
if (words.length > 0) transcripts.set(assetId, words);
|
|
const points = subjectPoints(saliency?.data ?? null, faces?.data ?? null);
|
|
if (points.length > 0) subjects.set(assetId, points);
|
|
}
|
|
return { transcripts, subjects };
|
|
}
|
|
|
|
/** Hardware encoders that failed a real render, per renderer, so later renders skip them. */
|
|
const failedEncoders = new WeakMap<object, Set<string>>();
|
|
|
|
export function softwareEncoder(profile: PlatformProfile): string {
|
|
return profile.video.codec === "h264" ? "libx264" : "libx265";
|
|
}
|
|
|
|
/** Final renders prefer a working hardware encoder of the profile's codec; drafts use software. */
|
|
export async function pickEncoder(
|
|
renderer: MediaServiceDeps["renderer"],
|
|
profile: PlatformProfile,
|
|
quality: MediaRenderQuality,
|
|
): Promise<string> {
|
|
const software = softwareEncoder(profile);
|
|
if (quality === "draft") return software;
|
|
const failed = failedEncoders.get(renderer);
|
|
const hardware = await renderer.hardwareEncoders();
|
|
return (
|
|
hardware.find(
|
|
(encoder) => encoder.startsWith(`${profile.video.codec}_`) && !failed?.has(encoder),
|
|
) ?? software
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Run the plan with the preferred encoder. A hardware encoder that fails
|
|
* mid-render (driver, device, or session limits) is retried once in software
|
|
* and skipped by this process afterwards; software failures are final.
|
|
*/
|
|
async function renderWithFallback(
|
|
input: RunInput,
|
|
profile: PlatformProfile,
|
|
quality: MediaRenderQuality,
|
|
compile: (videoEncoder: string) => RenderPlan,
|
|
update: (patch: MediaRenderPatch) => Promise<MediaRender>,
|
|
): Promise<RenderPlan> {
|
|
const { deps, signal } = input;
|
|
const encoder = await pickEncoder(deps.renderer, profile, quality);
|
|
const plan = compile(encoder);
|
|
const software = softwareEncoder(profile);
|
|
try {
|
|
await executePlan(input, plan, update);
|
|
return plan;
|
|
} catch (error) {
|
|
if (encoder === software || signal.aborted || error instanceof RenderCanceled) throw error;
|
|
if (error instanceof MediaToolMissingError) throw error;
|
|
getLogger().warn("media hardware encoder failed; rendering in software", {
|
|
renderId: input.renderId,
|
|
encoder,
|
|
reason: error instanceof Error ? error.message : String(error),
|
|
});
|
|
const failed = failedEncoders.get(deps.renderer) ?? new Set<string>();
|
|
failed.add(encoder);
|
|
failedEncoders.set(deps.renderer, failed);
|
|
}
|
|
const fallback = compile(software);
|
|
await executePlan(input, fallback, update);
|
|
return fallback;
|
|
}
|
|
|
|
/** Segments the auto-fix may swap in: the project's material, never paid-only. */
|
|
async function alternativeSegments(
|
|
deps: MediaServiceDeps,
|
|
spaceId: string,
|
|
project: { assetIds: string[]; collectionIds: string[] },
|
|
edl: Edl,
|
|
) {
|
|
const fromCollections = await deps.repos.collectionAssetIds(spaceId, project.collectionIds);
|
|
const assetIds = [
|
|
...new Set([
|
|
...project.assetIds,
|
|
...fromCollections,
|
|
...edl.tracks.video.map((clip) => clip.assetId),
|
|
]),
|
|
];
|
|
const segments = await deps.repos.listSegments(spaceId, assetIds);
|
|
return segments
|
|
.filter((segment) => !segment.paidOnly)
|
|
.map((segment) => ({
|
|
segmentId: segment.id,
|
|
assetId: segment.assetId,
|
|
startMs: segment.startMs,
|
|
endMs: segment.endMs,
|
|
}));
|
|
}
|
|
|
|
async function executePlan(
|
|
input: RunInput,
|
|
plan: RenderPlan,
|
|
update: (patch: MediaRenderPatch) => Promise<MediaRender>,
|
|
) {
|
|
const { deps, spaceId, signal } = input;
|
|
let lastWrite = Number.NEGATIVE_INFINITY;
|
|
let writing: Promise<void> | null = null;
|
|
let failure: unknown = null;
|
|
const onProgress = (progress: MediaRenderProgress) => {
|
|
const now = deps.now().getTime();
|
|
if (writing || now - lastWrite < PROGRESS_INTERVAL_MS) return;
|
|
lastWrite = now;
|
|
writing = update({ progress: progress.progress * RENDER_SHARE })
|
|
.then((render) => publishRender(deps, spaceId, render))
|
|
.catch((error: unknown) => {
|
|
failure ??= error;
|
|
})
|
|
.finally(() => {
|
|
writing = null;
|
|
});
|
|
};
|
|
try {
|
|
await deps.renderer.render(plan, { signal, onProgress });
|
|
} finally {
|
|
await writing;
|
|
}
|
|
if (failure) throw failure;
|
|
if (signal.aborted) throw new RenderCanceled();
|
|
}
|
|
|
|
async function finish(
|
|
input: RunInput,
|
|
plan: RenderPlan,
|
|
report: GateReport,
|
|
planId: string,
|
|
update: (patch: MediaRenderPatch) => Promise<MediaRender>,
|
|
context: AdapterContext,
|
|
) {
|
|
const { deps, spaceId, renderId, renderDir } = input;
|
|
const outputs: Array<MediaRenderOutput & { path: string; planId: string }> = [];
|
|
for (const output of plan.outputs) {
|
|
const fileName = OUTPUT_FILES[output.kind];
|
|
const relative = `${spaceId}/renders/${renderId}/${fileName}`;
|
|
const destination = await resolveInside(deps.mediaRoot, relative);
|
|
await rename(output.path, destination);
|
|
const { size } = await stat(destination);
|
|
let artifactId: string | null = null;
|
|
if (output.kind !== "video") {
|
|
artifactId = await registerArtifact(input, context, {
|
|
name: fileName,
|
|
mimeType: OUTPUT_MIME[output.kind],
|
|
file: destination,
|
|
});
|
|
}
|
|
outputs.push({
|
|
kind: output.kind,
|
|
artifactId,
|
|
fileName,
|
|
sizeBytes: size,
|
|
path: relative,
|
|
planId,
|
|
});
|
|
}
|
|
const done = await update({
|
|
status: "done",
|
|
progress: 1,
|
|
outputs,
|
|
gateReport: report,
|
|
error: null,
|
|
});
|
|
await rm(path.join(renderDir, "work"), { recursive: true, force: true });
|
|
await publishRender(deps, spaceId, done);
|
|
}
|
|
|
|
async function registerArtifact(
|
|
input: RunInput,
|
|
context: AdapterContext,
|
|
file: { name: string; mimeType: string; file: string },
|
|
): Promise<string> {
|
|
const { deps, spaceId } = input;
|
|
const bytes = new Uint8Array(await readFile(file.file));
|
|
const stored = await deps.artifacts.put(
|
|
{ name: file.name, mimeType: file.mimeType, bytes },
|
|
context,
|
|
);
|
|
try {
|
|
const row = await deps.prisma.artifact.create({
|
|
data: {
|
|
spaceId,
|
|
userId: context.userId,
|
|
name: file.name,
|
|
mimeType: file.mimeType,
|
|
size: bytes.byteLength,
|
|
hash: createHash("sha256").update(bytes).digest("hex"),
|
|
storageKey: stored.id,
|
|
},
|
|
select: { id: true },
|
|
});
|
|
input.createdArtifacts.push({ id: row.id, storageKey: stored.id });
|
|
return row.id;
|
|
} catch (error) {
|
|
await deps.artifacts.remove(stored.id, context).catch(() => undefined);
|
|
throw error;
|
|
}
|
|
}
|