From f3e438444a43b1054200eed09ea48e0d0050077e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 08:52:06 +0000 Subject: [PATCH 1/5] Put voice in the composer's send slot until there is something to send An empty draft shows the mic where the send arrow sits; it opens the voice conversation as the separate Voice button did. As soon as the draft holds text, a chip or an attachment, the slot turns back into Send. This frees the width the extra Voice pill took on small screens. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XcoERsCTbvjqDzE8qC82Wo --- apps/web/e2e/activity-list.spec.ts | 2 +- apps/web/e2e/bot-organization.spec.ts | 25 ++++++--- apps/web/e2e/golden.spec.ts | 19 +++---- apps/web/e2e/peer-messages.spec.ts | 2 +- apps/web/e2e/routine-execution.spec.ts | 2 +- apps/web/e2e/voice.spec.ts | 4 ++ apps/web/src/pages/shell/composer.tsx | 75 ++++++++++++-------------- 7 files changed, 68 insertions(+), 61 deletions(-) diff --git a/apps/web/e2e/activity-list.spec.ts b/apps/web/e2e/activity-list.spec.ts index 05ee392..1c7e360 100644 --- a/apps/web/e2e/activity-list.spec.ts +++ b/apps/web/e2e/activity-list.spec.ts @@ -78,7 +78,7 @@ test("sidebar Now and Recent surface active and terminal runs", async ({ page }, await captureActivitySidebar(page, testInfo, "58-activity-now"); await page.getByRole("button", { name: "Stop", exact: true }).click(); - await expect(page.getByRole("button", { name: "Send" })).toBeVisible({ timeout: 30_000 }); + await expect(page.getByRole("combobox", { name: "Message" })).toBeVisible({ timeout: 30_000 }); await page.reload(); await expect(activityToggle).toHaveAttribute("aria-pressed", "true"); diff --git a/apps/web/e2e/bot-organization.spec.ts b/apps/web/e2e/bot-organization.spec.ts index 510cb52..c32a4cd 100644 --- a/apps/web/e2e/bot-organization.spec.ts +++ b/apps/web/e2e/bot-organization.spec.ts @@ -195,14 +195,23 @@ test("chat composer controls are vertically centered", async ({ page }) => { await signup(page, `composer-layout-${stamp}@rakazo.test`, "password12", "Composer Layout"); await completeOnboarding(page); - const centers = await page.getByTestId("composer-bar").evaluate((composer) => - ["Add", "Message", "Voice", "Send"].map((label) => { - const element = composer.querySelector(`[aria-label="${label}"]`); - if (!element) throw new Error(`Missing composer control: ${label}`); - const box = element.getBoundingClientRect(); - return box.top + box.height / 2; - }), - ); + const bar = page.getByTestId("composer-bar"); + const centersOf = (labels: string[]) => + bar.evaluate( + (composer, controls) => + controls.map((label) => { + const element = composer.querySelector(`[aria-label="${label}"]`); + if (!element) throw new Error(`Missing composer control: ${label}`); + const box = element.getBoundingClientRect(); + return box.top + box.height / 2; + }), + labels, + ); + + const idle = await centersOf(["Add", "Message", "Voice"]); + await page.getByRole("combobox", { name: "Message" }).fill("hello"); + const typing = await centersOf(["Add", "Message", "Send"]); + const centers = [...idle, ...typing]; expect(Math.max(...centers) - Math.min(...centers)).toBeLessThanOrEqual(1); }); diff --git a/apps/web/e2e/golden.spec.ts b/apps/web/e2e/golden.spec.ts index 17ace91..447a582 100644 --- a/apps/web/e2e/golden.spec.ts +++ b/apps/web/e2e/golden.spec.ts @@ -378,12 +378,14 @@ test("sign-in, spawn, and stop work in the shell", async ({ page }, testInfo) => await expect(page.getByText("Messages sent now guide the next turn.")).toHaveCount(0); await expect(page.getByText(/^Steer /)).toHaveCount(0); await expect(composer).toHaveAttribute("placeholder", "Message…"); - await expect(page.getByRole("button", { name: "Send", exact: true })).toBeVisible(); + // An empty draft offers voice in the send slot; typing turns it into Send. + const voice = page + .getByTestId("composer-bar") + .getByRole("button", { name: "Voice", exact: true }); + await expect(voice).toBeVisible(); + await expect(page.getByRole("button", { name: "Send", exact: true })).toHaveCount(0); await composer.fill("Use the newer report and keep the answer short."); - await page.keyboard.press("Tab"); - await expect( - page.getByTestId("composer-bar").getByRole("button", { name: "Voice", exact: true }), - ).toBeFocused(); + await expect(voice).toHaveCount(0); await page.keyboard.press("Tab"); await expect(page.getByRole("button", { name: "Send", exact: true })).toBeFocused(); await page.keyboard.press("Enter"); @@ -393,7 +395,7 @@ test("sign-in, spawn, and stop work in the shell", async ({ page }, testInfo) => await expect(page.getByRole("button", { name: "Stop", exact: true })).toBeVisible(); await captureScreenshot(page, testInfo, "14-active-bot-work"); await page.setViewportSize({ width: 390, height: 844 }); - await expect(page.getByRole("button", { name: "Send", exact: true })).toBeVisible(); + await expect(voice).toBeVisible(); await expect(page.getByRole("button", { name: "Stop", exact: true })).toBeVisible(); await expect(composer).toHaveAttribute("placeholder", "Message…"); await captureScreenshot(page, testInfo, "14-active-bot-work-mobile"); @@ -414,14 +416,13 @@ test("sign-in, spawn, and stop work in the shell", async ({ page }, testInfo) => }); await page.getByRole("button", { name: "Stop", exact: true }).click(); await stopRequestStarted; - await expect(page.getByRole("button", { name: "Send", exact: true })).toBeDisabled(); + await expect(voice).toBeVisible(); await expect(page.getByRole("button", { name: "Stop", exact: true })).toBeDisabled(); releaseStopRequest(); - // Idle Send stays disabled with an empty draft; wait for Stop to leave instead. await expect(page.getByRole("button", { name: "Stop", exact: true })).toHaveCount(0, { timeout: 30_000, }); - await expect(page.getByRole("button", { name: "Send", exact: true })).toBeVisible(); + await expect(voice).toBeVisible(); await page.context().clearCookies(); await page.goto("/sign-in"); diff --git a/apps/web/e2e/peer-messages.spec.ts b/apps/web/e2e/peer-messages.spec.ts index d77da20..8852b55 100644 --- a/apps/web/e2e/peer-messages.spec.ts +++ b/apps/web/e2e/peer-messages.spec.ts @@ -45,7 +45,7 @@ test("shows peer chips in transcript and opens view-only peer chat", async ({ pa ) .toBe(true); - await expect(page.getByRole("button", { name: "Send" })).toBeVisible({ timeout: 60_000 }); + await expect(page.getByRole("combobox", { name: "Message" })).toBeVisible({ timeout: 60_000 }); const transcript = page.getByTestId("transcript"); const chip = transcript diff --git a/apps/web/e2e/routine-execution.spec.ts b/apps/web/e2e/routine-execution.spec.ts index 7f24174..478d054 100644 --- a/apps/web/e2e/routine-execution.spec.ts +++ b/apps/web/e2e/routine-execution.spec.ts @@ -150,7 +150,7 @@ test("routine test-run completes and survives reload", async ({ page }, testInfo await routine.click(); await page.getByRole("button", { name: "Test run" }).click(); await expect(page.getByText(/routine-run-now-ok/i).first()).toBeVisible({ timeout: 30_000 }); - await expect(page.getByRole("button", { name: "Send" })).toBeVisible({ timeout: 30_000 }); + await expect(page.getByRole("combobox", { name: "Message" })).toBeVisible({ timeout: 30_000 }); await captureScreenshot(page, testInfo, "34-routine-run-completed"); await page.reload(); diff --git a/apps/web/e2e/voice.spec.ts b/apps/web/e2e/voice.spec.ts index ee2295f..94e275e 100644 --- a/apps/web/e2e/voice.spec.ts +++ b/apps/web/e2e/voice.spec.ts @@ -54,6 +54,10 @@ test("voice settings connect a key, speak a reply, and open a call", async ({ pa const composer = page.getByPlaceholder(/Message/); await composer.fill("say hello"); + await expect( + page.getByTestId("composer-bar").getByRole("button", { name: "Voice", exact: true }), + ).toHaveCount(0); + await captureScreenshot(page, testInfo, "voice-composer-typing"); await page.keyboard.press("Enter"); // A reply can render more than one text bubble; speak the latest one. const speakReply = page.getByRole("button", { name: "Speak this reply" }).last(); diff --git a/apps/web/src/pages/shell/composer.tsx b/apps/web/src/pages/shell/composer.tsx index df10a7a..6f21915 100644 --- a/apps/web/src/pages/shell/composer.tsx +++ b/apps/web/src/pages/shell/composer.tsx @@ -135,6 +135,38 @@ export const Composer = memo(function Composer({ selectedSkill !== null || selectedMentions.length > 0 || pendingAttachments.length > 0; + // One slot, two jobs: an empty draft talks, a written one sends. + const voiceInSlot = Boolean(onVoice) && !canSend; + const primaryAction = voiceInSlot ? ( + + ) : ( + + ); useEffect(() => { if (!runError || !runErrorId) return; @@ -743,40 +775,9 @@ export const Composer = memo(function Composer({ className="max-h-36 min-h-[28px] min-w-[8rem] flex-1 resize-none overflow-y-auto bg-transparent py-1 text-[16.5px] leading-[1.55] text-foreground outline-none placeholder:text-muted-foreground/75 disabled:opacity-40 font-normal" /> - {onVoice ? ( - - ) : null} {running ? (
- + {primaryAction}
) : ( - + primaryAction )} From c98eb6e843cd1f943045d0684caad60398f17cf6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 10:51:03 +0000 Subject: [PATCH 2/5] Drop the stage's own voice button now that the composer carries the mic The empty-chat stage offered "Start talking" / "Set up voice" above the transcript, a second entry to the same voice conversation the composer's mic now opens. Keep the composer as the single way in. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XcoERsCTbvjqDzE8qC82Wo --- apps/web/src/components/stage/OoStage.tsx | 24 ----------- .../src/components/stage/StageVoiceEntry.tsx | 43 ------------------- apps/web/src/locales/de/messages.po | 9 ---- apps/web/src/locales/en/messages.po | 9 ---- apps/web/src/locales/es/messages.po | 9 ---- apps/web/src/locales/hi/messages.po | 9 ---- apps/web/src/locales/ko/messages.po | 9 ---- apps/web/src/locales/pt-BR/messages.po | 9 ---- apps/web/src/locales/ru/messages.po | 9 ---- apps/web/src/locales/tr/messages.po | 9 ---- apps/web/src/locales/zh-CN/messages.po | 9 ---- apps/web/src/pages/Shell.tsx | 4 -- 12 files changed, 152 deletions(-) delete mode 100644 apps/web/src/components/stage/StageVoiceEntry.tsx diff --git a/apps/web/src/components/stage/OoStage.tsx b/apps/web/src/components/stage/OoStage.tsx index e30c59d..7895b55 100644 --- a/apps/web/src/components/stage/OoStage.tsx +++ b/apps/web/src/components/stage/OoStage.tsx @@ -24,7 +24,6 @@ import { import { personaIdlePrompt, personaThinkingLines } from "../../lib/voice-persona-copy"; import { VoiceAwarenessStrip } from "../voice/VoiceAwarenessStrip"; import { StageHero } from "./StageHero"; -import { StageVoiceEntry } from "./StageVoiceEntry"; import { TextStageCanvas } from "./TextStageCanvas"; export function OoStage({ @@ -37,8 +36,6 @@ export function OoStage({ snapshot, running, hasMessages, - voiceReady = false, - onStartVoice, }: { botId: string; botName: string; @@ -49,8 +46,6 @@ export function OoStage({ snapshot: ThreadSnapshot | null; running: boolean; hasMessages: boolean; - voiceReady?: boolean; - onStartVoice?: () => void; }) { const { t } = useLingui(); const [reducedMotion, setReducedMotion] = useState(false); @@ -242,13 +237,6 @@ export function OoStage({ > {subtitle}

- {onStartVoice ? ( - - ) : null} ) : (

@@ -306,14 +294,10 @@ export function OoGroupStage({ groupName, snapshot, enrichMember, - voiceReady = false, - onStartVoice, }: { groupName?: string; snapshot: ThreadSnapshot | null; enrichMember?: (botId: string) => Pick | undefined; - voiceReady?: boolean; - onStartVoice?: () => void; }) { const { t } = useLingui(); const [reducedMotion, setReducedMotion] = useState(false); @@ -552,14 +536,6 @@ export function OoGroupStage({ {subtitle}

)} - - {onStartVoice && !hasMessages ? ( - - ) : null} ); diff --git a/apps/web/src/components/stage/StageVoiceEntry.tsx b/apps/web/src/components/stage/StageVoiceEntry.tsx deleted file mode 100644 index dd32f8b..0000000 --- a/apps/web/src/components/stage/StageVoiceEntry.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import { Trans, useLingui } from "@lingui/react/macro"; -import { Button } from "@rakazo/ui-web"; -import { Mic } from "lucide-react"; -import { VoiceAuroraRim } from "../voice/VoiceAurora"; - -export function StageVoiceEntry({ - voiceReady, - botColor, - onStartVoice, -}: { - voiceReady: boolean; - botColor?: string; - onStartVoice: () => void; -}) { - const { t } = useLingui(); - - return ( -
- -

- Voice first — transcript below -

-
- ); -} diff --git a/apps/web/src/locales/de/messages.po b/apps/web/src/locales/de/messages.po index 3a4c2c7..f58a927 100644 --- a/apps/web/src/locales/de/messages.po +++ b/apps/web/src/locales/de/messages.po @@ -4539,7 +4539,6 @@ msgstr "Servername" msgid "Server URL" msgstr "Server-URL" -#: src/components/stage/StageVoiceEntry.tsx #: src/pages/shell/composer.tsx msgid "Set up voice" msgstr "Stimme einrichten" @@ -4794,10 +4793,6 @@ msgstr "Chat starten" msgid "Start recording" msgstr "Aufnahme starten" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Start talking" -msgstr "Sprechen starten" - #: src/pages/shell/transcript.tsx msgid "Start the conversation — {activeName} is on stage above." msgstr "Starte die Unterhaltung — {activeName} steht oben auf der Bühne." @@ -5318,10 +5313,6 @@ msgstr "Stimme" msgid "Voice \"{0}\" not found." msgstr "Stimme „{0}“ nicht gefunden." -#: src/components/stage/StageVoiceEntry.tsx -msgid "Voice first — transcript below" -msgstr "Primär per Stimme — Transkript unten" - #: src/components/onboarding/OnboardingVoiceStep.tsx msgid "Voice is not available yet. You can turn it on later in settings." msgstr "Die Stimme ist noch nicht verfügbar. Du kannst sie später in den Einstellungen aktivieren." diff --git a/apps/web/src/locales/en/messages.po b/apps/web/src/locales/en/messages.po index 85a0267..621cc49 100644 --- a/apps/web/src/locales/en/messages.po +++ b/apps/web/src/locales/en/messages.po @@ -4539,7 +4539,6 @@ msgstr "Server name" msgid "Server URL" msgstr "Server URL" -#: src/components/stage/StageVoiceEntry.tsx #: src/pages/shell/composer.tsx msgid "Set up voice" msgstr "Set up voice" @@ -4794,10 +4793,6 @@ msgstr "Start chat" msgid "Start recording" msgstr "Start recording" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Start talking" -msgstr "Start talking" - #: src/pages/shell/transcript.tsx msgid "Start the conversation — {activeName} is on stage above." msgstr "Start the conversation — {activeName} is on stage above." @@ -5318,10 +5313,6 @@ msgstr "Voice" msgid "Voice \"{0}\" not found." msgstr "Voice \"{0}\" not found." -#: src/components/stage/StageVoiceEntry.tsx -msgid "Voice first — transcript below" -msgstr "Voice first — transcript below" - #: src/components/onboarding/OnboardingVoiceStep.tsx msgid "Voice is not available yet. You can turn it on later in settings." msgstr "Voice is not available yet. You can turn it on later in settings." diff --git a/apps/web/src/locales/es/messages.po b/apps/web/src/locales/es/messages.po index 214143b..65bb0af 100644 --- a/apps/web/src/locales/es/messages.po +++ b/apps/web/src/locales/es/messages.po @@ -4539,7 +4539,6 @@ msgstr "Nombre del servidor" msgid "Server URL" msgstr "URL del servidor" -#: src/components/stage/StageVoiceEntry.tsx #: src/pages/shell/composer.tsx msgid "Set up voice" msgstr "" @@ -4794,10 +4793,6 @@ msgstr "" msgid "Start recording" msgstr "Iniciar grabación" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Start talking" -msgstr "" - #: src/pages/shell/transcript.tsx msgid "Start the conversation — {activeName} is on stage above." msgstr "" @@ -5318,10 +5313,6 @@ msgstr "Voz" msgid "Voice \"{0}\" not found." msgstr "" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Voice first — transcript below" -msgstr "" - #: src/components/onboarding/OnboardingVoiceStep.tsx msgid "Voice is not available yet. You can turn it on later in settings." msgstr "" diff --git a/apps/web/src/locales/hi/messages.po b/apps/web/src/locales/hi/messages.po index 51e1b09..5f5c6d7 100644 --- a/apps/web/src/locales/hi/messages.po +++ b/apps/web/src/locales/hi/messages.po @@ -4539,7 +4539,6 @@ msgstr "सर्वर का नाम" msgid "Server URL" msgstr "सर्वर URL" -#: src/components/stage/StageVoiceEntry.tsx #: src/pages/shell/composer.tsx msgid "Set up voice" msgstr "" @@ -4794,10 +4793,6 @@ msgstr "" msgid "Start recording" msgstr "रिकॉर्डिंग शुरू करें" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Start talking" -msgstr "" - #: src/pages/shell/transcript.tsx msgid "Start the conversation — {activeName} is on stage above." msgstr "" @@ -5318,10 +5313,6 @@ msgstr "आवाज़" msgid "Voice \"{0}\" not found." msgstr "" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Voice first — transcript below" -msgstr "" - #: src/components/onboarding/OnboardingVoiceStep.tsx msgid "Voice is not available yet. You can turn it on later in settings." msgstr "" diff --git a/apps/web/src/locales/ko/messages.po b/apps/web/src/locales/ko/messages.po index 478118c..6431768 100644 --- a/apps/web/src/locales/ko/messages.po +++ b/apps/web/src/locales/ko/messages.po @@ -4539,7 +4539,6 @@ msgstr "서버 이름" msgid "Server URL" msgstr "서버 URL" -#: src/components/stage/StageVoiceEntry.tsx #: src/pages/shell/composer.tsx msgid "Set up voice" msgstr "" @@ -4794,10 +4793,6 @@ msgstr "" msgid "Start recording" msgstr "작업 녹화 시작" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Start talking" -msgstr "" - #: src/pages/shell/transcript.tsx msgid "Start the conversation — {activeName} is on stage above." msgstr "" @@ -5318,10 +5313,6 @@ msgstr "음성" msgid "Voice \"{0}\" not found." msgstr "" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Voice first — transcript below" -msgstr "" - #: src/components/onboarding/OnboardingVoiceStep.tsx msgid "Voice is not available yet. You can turn it on later in settings." msgstr "" diff --git a/apps/web/src/locales/pt-BR/messages.po b/apps/web/src/locales/pt-BR/messages.po index 0b3193e..e61b2ed 100644 --- a/apps/web/src/locales/pt-BR/messages.po +++ b/apps/web/src/locales/pt-BR/messages.po @@ -4539,7 +4539,6 @@ msgstr "Nome do servidor" msgid "Server URL" msgstr "URL do servidor" -#: src/components/stage/StageVoiceEntry.tsx #: src/pages/shell/composer.tsx msgid "Set up voice" msgstr "" @@ -4794,10 +4793,6 @@ msgstr "" msgid "Start recording" msgstr "Começar a gravar" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Start talking" -msgstr "" - #: src/pages/shell/transcript.tsx msgid "Start the conversation — {activeName} is on stage above." msgstr "" @@ -5318,10 +5313,6 @@ msgstr "Voz" msgid "Voice \"{0}\" not found." msgstr "" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Voice first — transcript below" -msgstr "" - #: src/components/onboarding/OnboardingVoiceStep.tsx msgid "Voice is not available yet. You can turn it on later in settings." msgstr "" diff --git a/apps/web/src/locales/ru/messages.po b/apps/web/src/locales/ru/messages.po index 026f751..ff7efaf 100644 --- a/apps/web/src/locales/ru/messages.po +++ b/apps/web/src/locales/ru/messages.po @@ -4539,7 +4539,6 @@ msgstr "Имя сервера" msgid "Server URL" msgstr "URL-адрес сервера" -#: src/components/stage/StageVoiceEntry.tsx #: src/pages/shell/composer.tsx msgid "Set up voice" msgstr "" @@ -4794,10 +4793,6 @@ msgstr "" msgid "Start recording" msgstr "Начать запись" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Start talking" -msgstr "" - #: src/pages/shell/transcript.tsx msgid "Start the conversation — {activeName} is on stage above." msgstr "" @@ -5318,10 +5313,6 @@ msgstr "Голос" msgid "Voice \"{0}\" not found." msgstr "" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Voice first — transcript below" -msgstr "" - #: src/components/onboarding/OnboardingVoiceStep.tsx msgid "Voice is not available yet. You can turn it on later in settings." msgstr "" diff --git a/apps/web/src/locales/tr/messages.po b/apps/web/src/locales/tr/messages.po index 9b782c4..de6e0b8 100644 --- a/apps/web/src/locales/tr/messages.po +++ b/apps/web/src/locales/tr/messages.po @@ -4539,7 +4539,6 @@ msgstr "Sunucu adı" msgid "Server URL" msgstr "Sunucu URL'si" -#: src/components/stage/StageVoiceEntry.tsx #: src/pages/shell/composer.tsx msgid "Set up voice" msgstr "" @@ -4794,10 +4793,6 @@ msgstr "" msgid "Start recording" msgstr "Kaydı başlat" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Start talking" -msgstr "" - #: src/pages/shell/transcript.tsx msgid "Start the conversation — {activeName} is on stage above." msgstr "" @@ -5318,10 +5313,6 @@ msgstr "Ses" msgid "Voice \"{0}\" not found." msgstr "" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Voice first — transcript below" -msgstr "" - #: src/components/onboarding/OnboardingVoiceStep.tsx msgid "Voice is not available yet. You can turn it on later in settings." msgstr "" diff --git a/apps/web/src/locales/zh-CN/messages.po b/apps/web/src/locales/zh-CN/messages.po index 4932ef1..4ea18b7 100644 --- a/apps/web/src/locales/zh-CN/messages.po +++ b/apps/web/src/locales/zh-CN/messages.po @@ -4539,7 +4539,6 @@ msgstr "服务器名称" msgid "Server URL" msgstr "服务器 URL" -#: src/components/stage/StageVoiceEntry.tsx #: src/pages/shell/composer.tsx msgid "Set up voice" msgstr "" @@ -4794,10 +4793,6 @@ msgstr "" msgid "Start recording" msgstr "开始录制" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Start talking" -msgstr "" - #: src/pages/shell/transcript.tsx msgid "Start the conversation — {activeName} is on stage above." msgstr "" @@ -5318,10 +5313,6 @@ msgstr "语音" msgid "Voice \"{0}\" not found." msgstr "" -#: src/components/stage/StageVoiceEntry.tsx -msgid "Voice first — transcript below" -msgstr "" - #: src/components/onboarding/OnboardingVoiceStep.tsx msgid "Voice is not available yet. You can turn it on later in settings." msgstr "" diff --git a/apps/web/src/pages/Shell.tsx b/apps/web/src/pages/Shell.tsx index e023af9..1fcfc84 100644 --- a/apps/web/src/pages/Shell.tsx +++ b/apps/web/src/pages/Shell.tsx @@ -3323,8 +3323,6 @@ export function ShellPage() { snapshot={activeSnapshot} running={transcriptRunning} hasMessages={transcriptMessages.length > 0} - voiceReady={Boolean(voiceStatus?.ready)} - onStartVoice={startVoiceEncounter} /> ) : null} {!callOpen && inGroup ? ( @@ -3332,8 +3330,6 @@ export function ShellPage() { groupName={activeGroup?.name ?? activeSnapshot?.groupName} snapshot={activeSnapshot} enrichMember={enrichStageMember} - voiceReady={Boolean(voiceStatus?.ready)} - onStartVoice={startVoiceEncounter} /> ) : null} Date: Mon, 5 Oct 2026 14:46:20 +0200 Subject: [PATCH 3/5] fix(evo-deploy): prefer the fetched remote branch when resolving the deploy ref (#160) A manual deploy.sh run without a SHA resolved the local 'main' first, which lags behind origin/main in the deploy checkout, and redeployed the old commit. Resolve / first and fall back to the ref itself for SHAs and tags. Claude-Session: https://claude.ai/code/session_01YMUEBB4pr4TtGSMhEAKonM Co-authored-by: Claude --- ops/evo-deploy/deploy.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ops/evo-deploy/deploy.sh b/ops/evo-deploy/deploy.sh index 054261c..3abcf17 100755 --- a/ops/evo-deploy/deploy.sh +++ b/ops/evo-deploy/deploy.sh @@ -286,10 +286,13 @@ git_fetch_and_checkout() { log "previous HEAD: $previous_sha" log "fetching $EVO_GIT_REMOTE…" run git -C "$EVO_DEPLOY_DIR" fetch "$EVO_GIT_REMOTE" --prune >&2 - if git -C "$EVO_DEPLOY_DIR" rev-parse --verify "${EVO_DEPLOY_REF}^{commit}" >/dev/null 2>&1; then - target_sha="$(git -C "$EVO_DEPLOY_DIR" rev-parse "${EVO_DEPLOY_REF}^{commit}")" - elif git -C "$EVO_DEPLOY_DIR" rev-parse --verify "${EVO_GIT_REMOTE}/${EVO_DEPLOY_REF}^{commit}" >/dev/null 2>&1; then + # Prefer the freshly fetched remote branch: the local `main` in the deploy + # checkout never moves (we reset HEAD, not the branch), so resolving it first + # would redeploy a stale commit. + if git -C "$EVO_DEPLOY_DIR" rev-parse --verify "${EVO_GIT_REMOTE}/${EVO_DEPLOY_REF}^{commit}" >/dev/null 2>&1; then target_sha="$(git -C "$EVO_DEPLOY_DIR" rev-parse "${EVO_GIT_REMOTE}/${EVO_DEPLOY_REF}^{commit}")" + elif git -C "$EVO_DEPLOY_DIR" rev-parse --verify "${EVO_DEPLOY_REF}^{commit}" >/dev/null 2>&1; then + target_sha="$(git -C "$EVO_DEPLOY_DIR" rev-parse "${EVO_DEPLOY_REF}^{commit}")" else die "cannot resolve ref: $EVO_DEPLOY_REF" fi From e55207f5d2615b62a19083bcc85f9c8e71461677 Mon Sep 17 00:00:00 2001 From: cynarAI <155274210+cynarAI@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:44:51 +0200 Subject: [PATCH 4/5] Open the side panel without booting the computer (#162) Showing the right side panel (computer preview and routines) used to boot a stopped computer right away. A boot is slow and heavy on the host, and most of the time the panel is opened for routines, not the screen. The panel now only reconnects a computer that already runs; a resting computer shows its Open action without hover, and Open still boots it. A stopped computer is no longer treated as needing maintenance. Claude-Session: https://claude.ai/code/session_01TafdPTJdWxTYt9SYbDuKQ8 Co-authored-by: Claude --- apps/web/e2e/teach-task.spec.ts | 8 +++-- apps/web/src/lib/thread-events.test.ts | 31 +++++++------------ apps/web/src/lib/thread-events.ts | 20 +++++------- apps/web/src/pages/Shell.tsx | 27 ++++------------ .../pages/shell/computer-panel-preview.tsx | 9 +++++- 5 files changed, 40 insertions(+), 55 deletions(-) diff --git a/apps/web/e2e/teach-task.spec.ts b/apps/web/e2e/teach-task.spec.ts index c4f456d..835d31c 100644 --- a/apps/web/e2e/teach-task.spec.ts +++ b/apps/web/e2e/teach-task.spec.ts @@ -1,5 +1,5 @@ import { expect, test } from "@playwright/test"; -import { captureScreenshot, completeOnboarding, signup } from "./helpers"; +import { activeBotId, captureScreenshot, completeOnboarding, rpc, signup } from "./helpers"; test("teach a task records interaction and saves a draft", async ({ page }, testInfo) => { const stamp = Date.now(); @@ -17,11 +17,15 @@ test("teach a task records interaction and saves a draft", async ({ page }, test await expect(sidePanel.getByRole("button", { name: "Take control" })).toHaveCount(0); await expect(sidePanel.getByTestId("computer-more-button")).toHaveCount(0); await expect(sidePanel.getByTestId("computer-preview")).toBeVisible(); + // Showing the panel never boots the computer; its Open action is visible without hover. + const openButton = sidePanel.getByTestId("computer-preview-open"); + await expect(openButton).toBeVisible(); + const botId = activeBotId(page); + expect((await rpc<{ state: string }>(page, "computer/status", { botId })).state).toBe("stopped"); await captureScreenshot(page, testInfo, "teach-sidepanel-simple"); const preview = sidePanel.getByTestId("computer-preview"); await preview.hover(); - const openButton = sidePanel.getByTestId("computer-preview-open"); await expect(openButton).toBeVisible(); await expect(openButton.getByText("Open", { exact: true })).toBeVisible(); await captureScreenshot(page, testInfo, "teach-sidepanel-open-hover"); diff --git a/apps/web/src/lib/thread-events.test.ts b/apps/web/src/lib/thread-events.test.ts index ab0da23..22c3b8d 100644 --- a/apps/web/src/lib/thread-events.test.ts +++ b/apps/web/src/lib/thread-events.test.ts @@ -9,9 +9,8 @@ import { activeThreadRuns, applyThreadSendReceipt, clearActiveThreadRuns, - computerPanelAutoBoot, - computerPanelAutoUsesBoot, computerPanelNeedsMaintenance, + computerPanelReconnects, computerTakeoverBlocked, isThreadSnapshotEvent, mergeThreadSnapshot, @@ -1513,25 +1512,19 @@ describe("computer event reduction", () => { }); }); - it("auto-boots stopped computers and recovers a running screen that has no URL", () => { - expect(computerPanelAutoBoot("stopped")).toBe("boot"); - expect(computerPanelAutoBoot("error")).toBe("boot"); - expect(computerPanelAutoBoot(undefined)).toBe("boot"); - expect(computerPanelAutoBoot("running", "https://screen.example")).toBe("wait"); - expect(computerPanelAutoBoot("running", null)).toBe("recover-screen"); - expect(computerPanelAutoBoot("booting")).toBe("wait"); - expect(computerPanelAutoBoot("suspended")).toBe("wait"); + it("reconnects only a running screen that has no URL and never boots on open", () => { + expect(computerPanelReconnects("stopped")).toBe(false); + expect(computerPanelReconnects("error")).toBe(false); + expect(computerPanelReconnects(undefined)).toBe(false); + expect(computerPanelReconnects("booting")).toBe(false); + expect(computerPanelReconnects("suspended")).toBe(false); + expect(computerPanelReconnects("running", "https://screen.example")).toBe(false); + expect(computerPanelReconnects("running", null)).toBe(true); }); - it("maps recover-screen to computer.boot, not computer.recover", () => { - expect(computerPanelAutoUsesBoot("recover-screen")).toBe(true); - expect(computerPanelAutoUsesBoot("boot")).toBe(true); - expect(computerPanelAutoUsesBoot("wait")).toBe(false); - }); - - it("shows maintenance only after a stopped or errored computer finishes booting", () => { + it("shows maintenance only for an errored computer that is not booting", () => { expect(computerPanelNeedsMaintenance("error", false)).toBe(true); - expect(computerPanelNeedsMaintenance("stopped", false)).toBe(true); + expect(computerPanelNeedsMaintenance("stopped", false)).toBe(false); expect(computerPanelNeedsMaintenance("error", true)).toBe(false); expect(computerPanelNeedsMaintenance("running", false)).toBe(false); expect(computerPanelNeedsMaintenance(undefined, false)).toBe(false); @@ -1541,7 +1534,7 @@ describe("computer event reduction", () => { const panel = "computer"; const booting = false; const showInSidePanel = (computerOpen: boolean) => - panel === "computer" && !computerOpen && computerPanelNeedsMaintenance("stopped", booting); + panel === "computer" && !computerOpen && computerPanelNeedsMaintenance("error", booting); expect(showInSidePanel(false)).toBe(true); expect(showInSidePanel(true)).toBe(false); diff --git a/apps/web/src/lib/thread-events.ts b/apps/web/src/lib/thread-events.ts index 705e910..82f1775 100644 --- a/apps/web/src/lib/thread-events.ts +++ b/apps/web/src/lib/thread-events.ts @@ -539,27 +539,23 @@ export function computerTakeoverBlocked( return isActive(runStatus as RunStatus); } -export function computerPanelAutoBoot( +/** + * Opening the side panel only reconnects a computer that already runs. Starting a stopped one + * is the user's choice (Open), because a boot is slow and heavy on the host. + * The reconnect must use computer.boot — never computer.recover (that destroys the sandbox). + */ +export function computerPanelReconnects( state: ComputerStatus["state"] | undefined, screenUrl?: string | null, -): "boot" | "recover-screen" | "wait" { - if (state === "booting" || state === "suspended") return "wait"; - if (state === "running") return screenUrl ? "wait" : "recover-screen"; - return "boot"; -} - -/** Auto panel reconnect must use computer.boot — never computer.recover (that destroys the sandbox). */ -export function computerPanelAutoUsesBoot( - action: ReturnType, ): boolean { - return action === "boot" || action === "recover-screen"; + return state === "running" && !screenUrl; } export function computerPanelNeedsMaintenance( state: ComputerStatus["state"] | undefined, booting: boolean, ): boolean { - return !booting && (state === "error" || state === "stopped"); + return !booting && state === "error"; } export function reduceComputerStatus( diff --git a/apps/web/src/pages/Shell.tsx b/apps/web/src/pages/Shell.tsx index 1fcfc84..765575c 100644 --- a/apps/web/src/pages/Shell.tsx +++ b/apps/web/src/pages/Shell.tsx @@ -124,8 +124,7 @@ import { activeThreadRuns, applyThreadSendReceipt, clearActiveThreadRuns, - computerPanelAutoBoot, - computerPanelAutoUsesBoot, + computerPanelReconnects, computerTakeoverBlocked, isComputerStatusEvent, prependThreadMessagePage, @@ -551,7 +550,6 @@ export function ShellPage() { outputTokens: number; runs: number; } | null>(null); - const autoBooted = useRef(null); const routineSavePending = useRef(false); const webhookSecretProvisionRef = useRef(new Map>()); const ensureWebhookSecret = (botId: string) => @@ -2503,11 +2501,7 @@ export function ShellPage() { } useEffect(() => { - if (panel !== "computer") { - autoBooted.current = null; - return; - } - if (!active) return; + if (panel !== "computer" || !active) return; const botId = active.id; let cancelled = false; void (async () => { @@ -2518,19 +2512,10 @@ export function ShellPage() { const state = snap?.computer?.state; const screen = state === "running" ? await refreshComputerScreen(botId) : null; if (cancelled || activeBotId.current !== botId) return; - const action = computerPanelAutoBoot(state, screen); - if (action === "wait") { - if (state === "running") autoBooted.current = botId; - return; - } - if (action === "boot" && autoBooted.current === botId) return; - autoBooted.current = botId; - if (!computerPanelAutoUsesBoot(action)) return; - await bootComputer({ - takeControl: false, - overlay: action === "boot", - force: true, - }).catch(() => undefined); + if (!computerPanelReconnects(state, screen)) return; + await bootComputer({ takeControl: false, overlay: false, force: true }).catch( + () => undefined, + ); })(); return () => { cancelled = true; diff --git a/apps/web/src/pages/shell/computer-panel-preview.tsx b/apps/web/src/pages/shell/computer-panel-preview.tsx index 93f0da1..c591690 100644 --- a/apps/web/src/pages/shell/computer-panel-preview.tsx +++ b/apps/web/src/pages/shell/computer-panel-preview.tsx @@ -35,6 +35,12 @@ export function ComputerPanelPreview({ onOpen: () => void; }) { const { t } = useLingui(); + // Opening the panel never starts the computer, so a resting one shows its Open action outright. + const resting = + !booting && + computer?.state !== "running" && + computer?.state !== "booting" && + !computersAreUnavailable(sandboxProvider); return ( <>
From 23d70971f8f76e685e3645865e44fea58a2c4b04 Mon Sep 17 00:00:00 2001 From: cynarAI <155274210+cynarAI@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:49:18 +0200 Subject: [PATCH 5/5] feat(media): export videos for named platforms and check each file against its limits (#158) * Export videos for named platforms and check each file against its limits Exports only knew aspect ratios, so a cut could still be rejected or degraded on upload: too long for the platform, over its size limit, or far off its loudness. export_video now takes platforms (TikTok, Instagram Reels and Feed, YouTube Shorts and YouTube, Facebook Reels, LinkedIn, X, Pinterest, Snapchat Spotlight), up to six targets with the presets. Each platform is rendered to its frame with the export's fit, its frame rate cap, a bitrate cap that also keeps the file under the size limit, and its loudness target in one loudnorm pass. Every render is probed and measured afterwards. A file over the size limit is encoded once more at a bitrate scaled from what it averaged, and audio more than 2 LU off the target gets one linear loudness pass. The result is stored on the output as a pass, warn or fail check with language-neutral issue codes. A platform the video is too long or too short for is not rendered: the card lists it with the reason, the other targets still render, and the bot is told to trim first. The limits come from the earlier profile research, with its sources, review date and unverified flag; content, disclosure and caption rules are left out because they need inference or a review flow. Preset definitions move to their own module so the platform schemas can use their aspect ratios without an import cycle. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u * Offer platforms in the media card's export menu and show their checks The card's Export menu now lists the platforms first, by brand name and untranslated, then the aspect ratios after a separator; the platform an output was made for is left out, like its own shape. A platform export shows the platform's name under the player, or, when its check did not pass, one line for the first issue instead ("Too long for TikTok (max 10:00)"), in the warning or destructive color. Skipped platforms get the same kind of line, and they stand in for the error of a job that skipped every platform, so the card does not say the same thing twice in two languages. The E2E spec seeds a platform export with a failed check and a skipped platform, and captures the card and the open menu. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u * fix(media): keep one skipped entry per platform on the card A job that recovers after a crash runs again and reports the same skipped platforms; the newest reason per platform now replaces the earlier one instead of listing it twice. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u --------- Co-authored-by: Claude --- apps/web/e2e/media-card.spec.ts | 141 +++++++-- .../src/components/media/MediaCard.test.tsx | 63 ++++ apps/web/src/components/media/MediaCard.tsx | 55 +++- .../components/media/media-platforms.test.ts | 75 +++++ .../src/components/media/media-platforms.ts | 72 +++++ apps/web/src/locales/de/messages.po | 40 +++ apps/web/src/locales/en/messages.po | 40 +++ apps/web/src/locales/es/messages.po | 40 +++ apps/web/src/locales/hi/messages.po | 40 +++ apps/web/src/locales/ko/messages.po | 40 +++ apps/web/src/locales/pt-BR/messages.po | 40 +++ apps/web/src/locales/ru/messages.po | 40 +++ apps/web/src/locales/tr/messages.po | 40 +++ apps/web/src/locales/zh-CN/messages.po | 40 +++ packages/adapter-kit/src/media.ts | 7 + packages/adapters/src/builtin-tools.ts | 8 +- .../src/media-runtime.postgres.test.ts | 19 ++ packages/adapters/src/media-runtime.ts | 55 +++- packages/adapters/src/media/ffmpeg.test.ts | 71 +++++ packages/adapters/src/media/ffmpeg.ts | 93 ++++++ .../src/media/image-operations.test.ts | 1 + .../adapters/src/media/media-test-support.ts | 12 +- .../src/media/platform-export.test.ts | 188 ++++++++++++ .../adapters/src/media/platform-export.ts | 221 ++++++++++++++ .../adapters/src/media/transcription.test.ts | 1 + .../src/media/video-edit-plan.test.ts | 84 +++++- .../adapters/src/media/video-edit-plan.ts | 81 ++++- .../media/video-operations.postgres.test.ts | 57 ++++ .../src/media/video-operations.test.ts | 95 +++++- .../adapters/src/media/video-operations.ts | 234 +++++++++++++-- packages/contracts/src/index.ts | 2 + packages/contracts/src/media-generation.ts | 2 +- .../contracts/src/media-platforms.test.ts | 69 +++++ packages/contracts/src/media-platforms.ts | 282 ++++++++++++++++++ packages/contracts/src/media-presets.ts | 28 ++ packages/contracts/src/media-video.test.ts | 29 +- packages/contracts/src/media-video.ts | 30 +- packages/contracts/src/media.ts | 38 +-- 38 files changed, 2363 insertions(+), 110 deletions(-) create mode 100644 apps/web/src/components/media/media-platforms.test.ts create mode 100644 apps/web/src/components/media/media-platforms.ts create mode 100644 packages/adapters/src/media/platform-export.test.ts create mode 100644 packages/adapters/src/media/platform-export.ts create mode 100644 packages/contracts/src/media-platforms.test.ts create mode 100644 packages/contracts/src/media-platforms.ts create mode 100644 packages/contracts/src/media-presets.ts diff --git a/apps/web/e2e/media-card.spec.ts b/apps/web/e2e/media-card.spec.ts index 3b46bbe..6697d4a 100644 --- a/apps/web/e2e/media-card.spec.ts +++ b/apps/web/e2e/media-card.spec.ts @@ -1,3 +1,6 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; import type { Page } from "@playwright/test"; import { expect, test } from "@playwright/test"; import { @@ -92,6 +95,38 @@ async function seedMediaMessage(page: Page, botId: string, block: () => MediaBlo } } +/** CI has no ffmpeg or transcription server; the card asks what the space can run. */ +async function offerFollowUps(page: Page) { + await page.route("**/rpc/media/capabilities", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ + json: { + generate_image: false, + generate_video: false, + edit_image: false, + edit_video: false, + export_video: true, + transcribe: true, + }, + }), + }), + ); +} + +/** Records follow-up jobs the card starts instead of running them. */ +async function recordStartedJobs(page: Page): Promise { + const started: unknown[] = []; + await page.route("**/rpc/media/jobs/start", async (route) => { + started.push((route.request().postDataJSON() as { json: unknown }).json); + await route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }), + }); + }); + return started; +} + test("renders a media job card while it runs and when it is done", async ({ page }, testInfo) => { const stamp = Date.now(); await signup(page, `media-card-${stamp}@rakazo.test`, "password12", "Media Card"); @@ -150,30 +185,8 @@ test("renders a media job card while it runs and when it is done", async ({ page ], }, ); - // CI has no transcription server; the card asks what the space can run. - await page.route("**/rpc/media/capabilities", (route) => - route.fulfill({ - contentType: "application/json", - body: JSON.stringify({ - json: { - generate_image: false, - generate_video: false, - edit_image: false, - edit_video: false, - export_video: true, - transcribe: true, - }, - }), - }), - ); - const started: unknown[] = []; - await page.route("**/rpc/media/jobs/start", async (route) => { - started.push((route.request().postDataJSON() as { json: unknown }).json); - await route.fulfill({ - contentType: "application/json", - body: JSON.stringify({ json: { jobId: "media-job-2", messageId: "media-message-next" } }), - }); - }); + await offerFollowUps(page); + const started = await recordStartedJobs(page); await page.reload(); const card = page.getByTestId("media-card"); @@ -209,3 +222,83 @@ test("renders a media job card while it runs and when it is done", async ({ page input: { artifact_id: voice.id, title: "voice-note.wav" }, }); }); + +test("a platform export names its platform, shows a failed check and exports for others", async ({ + page, +}, testInfo) => { + const stamp = Date.now(); + await signup(page, `media-platform-${stamp}@rakazo.test`, "password12", "Media Platform"); + await completeOnboarding(page); + const botId = activeBotId(page); + + const fixture = path.join( + path.dirname(fileURLToPath(import.meta.url)), + "fixtures", + "media-clip.mp4", + ); + const clip = await rpc<{ id: string; size: number }>(page, "artifacts/create", { + botId, + name: "clip-tiktok.mp4", + mimeType: "video/mp4", + contentBase64: readFileSync(fixture).toString("base64"), + }); + await seedMediaMessage(page, botId, () => ({ + kind: "media", + jobId: "media-job-platforms", + operation: "export_video", + status: "done", + title: "TikTok · Snapchat Spotlight", + outputs: [ + { + artifactId: clip.id, + kind: "video", + mimeType: "video/mp4", + name: "clip-tiktok.mp4", + size: clip.size, + width: 1080, + height: 1920, + durationMs: 1000, + platform: "tiktok", + check: { + status: "fail", + issues: [{ code: "file_too_large", maxMb: 287, actualMb: 301.2 }], + }, + }, + ], + skipped: [ + { + platform: "snapchat-spotlight", + issues: [{ code: "too_short", minMs: 5000, actualMs: 1000 }], + }, + ], + })); + await offerFollowUps(page); + const started = await recordStartedJobs(page); + await page.reload(); + + const card = page.getByTestId("media-card"); + await expect(card).toHaveAttribute("data-status", "done", { timeout: 30_000 }); + await expect(card).toContainText("Too large for TikTok (max 287 MB)"); + await expect(card).toContainText("Too short for Snapchat Spotlight (min 0:05)"); + await captureScreenshot(page, testInfo, "media-card-platform-check"); + + await card.getByRole("button", { name: "Export" }).click(); + const menu = page.getByRole("menu"); + await expect(menu.getByRole("menuitem", { name: "YouTube Shorts" })).toBeVisible(); + await expect(menu.getByRole("menuitem", { name: "Instagram Reels" })).toBeVisible(); + // Not the platform or the shape it already has. + await expect(menu.getByRole("menuitem", { name: "TikTok" })).toHaveCount(0); + await expect(menu.getByRole("menuitem", { name: "9:16" })).toHaveCount(0); + await expect(menu.getByRole("separator")).toHaveCount(1); + await expect(menu.getByRole("menuitem", { name: "1:1" })).toBeVisible(); + await captureScreenshot(page, testInfo, "media-card-platform-export-menu"); + + await menu.getByRole("menuitem", { name: "YouTube Shorts" }).click(); + await expect.poll(() => started.length).toBe(1); + expect(started[0]).toEqual({ + botId, + messageId: expect.stringMatching(/^media-message-\d+$/), + operation: "export_video", + input: { artifact_id: clip.id, platforms: ["youtube-shorts"] }, + }); +}); diff --git a/apps/web/src/components/media/MediaCard.test.tsx b/apps/web/src/components/media/MediaCard.test.tsx index f1d4ce0..eb9513d 100644 --- a/apps/web/src/components/media/MediaCard.test.tsx +++ b/apps/web/src/components/media/MediaCard.test.tsx @@ -8,6 +8,10 @@ vi.mock("@lingui/react/macro", () => { parts.reduce((text, part, index) => `${text}${part}${values[index] ?? ""}`, ""); return { useLingui: () => ({ t }), Trans: ({ children }: { children: ReactNode }) => children }; }); +vi.mock("@lingui/core/macro", () => ({ + t: (parts: TemplateStringsArray, ...values: unknown[]) => + parts.reduce((out, part, index) => `${out}${String(values[index - 1])}${part}`), +})); vi.mock("../../pages/shell/message-cards", () => ({ ArtifactImage: ({ artifactId, name }: { artifactId: string; name: string }) => ( {name} @@ -147,6 +151,65 @@ describe("media card", () => { expect(html).not.toContain("Transcribe"); }); + it("names the platform of an export, and its first problem only when the check did not pass", () => { + const tiktok = { + ...video, + artifactId: "tiktok-1", + platform: "tiktok" as const, + check: { status: "pass" as const, issues: [] }, + }; + const shorts = { + ...video, + artifactId: "shorts-1", + platform: "youtube-shorts" as const, + check: { + status: "warn" as const, + issues: [ + { code: "loudness_off" as const, targetLufs: -14, actualLufs: -18 }, + { code: "true_peak_high" as const, maxDb: -1, actualDb: 0 }, + ], + }, + }; + const html = renderToString( + , + ); + expect(html).toContain('

TikTok

'); + expect(html).toContain( + '

Too quiet for YouTube Shorts (target -14 LUFS)

', + ); + expect(html).not.toContain("Peaks too high"); + expect(html.match(/text-ui-body-sm text-(muted-foreground|warning|destructive)/g)).toHaveLength( + 2, + ); + }); + + it("lists skipped platforms, which also explain a job that rendered nothing", () => { + const skipped = [ + { + platform: "snapchat-spotlight" as const, + issues: [{ code: "too_long" as const, maxMs: 60_000, actualMs: 75_000 }], + }, + ]; + const partial = renderToString( + , + ); + expect(partial).toContain("Too long for Snapchat Spotlight (max 1:00)"); + const failed = renderToString( + , + ); + expect(failed).toContain("Too long for Snapchat Spotlight (max 1:00)"); + expect(failed).not.toContain("Not exported"); + }); + it("shows why a job failed", () => { const html = renderToString( , diff --git a/apps/web/src/components/media/MediaCard.tsx b/apps/web/src/components/media/MediaCard.tsx index 80e7db4..85375eb 100644 --- a/apps/web/src/components/media/MediaCard.tsx +++ b/apps/web/src/components/media/MediaCard.tsx @@ -4,6 +4,7 @@ import type { MediaJobStatus, MediaOperation, MediaOutput, + MediaPlatformId, MediaPresetId, } from "@rakazo/contracts"; import { MEDIA_JOB_ACTIVE_STATUSES, MEDIA_PRESETS, MediaPresetIdSchema } from "@rakazo/contracts"; @@ -14,6 +15,7 @@ import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, + DropdownMenuSeparator, DropdownMenuTrigger, } from "@rakazo/ui-web"; import type { LucideIcon } from "lucide-react"; @@ -26,12 +28,14 @@ import { ArtifactImage } from "../../pages/shell/message-cards"; import { ArtifactFileCard } from "../ArtifactFileCard"; import { Shimmer } from "../ai/primitives"; import { MediaPlayer } from "./MediaPlayer"; +import { exportPlatformsFor, platformIssueLine, platformName } from "./media-platforms"; const CARD_CLASS = "w-[min(480px,96%)] rounded-[18px] border border-border bg-card/75 p-3.5 text-start shadow-xs"; export type MediaCardAction = | { kind: "export"; artifactId: string; preset: MediaPresetId } + | { kind: "export"; artifactId: string; platform: MediaPlatformId } | { kind: "transcribe"; artifactId: string; name: string } | { kind: "cancel" }; @@ -114,6 +118,7 @@ function OutputActions({ }) { const { t } = useLingui(); const name = output.name; + const platforms = output.kind === "video" ? exportPlatformsFor(output) : []; const presets = output.kind === "video" ? exportPresetsFor(output) : []; return (
@@ -129,7 +134,7 @@ function OutputActions({ > - {onAction && available.export && presets.length > 0 ? ( + {onAction && available.export && platforms.length + presets.length > 0 ? ( + {platforms.map((platform) => ( + + onAction({ kind: "export", artifactId: output.artifactId, platform }) + } + > + {platformName(platform)} + + ))} + {platforms.length > 0 && presets.length > 0 ? : null} {presets.map((preset) => ( + {issue ? platformIssueLine(output.platform, issue) : platformName(output.platform)} +

+ ); +} + /** * The live card of a media job: what it makes, how far it is, and the results with the * follow-ups that fit them (export other formats, transcribe). Nothing else is shown. @@ -191,6 +227,10 @@ export function MediaCardView({ const percent = Math.round((block.progress ?? 0) * 100); const images = block.outputs.filter((output) => output.kind === "image"); const others = block.outputs.filter((output) => output.kind !== "image"); + const skipped = block.skipped ?? []; + // When nothing was rendered because every platform was skipped, the skipped lines say why. + const showError = + block.status === "failed" && block.error && !(skipped.length && !block.outputs.length); const act = onAction ? (action: MediaCardAction) => { @@ -251,7 +291,7 @@ export function MediaCardView({ />
) : null} - {block.status === "failed" && block.error ? ( + {showError ? (

{block.error}

@@ -279,6 +319,7 @@ export function MediaCardView({ width={output.width} height={output.height} /> + {active ? null : ( )} @@ -293,6 +334,11 @@ export function MediaCardView({ )}
))} + {skipped.map((target) => ( +

+ {platformIssueLine(target.platform, target.issues[0]!)} +

+ ))} {actionError ? (

{actionError} @@ -383,7 +429,10 @@ export function MediaCard({ ...(action.kind === "export" ? { operation: "export_video", - input: { artifact_id: action.artifactId, presets: [action.preset] }, + input: + "platform" in action + ? { artifact_id: action.artifactId, platforms: [action.platform] } + : { artifact_id: action.artifactId, presets: [action.preset] }, } : // The new card is titled with the file it transcribes. { diff --git a/apps/web/src/components/media/media-platforms.test.ts b/apps/web/src/components/media/media-platforms.test.ts new file mode 100644 index 0000000..4906f0c --- /dev/null +++ b/apps/web/src/components/media/media-platforms.test.ts @@ -0,0 +1,75 @@ +import type { MediaCheckIssue } from "@rakazo/contracts"; +import { describe, expect, it, vi } from "vitest"; + +// The macros compile at build time; here the English source comes back with its values filled in. +vi.mock("@lingui/core/macro", () => ({ + t: (parts: TemplateStringsArray, ...values: unknown[]) => + parts.reduce((out, part, index) => `${out}${String(values[index - 1])}${part}`), +})); + +import { + clockDuration, + exportPlatformsFor, + platformIssueLine, + platformName, +} from "./media-platforms"; + +describe("export platforms", () => { + it("offers every platform but the one the video was exported for, by brand name", () => { + expect(exportPlatformsFor({})).toHaveLength(10); + expect(exportPlatformsFor({ platform: "tiktok" })).not.toContain("tiktok"); + expect(exportPlatformsFor({}).slice(0, 4).map(platformName)).toEqual([ + "TikTok", + "Instagram Reels", + "Instagram Feed", + "YouTube Shorts", + ]); + }); +}); + +describe("platform issue lines", () => { + it("formats limits as a clock", () => { + expect(clockDuration(600_000)).toBe("10:00"); + expect(clockDuration(5000)).toBe("0:05"); + expect(clockDuration(3_723_000)).toBe("1:02:03"); + }); + + it.each<[MediaCheckIssue, string]>([ + [{ code: "too_long", maxMs: 600_000, actualMs: 720_000 }, "Too long for TikTok (max 10:00)"], + [{ code: "too_short", minMs: 3000, actualMs: 1000 }, "Too short for TikTok (min 0:03)"], + [{ code: "file_too_large", maxMb: 287, actualMb: 301.2 }, "Too large for TikTok (max 287 MB)"], + [ + { code: "bitrate_high", maxKbps: 8000, actualKbps: 9500 }, + "Bitrate too high for TikTok (max 8,000 kbps)", + ], + [ + { code: "loudness_off", targetLufs: -14, actualLufs: -18.2 }, + "Too quiet for TikTok (target -14 LUFS)", + ], + [ + { code: "loudness_off", targetLufs: -14, actualLufs: -9.5 }, + "Too loud for TikTok (target -14 LUFS)", + ], + [ + { code: "true_peak_high", maxDb: -1, actualDb: 0.2 }, + "Peaks too high for TikTok (max -1 dBTP)", + ], + [ + { + code: "resolution_mismatch", + width: 1080, + height: 1920, + actualWidth: 720, + actualHeight: 1280, + }, + "Wrong size for TikTok (needs 1080×1920)", + ], + [ + { code: "fps_mismatch", maxFps: 30, actualFps: 60 }, + "Frame rate too high for TikTok (max 30 fps)", + ], + [{ code: "codec_mismatch" }, "Wrong format for TikTok (needs H.264)"], + ])("words %j in one short line", (issue, line) => { + expect(platformIssueLine("tiktok", issue)).toBe(line); + }); +}); diff --git a/apps/web/src/components/media/media-platforms.ts b/apps/web/src/components/media/media-platforms.ts new file mode 100644 index 0000000..ad60f6a --- /dev/null +++ b/apps/web/src/components/media/media-platforms.ts @@ -0,0 +1,72 @@ +import { i18n } from "@lingui/core"; +import { t } from "@lingui/core/macro"; +import type { MediaCheckIssue, MediaOutput, MediaPlatformId } from "@rakazo/contracts"; +import { MEDIA_PLATFORM_PROFILES, MediaPlatformIdSchema } from "@rakazo/contracts"; + +/** Platforms a video can be exported for: all but the one it was exported for. */ +export function exportPlatformsFor(output: Pick): MediaPlatformId[] { + return MediaPlatformIdSchema.options.filter((id) => id !== output.platform); +} + +/** The platform's brand name, never translated. */ +export function platformName(platform: MediaPlatformId): string { + return MEDIA_PLATFORM_PROFILES[platform].name; +} + +/** "10:00", or "1:02:03" past an hour. */ +export function clockDuration(ms: number): string { + const total = Math.round(ms / 1000); + const hours = Math.floor(total / 3600); + const minutes = Math.floor((total % 3600) / 60); + const seconds = String(total % 60).padStart(2, "0"); + return hours > 0 + ? `${hours}:${String(minutes).padStart(2, "0")}:${seconds}` + : `${minutes}:${seconds}`; +} + +function amount(value: number): string { + return new Intl.NumberFormat(i18n.locale || "en", { maximumFractionDigits: 1 }).format(value); +} + +/** One short line for a check issue, e.g. "Too long for TikTok (max 10:00)". */ +export function platformIssueLine(platform: MediaPlatformId, issue: MediaCheckIssue): string { + const name = platformName(platform); + switch (issue.code) { + case "too_long": { + const max = clockDuration(issue.maxMs); + return t`Too long for ${name} (max ${max})`; + } + case "too_short": { + const min = clockDuration(issue.minMs); + return t`Too short for ${name} (min ${min})`; + } + case "file_too_large": { + const max = amount(issue.maxMb); + return t`Too large for ${name} (max ${max} MB)`; + } + case "bitrate_high": { + const max = amount(issue.maxKbps); + return t`Bitrate too high for ${name} (max ${max} kbps)`; + } + case "loudness_off": { + const target = amount(issue.targetLufs); + return issue.actualLufs < issue.targetLufs + ? t`Too quiet for ${name} (target ${target} LUFS)` + : t`Too loud for ${name} (target ${target} LUFS)`; + } + case "true_peak_high": { + const max = amount(issue.maxDb); + return t`Peaks too high for ${name} (max ${max} dBTP)`; + } + case "resolution_mismatch": { + const size = `${issue.width}×${issue.height}`; + return t`Wrong size for ${name} (needs ${size})`; + } + case "fps_mismatch": { + const max = amount(issue.maxFps); + return t`Frame rate too high for ${name} (max ${max} fps)`; + } + case "codec_mismatch": + return t`Wrong format for ${name} (needs H.264)`; + } +} diff --git a/apps/web/src/locales/de/messages.po b/apps/web/src/locales/de/messages.po index d42f5fa..dec4450 100644 --- a/apps/web/src/locales/de/messages.po +++ b/apps/web/src/locales/de/messages.po @@ -836,6 +836,10 @@ msgstr "Bearer-Token" msgid "Bell" msgstr "Glöckchen" +#: src/components/media/media-platforms.ts +msgid "Bitrate too high for {name} (max {max} kbps)" +msgstr "Bitrate zu hoch für {name} (maximal {max} kbps)" + #: src/components/NotificationSettingsPanel.tsx msgid "Blocked in your browser settings" msgstr "In deinen Browser-Einstellungen blockiert" @@ -2551,6 +2555,10 @@ msgstr "Passwort vergessen?" msgid "Forward" msgstr "Vorwärts" +#: src/components/media/media-platforms.ts +msgid "Frame rate too high for {name} (max {max} fps)" +msgstr "Bildrate zu hoch für {name} (maximal {max} fps)" + #: src/components/OoAtelier.tsx msgid "Freckles" msgstr "Sommersprossen" @@ -3857,6 +3865,10 @@ msgstr "Pfote" msgid "Peach" msgstr "Pfirsich" +#: src/components/media/media-platforms.ts +msgid "Peaks too high for {name} (max {max} dBTP)" +msgstr "Pegelspitzen zu hoch für {name} (maximal {max} dBTP)" + #: src/components/OoAtelier.tsx msgid "Penguin" msgstr "Pinguin" @@ -5116,6 +5128,26 @@ msgstr "Heute" msgid "Today, {heuteActionCount} need you" msgstr "Heute, {heuteActionCount} brauchen dich" +#: src/components/media/media-platforms.ts +msgid "Too large for {name} (max {max} MB)" +msgstr "Zu groß für {name} (maximal {max} MB)" + +#: src/components/media/media-platforms.ts +msgid "Too long for {name} (max {max})" +msgstr "Zu lang für {name} (maximal {max})" + +#: src/components/media/media-platforms.ts +msgid "Too loud for {name} (target {target} LUFS)" +msgstr "Zu laut für {name} (Zielwert {target} LUFS)" + +#: src/components/media/media-platforms.ts +msgid "Too quiet for {name} (target {target} LUFS)" +msgstr "Zu leise für {name} (Zielwert {target} LUFS)" + +#: src/components/media/media-platforms.ts +msgid "Too short for {name} (min {min})" +msgstr "Zu kurz für {name} (mindestens {min})" + #: src/pages/PluginsOverlay.tsx msgid "Tool sources" msgstr "Toolquellen" @@ -5535,6 +5567,14 @@ msgstr "Arbeitet…" msgid "Working…" msgstr "Wird verarbeitet…" +#: src/components/media/media-platforms.ts +msgid "Wrong format for {name} (needs H.264)" +msgstr "Falsches Format für {name} (braucht H.264)" + +#: src/components/media/media-platforms.ts +msgid "Wrong size for {name} (needs {size})" +msgstr "Falsche Größe für {name} (braucht {size})" + #: src/pages/shell/roster-format.ts msgid "Yesterday" msgstr "Gestern" diff --git a/apps/web/src/locales/en/messages.po b/apps/web/src/locales/en/messages.po index 4f56b03..e7f8af5 100644 --- a/apps/web/src/locales/en/messages.po +++ b/apps/web/src/locales/en/messages.po @@ -836,6 +836,10 @@ msgstr "Bearer token" msgid "Bell" msgstr "Bell" +#: src/components/media/media-platforms.ts +msgid "Bitrate too high for {name} (max {max} kbps)" +msgstr "Bitrate too high for {name} (max {max} kbps)" + #: src/components/NotificationSettingsPanel.tsx msgid "Blocked in your browser settings" msgstr "Blocked in your browser settings" @@ -2551,6 +2555,10 @@ msgstr "Forgot password?" msgid "Forward" msgstr "Forward" +#: src/components/media/media-platforms.ts +msgid "Frame rate too high for {name} (max {max} fps)" +msgstr "Frame rate too high for {name} (max {max} fps)" + #: src/components/OoAtelier.tsx msgid "Freckles" msgstr "Freckles" @@ -3857,6 +3865,10 @@ msgstr "Paw" msgid "Peach" msgstr "Peach" +#: src/components/media/media-platforms.ts +msgid "Peaks too high for {name} (max {max} dBTP)" +msgstr "Peaks too high for {name} (max {max} dBTP)" + #: src/components/OoAtelier.tsx msgid "Penguin" msgstr "Penguin" @@ -5116,6 +5128,26 @@ msgstr "Today" msgid "Today, {heuteActionCount} need you" msgstr "Today, {heuteActionCount} need you" +#: src/components/media/media-platforms.ts +msgid "Too large for {name} (max {max} MB)" +msgstr "Too large for {name} (max {max} MB)" + +#: src/components/media/media-platforms.ts +msgid "Too long for {name} (max {max})" +msgstr "Too long for {name} (max {max})" + +#: src/components/media/media-platforms.ts +msgid "Too loud for {name} (target {target} LUFS)" +msgstr "Too loud for {name} (target {target} LUFS)" + +#: src/components/media/media-platforms.ts +msgid "Too quiet for {name} (target {target} LUFS)" +msgstr "Too quiet for {name} (target {target} LUFS)" + +#: src/components/media/media-platforms.ts +msgid "Too short for {name} (min {min})" +msgstr "Too short for {name} (min {min})" + #: src/pages/PluginsOverlay.tsx msgid "Tool sources" msgstr "Tool sources" @@ -5535,6 +5567,14 @@ msgstr "Working…" msgid "Working…" msgstr "Working…" +#: src/components/media/media-platforms.ts +msgid "Wrong format for {name} (needs H.264)" +msgstr "Wrong format for {name} (needs H.264)" + +#: src/components/media/media-platforms.ts +msgid "Wrong size for {name} (needs {size})" +msgstr "Wrong size for {name} (needs {size})" + #: src/pages/shell/roster-format.ts msgid "Yesterday" msgstr "Yesterday" diff --git a/apps/web/src/locales/es/messages.po b/apps/web/src/locales/es/messages.po index 42f6f72..d7e1dfc 100644 --- a/apps/web/src/locales/es/messages.po +++ b/apps/web/src/locales/es/messages.po @@ -836,6 +836,10 @@ msgstr "Token Bearer" msgid "Bell" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Bitrate too high for {name} (max {max} kbps)" +msgstr "" + #: src/components/NotificationSettingsPanel.tsx msgid "Blocked in your browser settings" msgstr "" @@ -2551,6 +2555,10 @@ msgstr "¿Olvidaste la contraseña?" msgid "Forward" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Frame rate too high for {name} (max {max} fps)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Freckles" msgstr "" @@ -3857,6 +3865,10 @@ msgstr "" msgid "Peach" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Peaks too high for {name} (max {max} dBTP)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Penguin" msgstr "" @@ -5116,6 +5128,26 @@ msgstr "" msgid "Today, {heuteActionCount} need you" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Too large for {name} (max {max} MB)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too long for {name} (max {max})" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too loud for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too quiet for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too short for {name} (min {min})" +msgstr "" + #: src/pages/PluginsOverlay.tsx msgid "Tool sources" msgstr "Fuentes de herramientas" @@ -5535,6 +5567,14 @@ msgstr "Trabajando…" msgid "Working…" msgstr "Trabajando…" +#: src/components/media/media-platforms.ts +msgid "Wrong format for {name} (needs H.264)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Wrong size for {name} (needs {size})" +msgstr "" + #: src/pages/shell/roster-format.ts msgid "Yesterday" msgstr "Ayer" diff --git a/apps/web/src/locales/hi/messages.po b/apps/web/src/locales/hi/messages.po index 4a29ccb..5f3916b 100644 --- a/apps/web/src/locales/hi/messages.po +++ b/apps/web/src/locales/hi/messages.po @@ -836,6 +836,10 @@ msgstr "बियरर टोकन" msgid "Bell" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Bitrate too high for {name} (max {max} kbps)" +msgstr "" + #: src/components/NotificationSettingsPanel.tsx msgid "Blocked in your browser settings" msgstr "" @@ -2551,6 +2555,10 @@ msgstr "पासवर्ड भूल गए?" msgid "Forward" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Frame rate too high for {name} (max {max} fps)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Freckles" msgstr "" @@ -3857,6 +3865,10 @@ msgstr "" msgid "Peach" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Peaks too high for {name} (max {max} dBTP)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Penguin" msgstr "" @@ -5116,6 +5128,26 @@ msgstr "" msgid "Today, {heuteActionCount} need you" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Too large for {name} (max {max} MB)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too long for {name} (max {max})" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too loud for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too quiet for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too short for {name} (min {min})" +msgstr "" + #: src/pages/PluginsOverlay.tsx msgid "Tool sources" msgstr "टूल स्रोत" @@ -5535,6 +5567,14 @@ msgstr "काम चल रहा है…" msgid "Working…" msgstr "काम चल रहा है…" +#: src/components/media/media-platforms.ts +msgid "Wrong format for {name} (needs H.264)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Wrong size for {name} (needs {size})" +msgstr "" + #: src/pages/shell/roster-format.ts msgid "Yesterday" msgstr "कल" diff --git a/apps/web/src/locales/ko/messages.po b/apps/web/src/locales/ko/messages.po index 8eeedda..d80d4a0 100644 --- a/apps/web/src/locales/ko/messages.po +++ b/apps/web/src/locales/ko/messages.po @@ -836,6 +836,10 @@ msgstr "Bearer 토큰" msgid "Bell" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Bitrate too high for {name} (max {max} kbps)" +msgstr "" + #: src/components/NotificationSettingsPanel.tsx msgid "Blocked in your browser settings" msgstr "" @@ -2551,6 +2555,10 @@ msgstr "비밀번호를 잊으셨나요?" msgid "Forward" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Frame rate too high for {name} (max {max} fps)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Freckles" msgstr "" @@ -3857,6 +3865,10 @@ msgstr "" msgid "Peach" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Peaks too high for {name} (max {max} dBTP)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Penguin" msgstr "" @@ -5116,6 +5128,26 @@ msgstr "" msgid "Today, {heuteActionCount} need you" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Too large for {name} (max {max} MB)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too long for {name} (max {max})" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too loud for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too quiet for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too short for {name} (min {min})" +msgstr "" + #: src/pages/PluginsOverlay.tsx msgid "Tool sources" msgstr "도구 소스" @@ -5535,6 +5567,14 @@ msgstr "처리 중…" msgid "Working…" msgstr "처리 중…" +#: src/components/media/media-platforms.ts +msgid "Wrong format for {name} (needs H.264)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Wrong size for {name} (needs {size})" +msgstr "" + #: src/pages/shell/roster-format.ts msgid "Yesterday" msgstr "어제" diff --git a/apps/web/src/locales/pt-BR/messages.po b/apps/web/src/locales/pt-BR/messages.po index 0a46646..a053432 100644 --- a/apps/web/src/locales/pt-BR/messages.po +++ b/apps/web/src/locales/pt-BR/messages.po @@ -836,6 +836,10 @@ msgstr "Token portador" msgid "Bell" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Bitrate too high for {name} (max {max} kbps)" +msgstr "" + #: src/components/NotificationSettingsPanel.tsx msgid "Blocked in your browser settings" msgstr "" @@ -2551,6 +2555,10 @@ msgstr "Esqueceu a senha?" msgid "Forward" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Frame rate too high for {name} (max {max} fps)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Freckles" msgstr "" @@ -3857,6 +3865,10 @@ msgstr "" msgid "Peach" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Peaks too high for {name} (max {max} dBTP)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Penguin" msgstr "" @@ -5116,6 +5128,26 @@ msgstr "" msgid "Today, {heuteActionCount} need you" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Too large for {name} (max {max} MB)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too long for {name} (max {max})" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too loud for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too quiet for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too short for {name} (min {min})" +msgstr "" + #: src/pages/PluginsOverlay.tsx msgid "Tool sources" msgstr "Fontes de ferramentas" @@ -5535,6 +5567,14 @@ msgstr "Funcionando" msgid "Working…" msgstr "Funcionando" +#: src/components/media/media-platforms.ts +msgid "Wrong format for {name} (needs H.264)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Wrong size for {name} (needs {size})" +msgstr "" + #: src/pages/shell/roster-format.ts msgid "Yesterday" msgstr "Ontem" diff --git a/apps/web/src/locales/ru/messages.po b/apps/web/src/locales/ru/messages.po index 5c2559d..cee96f9 100644 --- a/apps/web/src/locales/ru/messages.po +++ b/apps/web/src/locales/ru/messages.po @@ -836,6 +836,10 @@ msgstr "Bearer-токен" msgid "Bell" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Bitrate too high for {name} (max {max} kbps)" +msgstr "" + #: src/components/NotificationSettingsPanel.tsx msgid "Blocked in your browser settings" msgstr "" @@ -2551,6 +2555,10 @@ msgstr "Забыли пароль?" msgid "Forward" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Frame rate too high for {name} (max {max} fps)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Freckles" msgstr "" @@ -3857,6 +3865,10 @@ msgstr "" msgid "Peach" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Peaks too high for {name} (max {max} dBTP)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Penguin" msgstr "" @@ -5116,6 +5128,26 @@ msgstr "" msgid "Today, {heuteActionCount} need you" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Too large for {name} (max {max} MB)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too long for {name} (max {max})" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too loud for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too quiet for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too short for {name} (min {min})" +msgstr "" + #: src/pages/PluginsOverlay.tsx msgid "Tool sources" msgstr "Источники инструментов" @@ -5535,6 +5567,14 @@ msgstr "Выполняется…" msgid "Working…" msgstr "Выполняется…" +#: src/components/media/media-platforms.ts +msgid "Wrong format for {name} (needs H.264)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Wrong size for {name} (needs {size})" +msgstr "" + #: src/pages/shell/roster-format.ts msgid "Yesterday" msgstr "Вчера" diff --git a/apps/web/src/locales/tr/messages.po b/apps/web/src/locales/tr/messages.po index 8212fde..4f13318 100644 --- a/apps/web/src/locales/tr/messages.po +++ b/apps/web/src/locales/tr/messages.po @@ -836,6 +836,10 @@ msgstr "Bearer token" msgid "Bell" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Bitrate too high for {name} (max {max} kbps)" +msgstr "" + #: src/components/NotificationSettingsPanel.tsx msgid "Blocked in your browser settings" msgstr "" @@ -2551,6 +2555,10 @@ msgstr "Parolanızı mı unuttunuz?" msgid "Forward" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Frame rate too high for {name} (max {max} fps)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Freckles" msgstr "" @@ -3857,6 +3865,10 @@ msgstr "" msgid "Peach" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Peaks too high for {name} (max {max} dBTP)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Penguin" msgstr "" @@ -5116,6 +5128,26 @@ msgstr "" msgid "Today, {heuteActionCount} need you" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Too large for {name} (max {max} MB)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too long for {name} (max {max})" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too loud for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too quiet for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too short for {name} (min {min})" +msgstr "" + #: src/pages/PluginsOverlay.tsx msgid "Tool sources" msgstr "Araç kaynakları" @@ -5535,6 +5567,14 @@ msgstr "Çalışıyor…" msgid "Working…" msgstr "Çalışıyor…" +#: src/components/media/media-platforms.ts +msgid "Wrong format for {name} (needs H.264)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Wrong size for {name} (needs {size})" +msgstr "" + #: src/pages/shell/roster-format.ts msgid "Yesterday" msgstr "Dün" diff --git a/apps/web/src/locales/zh-CN/messages.po b/apps/web/src/locales/zh-CN/messages.po index 31ec6e5..b8db110 100644 --- a/apps/web/src/locales/zh-CN/messages.po +++ b/apps/web/src/locales/zh-CN/messages.po @@ -836,6 +836,10 @@ msgstr "Bearer token" msgid "Bell" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Bitrate too high for {name} (max {max} kbps)" +msgstr "" + #: src/components/NotificationSettingsPanel.tsx msgid "Blocked in your browser settings" msgstr "" @@ -2551,6 +2555,10 @@ msgstr "忘记密码？" msgid "Forward" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Frame rate too high for {name} (max {max} fps)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Freckles" msgstr "" @@ -3857,6 +3865,10 @@ msgstr "" msgid "Peach" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Peaks too high for {name} (max {max} dBTP)" +msgstr "" + #: src/components/OoAtelier.tsx msgid "Penguin" msgstr "" @@ -5116,6 +5128,26 @@ msgstr "" msgid "Today, {heuteActionCount} need you" msgstr "" +#: src/components/media/media-platforms.ts +msgid "Too large for {name} (max {max} MB)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too long for {name} (max {max})" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too loud for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too quiet for {name} (target {target} LUFS)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Too short for {name} (min {min})" +msgstr "" + #: src/pages/PluginsOverlay.tsx msgid "Tool sources" msgstr "工具源" @@ -5535,6 +5567,14 @@ msgstr "处理中…" msgid "Working…" msgstr "处理中…" +#: src/components/media/media-platforms.ts +msgid "Wrong format for {name} (needs H.264)" +msgstr "" + +#: src/components/media/media-platforms.ts +msgid "Wrong size for {name} (needs {size})" +msgstr "" + #: src/pages/shell/roster-format.ts msgid "Yesterday" msgstr "昨天" diff --git a/packages/adapter-kit/src/media.ts b/packages/adapter-kit/src/media.ts index c9610ea..9a4ea34 100644 --- a/packages/adapter-kit/src/media.ts +++ b/packages/adapter-kit/src/media.ts @@ -1,9 +1,12 @@ import type { + MediaCheck, MediaJobStatus, MediaOperation, MediaOutput, MediaOutputKind, + MediaPlatformId, MediaPresetId, + MediaSkippedTarget, } from "@rakazo/contracts"; import type { AdapterDescriptor } from "./types.js"; @@ -137,6 +140,8 @@ export interface MediaJobOutputFile { posterPath?: string; parentArtifactId?: string; preset?: MediaPresetId; + platform?: MediaPlatformId; + check?: MediaCheck; } /** An artifact the job may read, copied into the job's temp directory. */ @@ -168,6 +173,8 @@ export interface MediaJobContext { fetchArtifact(artifactId: string): Promise; /** Stores a produced file as an artifact and adds it to the media card. */ addOutput(file: MediaJobOutputFile): Promise; + /** Shows a platform the job did not render on the media card, with the reason. */ + addSkipped(target: MediaSkippedTarget): Promise; } export interface MediaOperationResult { diff --git a/packages/adapters/src/builtin-tools.ts b/packages/adapters/src/builtin-tools.ts index ddf8695..14127db 100644 --- a/packages/adapters/src/builtin-tools.ts +++ b/packages/adapters/src/builtin-tools.ts @@ -7,6 +7,7 @@ import { ExportVideoInputSchema, GenerateImageInputSchema, GenerateVideoInputSchema, + MEDIA_PLATFORM_PROFILES, MODEL_TIERS, SecretAskPurpose, SecretHttpRequest, @@ -34,6 +35,11 @@ export const SUBAGENT_MODEL_TIER_DESCRIPTION = const VIDEO_PRESETS_AND_FITS = "Presets: vertical 9:16, landscape 16:9, square 1:1, portrait 4:5, classic 4:3, cinema 21:9. fit: crop (default) fills the frame and cuts the edges, pad adds bars, blur fills the bars with a blurred copy."; +/** "tiktok 600 s, …": each platform id with the longest video it takes. */ +const VIDEO_PLATFORMS = Object.values(MEDIA_PLATFORM_PROFILES) + .map((profile) => `${profile.id} ${profile.duration.hardMaxMs / 1000} s`) + .join(", "); + export const DELEGATION_TOOL_NAMES = new Set([ "run_subagent", "spawn_bot", @@ -564,7 +570,7 @@ export const builtinAgentTools: ConnectorTool[] = [ }, { name: "export_video", - description: `Export a video artifact as one H.264/AAC MP4 per preset. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`, + description: `Export a video artifact as one H.264/AAC MP4 per platform and per preset, 6 at most. When the user names where it will be posted, use platforms: each file is fitted to that platform's frame, frame rate, bitrate and loudness, then checked. Platforms (longest video): ${VIDEO_PLATFORMS}. A platform the video is too long or too short for is skipped: trim with edit_video first. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`, inputSchema: mediaParameters(ExportVideoInputSchema), }, { diff --git a/packages/adapters/src/media-runtime.postgres.test.ts b/packages/adapters/src/media-runtime.postgres.test.ts index 2852dbe..b6b7d74 100644 --- a/packages/adapters/src/media-runtime.postgres.test.ts +++ b/packages/adapters/src/media-runtime.postgres.test.ts @@ -335,6 +335,25 @@ describePostgres("media job runtime (PostgreSQL, fake operations)", () => { expect(existsSync(tempDir)).toBe(false); }); + it("keeps one skipped entry per platform when a job reports it again", async () => { + const tooLong = { code: "too_long", maxMs: 600_000, actualMs: 700_000 } as const; + const tooShort = { code: "too_short", minMs: 3_000, actualMs: 1_000 } as const; + const ctx = await setup({ + run: async (context) => { + await context.addSkipped({ platform: "tiktok", issues: [tooLong] }); + await context.addSkipped({ platform: "tiktok", issues: [tooShort] }); + await context.addSkipped({ platform: "youtube", issues: [tooShort] }); + return { summary: "Skipped." }; + }, + }); + const { jobId } = await ctx.start({ prompt: "fox" }); + await ctx.runtime.runJob(jobId); + expect((await ctx.card(jobId)).skipped).toEqual([ + { platform: "tiktok", issues: [tooShort] }, + { platform: "youtube", issues: [tooShort] }, + ]); + }); + it("refuses outputs outside the temp dir and sources of another user", async () => { const other = await setup(); const foreign = await other.sourceArtifact(); diff --git a/packages/adapters/src/media-runtime.ts b/packages/adapters/src/media-runtime.ts index d699a40..4eb7702 100644 --- a/packages/adapters/src/media-runtime.ts +++ b/packages/adapters/src/media-runtime.ts @@ -32,12 +32,14 @@ import type { MediaJobStatus, MediaOperation, MediaOutput, + MediaSkippedTarget, MessageBlock, } from "@rakazo/contracts"; import { attachmentKindOf, MEDIA_JOB_ACTIVE_STATUSES, MediaOperationSchema, + MediaSkippedTargetSchema, } from "@rakazo/contracts"; import { attachmentExtensionForMimeType, inferAttachmentMimeType } from "@rakazo/core"; import type { MediaJob, PrismaClient, ThreadEvents } from "@rakazo/db"; @@ -160,7 +162,17 @@ function issuesText(error: z.ZodError): string { .join("; "); } -function cardBlock(job: MediaJob, outputs: MediaOutput[]): MediaJobBlock { +/** What a write adds to the card: stored outputs and platforms the job did not render. */ +interface CardAdditions { + outputs?: MediaOutput[]; + skipped?: MediaSkippedTarget[]; +} + +function cardBlock( + job: MediaJob, + outputs: MediaOutput[], + skipped: MediaSkippedTarget[] = [], +): MediaJobBlock { return { kind: "media", jobId: job.id, @@ -170,6 +182,7 @@ function cardBlock(job: MediaJob, outputs: MediaOutput[]): MediaJobBlock { ...(job.progress === null ? {} : { progress: job.progress }), ...(job.error === null ? {} : { error: job.error }), outputs, + ...(skipped.length ? { skipped } : {}), }; } @@ -346,8 +359,9 @@ export function createMediaRuntime( const writeCard = async ( tx: Prisma.TransactionClient, job: MediaJob, - appended: MediaOutput[] = [], + added: CardAdditions = {}, ): Promise<{ exists: boolean; seq?: number; outputs: MediaOutput[] }> => { + const appended = added.outputs ?? []; if (!job.messageId) return { exists: false, outputs: appended }; const message = await tx.message.findFirst({ where: { id: job.messageId, threadId: job.threadId }, @@ -356,7 +370,16 @@ export function createMediaRuntime( if (!message) return { exists: false, outputs: appended }; const previous = mediaBlockOf(message.blocks, job.id); const outputs = [...(previous?.outputs ?? []), ...appended]; - const next = cardBlock(job, outputs); + // A recovered job runs again, so the newest reason per platform replaces an earlier one. + const skipped = [ + ...new Map( + [...(previous?.skipped ?? []), ...(added.skipped ?? [])].map((entry) => [ + entry.platform, + entry, + ]), + ).values(), + ]; + const next = cardBlock(job, outputs, skipped); const current = Array.isArray(message.blocks) ? (message.blocks as MessageBlock[]) : []; const blocks = previous ? current.map((block) => (block === previous ? next : block)) @@ -386,7 +409,7 @@ export function createMediaRuntime( jobId: string, token: string | null, data: Prisma.MediaJobUpdateManyMutationInput, - appended: MediaOutput[] = [], + added: CardAdditions = {}, ) => { const committed = await withTransactionRetry(() => deps.prisma.$transaction(async (tx) => { @@ -408,7 +431,7 @@ export function createMediaRuntime( const updated = await tx.mediaJob.updateMany({ where: fence, data }); if (!updated.count) return null; const job = await tx.mediaJob.findUniqueOrThrow({ where: { id: jobId } }); - const card = await writeCard(tx, job, appended); + const card = await writeCard(tx, job, added); return { job, card }; }), ); @@ -801,10 +824,15 @@ export function createMediaRuntime( ...(posterArtifactId ? { posterArtifactId } : {}), ...(parent ? { parentArtifactId: parent.id } : {}), ...(file.preset ? { preset: file.preset } : {}), + ...(file.platform ? { platform: file.platform } : {}), + ...(file.check ? { check: file.check } : {}), }; - const written = await updateJob(jobId, token, { outputArtifactIds: { push: stored } }, [ - output, - ]); + const written = await updateJob( + jobId, + token, + { outputArtifactIds: { push: stored } }, + { outputs: [output] }, + ); if (!written) { abort("lease"); throw new Error("Media job lost its lease"); @@ -816,6 +844,17 @@ export function createMediaRuntime( throw error; } }, + addSkipped: async (target) => { + signal.throwIfAborted(); + const written = await updateJob( + jobId, + token, + { updatedAt: new Date() }, + { skipped: [MediaSkippedTargetSchema.parse(target)] }, + ); + if (!written) abort("lease"); + else if (!written.card.exists) abort("abandoned"); + }, }; try { diff --git a/packages/adapters/src/media/ffmpeg.test.ts b/packages/adapters/src/media/ffmpeg.test.ts index e431827..5cde9ff 100644 --- a/packages/adapters/src/media/ffmpeg.test.ts +++ b/packages/adapters/src/media/ffmpeg.test.ts @@ -10,7 +10,9 @@ import { FfmpegError, ffmpegInputArgs, ffmpegSeconds, + measureLoudness, packetDurationMs, + parseLoudnormOutput, parseProbeOutput, probeMedia, renderPoster, @@ -18,6 +20,7 @@ import { } from "./ffmpeg.js"; import type { TestClips } from "./media-test-support.js"; import { createTestClips } from "./media-test-support.js"; +import { planLoudnessFix } from "./video-edit-plan.js"; const ffmpeg = await detectFfmpeg(); @@ -73,6 +76,18 @@ describe("probe parsing", () => { }); }); + it("reads the stream's average bitrate when the container records it", () => { + const probe = parseProbeOutput( + JSON.stringify({ + format: { duration: "3" }, + streams: [ + stream({ width: 1080, height: 1920, avg_frame_rate: "30/1", bit_rate: "522994" }), + ], + }), + ); + expect(probe.video?.bitRateKbps).toBe(523); + }); + it("reads legacy rotate tags, skips cover art and falls back to stream durations", () => { const probe = parseProbeOutput( JSON.stringify({ @@ -108,6 +123,40 @@ describe("probe parsing", () => { }); }); +describe("loudness parsing", () => { + it("reads loudnorm's JSON block from the end of the log", () => { + const log = [ + "Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'file:out.mp4':", + "[Parsed_loudnorm_0 @ 0x5555] ", + "{", + '\t"input_i" : "-17.42",', + '\t"input_tp" : "-3.10",', + '\t"input_lra" : "5.20",', + '\t"input_thresh" : "-27.71",', + '\t"output_i" : "-14.02",', + '\t"normalization_type" : "dynamic",', + '\t"target_offset" : "0.02"', + "}", + ].join("\n"); + expect(parseLoudnormOutput(log)).toEqual({ + integratedLufs: -17.42, + truePeakDb: -3.1, + rangeLu: 5.2, + thresholdLufs: -27.71, + offsetLu: 0.02, + }); + }); + + it("reads digital silence as minus infinity and refuses a log without numbers", () => { + const silent = parseLoudnormOutput( + '{"input_i":"-inf","input_tp":"-inf","input_lra":"0.00","input_thresh":"-70.00","target_offset":"inf"}', + ); + expect(silent?.integratedLufs).toBe(Number.NEGATIVE_INFINITY); + expect(parseLoudnormOutput("Conversion failed!")).toBeNull(); + expect(parseLoudnormOutput('{"input_i":"-14.0"}')).toBeNull(); + }); +}); + describe("ffmpeg arguments", () => { it("opens inputs through the file protocol with the demuxer of their MIME type", () => { expect( @@ -151,6 +200,28 @@ describe.skipIf(!ffmpeg)("ffmpeg integration", () => { expect(silent.video).toMatchObject({ width: 180, height: 320 }); }); + it("measures loudness and brings it to the target with a linear second pass", async () => { + const target = { integratedLufs: -14, truePeakDb: -1 }; + const before = await measureLoudness(clips.landscape, target, { mimeType: "video/mp4" }); + // The generated tone sits well below the streaming target. + expect(before.integratedLufs).toBeLessThan(-18); + const fixed = join(clips.dir, "loudness-fixed.mp4"); + await runFfmpeg( + planLoudnessFix({ + inputPath: clips.landscape, + outputPath: fixed, + durationMs: 3000, + target, + measured: before, + }), + ); + const after = await measureLoudness(fixed, target, { mimeType: "video/mp4" }); + expect(Math.abs(after.integratedLufs - target.integratedLufs)).toBeLessThan(1); + const probe = await probeMedia(fixed, { mimeType: "video/mp4" }); + expect(probe.video).toMatchObject({ codec: "h264", width: 320, height: 180 }); + expect(Math.abs(probe.durationMs - 3000)).toBeLessThan(50); + }); + it("refuses a playlist disguised as an MP4 instead of following its URLs", async () => { const playlist = join(clips.dir, "playlist.mp4"); await writeFile( diff --git a/packages/adapters/src/media/ffmpeg.ts b/packages/adapters/src/media/ffmpeg.ts index c184eb7..bf22a3e 100644 --- a/packages/adapters/src/media/ffmpeg.ts +++ b/packages/adapters/src/media/ffmpeg.ts @@ -27,6 +27,8 @@ export interface MediaProbe { fps: number; /** Pixel aspect ratio of the displayed frame; 1 for square pixels. */ sampleAspectRatio: number; + /** Average bitrate of the stream when the container records it. */ + bitRateKbps?: number; }; audio?: { codec: string; sampleRate: number; channels: number }; hasAudio: boolean; @@ -212,6 +214,7 @@ interface ProbeStream { sample_rate?: string; channels?: number; duration?: string; + bit_rate?: string; disposition?: { attached_pic?: number }; tags?: { rotate?: string }; side_data_list?: { rotation?: number }[]; @@ -265,6 +268,7 @@ function probeVideo(stream: ProbeStream, width: number, height: number) { const sideways = rotation === 90 || rotation === 270; const sar = ratio(stream.sample_aspect_ratio, ":") || 1; const fps = ratio(stream.avg_frame_rate, "/") || ratio(stream.r_frame_rate, "/"); + const bitRate = positive(Number(stream.bit_rate)); return { codec: stream.codec_name ?? "", width: sideways ? height : width, @@ -272,6 +276,7 @@ function probeVideo(stream: ProbeStream, width: number, height: number) { rotation, fps: fps > 0 && fps <= MAX_PLAUSIBLE_FPS ? fps : 0, sampleAspectRatio: sideways ? 1 / sar : sar, + ...(bitRate ? { bitRateKbps: Math.round(bitRate / 1000) } : {}), }; } @@ -505,6 +510,94 @@ export async function extractAudio( ); } +/** EBU R128 values of a file's first audio track, as ffmpeg's loudnorm filter measures them. */ +export interface LoudnessMeasurement { + /** Integrated loudness in LUFS; -Infinity for digital silence. */ + integratedLufs: number; + truePeakDb: number; + rangeLu: number; + thresholdLufs: number; + /** Gain loudnorm's second pass adds after limiting, for the targets it measured against. */ + offsetLu: number; +} + +const LOUDNESS_TIMEOUT_MS = 10 * 60_000; + +/** loudnorm's filter string: the target, and the first pass's numbers for a linear second pass. */ +export function loudnormFilter( + target: { integratedLufs: number; truePeakDb: number }, + measured?: LoudnessMeasurement, +): string { + const options = [ + `I=${target.integratedLufs}`, + `TP=${target.truePeakDb}`, + // A wide loudness range keeps loudnorm linear (no pumping) for most speech and music. + "LRA=11", + ...(measured + ? [ + `measured_I=${measured.integratedLufs}`, + `measured_TP=${measured.truePeakDb}`, + `measured_LRA=${measured.rangeLu}`, + `measured_thresh=${measured.thresholdLufs}`, + `offset=${measured.offsetLu}`, + "linear=true", + ] + : []), + ]; + return `loudnorm=${options.join(":")}`; +} + +/** Measures the loudness of a file with an audio track against a target. */ +export async function measureLoudness( + input: string, + target: { integratedLufs: number; truePeakDb: number }, + options: { signal?: AbortSignal; mimeType?: string } = {}, +): Promise { + const { ffmpegPath } = await requireFfmpeg(); + const result = await capture( + ffmpegPath, + [ + ...["-hide_banner", "-nostdin", "-nostats", "-loglevel", "info"], + ...ffmpegInputArgs(input, { mimeType: options.mimeType }), + ...["-map", "0:a:0", "-af", `${loudnormFilter(target)}:print_format=json`], + ...["-f", "null", "-"], + ], + { signal: options.signal, timeoutMs: LOUDNESS_TIMEOUT_MS }, + ); + if (result.code !== 0) throw ffmpegFailure("ffmpeg", result.code, result.stderr); + const measured = parseLoudnormOutput(result.stderr); + if (!measured) throw new Error("ffmpeg did not report the loudness."); + return measured; +} + +/** Reads the JSON block loudnorm prints with print_format=json at the end of its log. */ +export function parseLoudnormOutput(log: string): LoudnessMeasurement | null { + const start = log.lastIndexOf("{"); + const end = log.lastIndexOf("}"); + if (start < 0 || end < start) return null; + let data: Record; + try { + data = JSON.parse(log.slice(start, end + 1)) as Record; + } catch { + return null; + } + const value = (key: string) => { + const text = data[key]?.trim(); + // Digital silence measures as -inf, and its offset as inf. + if (text === "-inf") return Number.NEGATIVE_INFINITY; + if (text === "inf") return Number.POSITIVE_INFINITY; + return text ? Number(text) : Number.NaN; + }; + const measured = { + integratedLufs: value("input_i"), + truePeakDb: value("input_tp"), + rangeLu: value("input_lra"), + thresholdLufs: value("input_thresh"), + offsetLu: value("target_offset"), + }; + return Object.values(measured).some(Number.isNaN) ? null : measured; +} + /** Writes one JPEG frame at atMs (default 0), falling back to the first frame past the end. */ export async function renderPoster( input: string, diff --git a/packages/adapters/src/media/image-operations.test.ts b/packages/adapters/src/media/image-operations.test.ts index 84da555..88c906d 100644 --- a/packages/adapters/src/media/image-operations.test.ts +++ b/packages/adapters/src/media/image-operations.test.ts @@ -103,6 +103,7 @@ async function run( }; return output; }, + addSkipped: async () => undefined, }; const result = await operation(name).handler.run(context); return { result, outputs, progress }; diff --git a/packages/adapters/src/media/media-test-support.ts b/packages/adapters/src/media/media-test-support.ts index 940419a..f5e06ff 100644 --- a/packages/adapters/src/media/media-test-support.ts +++ b/packages/adapters/src/media/media-test-support.ts @@ -2,7 +2,7 @@ import { copyFile, mkdtemp, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit"; -import type { MediaOutput, MediaOutputKind } from "@rakazo/contracts"; +import type { MediaOutput, MediaOutputKind, MediaSkippedTarget } from "@rakazo/contracts"; import { attachmentKindOf } from "@rakazo/contracts"; import { runFfmpeg } from "./ffmpeg.js"; @@ -50,12 +50,13 @@ export interface TestArtifact { kind?: MediaOutputKind; } -/** A MediaJobContext over local files that records progress and outputs. */ +/** A MediaJobContext over local files that records progress, outputs and skipped platforms. */ export async function fakeMediaJobContext(input: unknown, artifacts: Record) { const tempDir = await mkdtemp(join(tmpdir(), "evoo-media-job-")); const controller = new AbortController(); const progress: number[] = []; const outputs: (MediaJobOutputFile & { size: number; posterSize?: number })[] = []; + const skipped: MediaSkippedTarget[] = []; const context: MediaJobContext = { jobId: "job-1", spaceId: "space-1", @@ -99,8 +100,13 @@ export async function fakeMediaJobContext(input: unknown, artifacts: Record { + skipped.push(target); + }, }; - return { context, controller, progress, outputs, tempDir }; + return { context, controller, progress, outputs, skipped, tempDir }; } diff --git a/packages/adapters/src/media/platform-export.test.ts b/packages/adapters/src/media/platform-export.test.ts new file mode 100644 index 0000000..630882f --- /dev/null +++ b/packages/adapters/src/media/platform-export.test.ts @@ -0,0 +1,188 @@ +import { MEDIA_PLATFORM_PROFILES } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { ExportMeasurement } from "./platform-export.js"; +import { + BITRATE_TOLERANCE, + checkExport, + DURATION_TOLERANCE_MS, + describeSkipped, + durationIssues, + LOUDNESS_TOLERANCE_LU, + MIN_VIDEO_KBPS, + needsLoudnessFix, + platformTarget, + sizeBudgetKbps, + sizeFixKbps, + TRUE_PEAK_TOLERANCE_DB, +} from "./platform-export.js"; + +const tiktok = MEDIA_PLATFORM_PROFILES.tiktok; + +/** A clean 30 s TikTok render; tests override what they examine. */ +function measured(overrides: Partial = {}): ExportMeasurement { + return { + width: 1080, + height: 1920, + fps: 30, + codec: "h264", + durationMs: 30_000, + sizeBytes: 20_000_000, + videoKbps: 5000, + loudness: { + integratedLufs: -14.2, + truePeakDb: -1.3, + rangeLu: 4, + thresholdLufs: -24.5, + offsetLu: 0.1, + }, + ...overrides, + }; +} + +function loudness(integratedLufs: number, truePeakDb = -2) { + return { integratedLufs, truePeakDb, rangeLu: 4, thresholdLufs: -30, offsetLu: 0 }; +} + +describe("platform targets", () => { + it("renders to the profile and caps the bitrate below its size limit for long videos", () => { + expect(platformTarget(tiktok, "blur", 30_000)).toEqual({ + width: 1080, + height: 1920, + fit: "blur", + maxFps: 30, + maxBitrateKbps: 8000, + integratedLufs: -14, + truePeakDb: -1, + }); + // 287 MB over 10 minutes leaves about 3.6 Mbps of video after headroom and audio. + const tenMinutes = platformTarget(tiktok, "crop", 600_000).maxBitrateKbps; + expect(tenMinutes).toBe(sizeBudgetKbps(tiktok, 600_000)); + expect(tenMinutes).toBeGreaterThan(3000); + expect(tenMinutes).toBeLessThan(3700); + expect(tenMinutes * 600 + 160 * 600).toBeLessThan(287 * 8000); + expect(sizeBudgetKbps(tiktok, 1e9)).toBe(MIN_VIDEO_KBPS); + }); +}); + +describe("duration limits", () => { + it("refuses videos past the limits, with a small tolerance for container rounding", () => { + expect(durationIssues(tiktok, 600_000 + DURATION_TOLERANCE_MS)).toEqual([]); + expect(durationIssues(tiktok, 720_000)).toEqual([ + { code: "too_long", maxMs: 600_000, actualMs: 720_000 }, + ]); + expect(durationIssues(tiktok, 3000 - DURATION_TOLERANCE_MS)).toEqual([]); + expect(durationIssues(tiktok, 2000)).toEqual([ + { code: "too_short", minMs: 3000, actualMs: 2000 }, + ]); + }); + + it("words a skipped platform for the bot with what to do about it", () => { + expect(describeSkipped(tiktok, durationIssues(tiktok, 720_000))).toBe( + "Not exported for TikTok: 720 s is longer than its 600 s limit; trim the video first.", + ); + expect(describeSkipped(tiktok, durationIssues(tiktok, 1500))).toBe( + "Not exported for TikTok: 1.5 s is shorter than its 3 s minimum; use a longer video.", + ); + }); +}); + +describe("platform check", () => { + it("passes a render within every tolerance", () => { + expect( + checkExport( + tiktok, + measured({ + videoKbps: 8000 * (1 + BITRATE_TOLERANCE), + fps: 30.2, + loudness: loudness(-14 - LOUDNESS_TOLERANCE_LU, -1 + TRUE_PEAK_TOLERANCE_DB), + }), + ), + ).toEqual({ status: "pass", issues: [] }); + }); + + it("warns about what the platform would degrade", () => { + expect( + checkExport( + tiktok, + measured({ + width: 720, + height: 1280, + fps: 60, + videoKbps: 9500, + loudness: loudness(-17.26, -0.2), + }), + ), + ).toEqual({ + status: "warn", + issues: [ + { + code: "resolution_mismatch", + width: 1080, + height: 1920, + actualWidth: 720, + actualHeight: 1280, + }, + { code: "fps_mismatch", maxFps: 30, actualFps: 60 }, + { code: "bitrate_high", maxKbps: 8000, actualKbps: 9500 }, + { code: "loudness_off", targetLufs: -14, actualLufs: -17.3 }, + { code: "true_peak_high", maxDb: -1, actualDb: -0.2 }, + ], + }); + }); + + it("fails what the platform would reject, most severe first", () => { + const check = checkExport( + tiktok, + measured({ codec: "hevc", sizeBytes: 300_456_789, loudness: loudness(-20) }), + ); + expect(check.status).toBe("fail"); + expect(check.issues).toEqual([ + { code: "file_too_large", maxMb: 287, actualMb: 300.5 }, + { code: "codec_mismatch" }, + { code: "loudness_off", targetLufs: -14, actualLufs: -20 }, + ]); + expect(checkExport(tiktok, measured({ durationMs: 601_000 })).issues[0]).toEqual({ + code: "too_long", + maxMs: 600_000, + actualMs: 601_000, + }); + }); + + it("does not judge the loudness of silent or missing audio, or an unknown frame rate", () => { + expect(checkExport(tiktok, measured({ loudness: null, fps: 0 })).status).toBe("pass"); + expect( + checkExport(tiktok, measured({ loudness: loudness(Number.NEGATIVE_INFINITY) })).status, + ).toBe("pass"); + expect(checkExport(tiktok, measured({ loudness: loudness(-65) })).status).toBe("pass"); + }); +}); + +describe("automatic fixes", () => { + it("re-encodes an oversized file once at a bitrate scaled from what it averaged", () => { + expect(sizeFixKbps(tiktok, measured(), 8000)).toBeNull(); + // 330 MB at 4400 kbps average: scale by 287/330 and keep 10% headroom. + const lower = sizeFixKbps(tiktok, measured({ sizeBytes: 330_000_000, videoKbps: 4400 }), 8000); + expect(lower).toBe(Math.floor(4400 * (287 / 330) * 0.9)); + // Without a stream bitrate, the file size and length give the average. + expect( + sizeFixKbps( + tiktok, + measured({ sizeBytes: 330_000_000, videoKbps: undefined, durationMs: 600_000 }), + 8000, + ), + ).toBe(Math.floor((4400 - 160) * (287 / 330) * 0.9)); + // Nothing lower is left to try. + expect( + sizeFixKbps(tiktok, measured({ sizeBytes: 330_000_000, videoKbps: 200 }), MIN_VIDEO_KBPS), + ).toBeNull(); + }); + + it("re-normalizes audible audio that missed the loudness target by more than the tolerance", () => { + expect(needsLoudnessFix(tiktok, measured())).toBe(false); + expect(needsLoudnessFix(tiktok, measured({ loudness: loudness(-16) }))).toBe(false); + expect(needsLoudnessFix(tiktok, measured({ loudness: loudness(-16.5) }))).toBe(true); + expect(needsLoudnessFix(tiktok, measured({ loudness: loudness(-10) }))).toBe(true); + expect(needsLoudnessFix(tiktok, measured({ loudness: null }))).toBe(false); + expect(needsLoudnessFix(tiktok, measured({ loudness: loudness(-70) }))).toBe(false); + }); +}); diff --git a/packages/adapters/src/media/platform-export.ts b/packages/adapters/src/media/platform-export.ts new file mode 100644 index 0000000..e887248 --- /dev/null +++ b/packages/adapters/src/media/platform-export.ts @@ -0,0 +1,221 @@ +import type { MediaCheck, MediaCheckIssue, MediaFit, PlatformProfile } from "@rakazo/contracts"; +import type { LoudnessMeasurement } from "./ffmpeg.js"; +import type { ExportTarget } from "./video-edit-plan.js"; +import { AUDIO_BITRATE_KBPS } from "./video-edit-plan.js"; + +/** + * Pure rules of a platform export: what to render, whether the result fits the platform, and + * which one-time fix to try when it does not. + */ + +/** Container durations differ from the cut by a few audio frames. */ +export const DURATION_TOLERANCE_MS = 100; +/** Average video bitrate above the platform's cap by more than this share earns a warning. */ +export const BITRATE_TOLERANCE = 0.1; +/** Frame rate above the platform's cap by more than this share earns a warning. */ +export const FPS_TOLERANCE = 0.01; +/** Integrated loudness this far from the target is re-normalized once, then warned about. */ +export const LOUDNESS_TOLERANCE_LU = 2; +/** True peak above the platform's ceiling by more than this earns a warning (AAC adds a little). */ +export const TRUE_PEAK_TOLERANCE_DB = 0.5; +/** Integrated loudness below this is silence: nothing to normalize or check. */ +export const SILENCE_LUFS = -60; +/** Share of the file size limit the first encode budgets for, and a re-encode after an overshoot. */ +export const SIZE_HEADROOM = 0.95; +export const SIZE_FIX_HEADROOM = 0.9; +/** Below this the picture falls apart; a file that still does not fit fails the check instead. */ +export const MIN_VIDEO_KBPS = 300; +const BYTES_PER_MB = 1_000_000; + +const FAILING_CODES = new Set([ + "too_long", + "too_short", + "file_too_large", + "codec_mismatch", +]); + +/** What a rendered file measured. */ +export interface ExportMeasurement { + width: number; + height: number; + /** 0 when unknown. */ + fps: number; + codec: string; + durationMs: number; + sizeBytes: number; + videoKbps?: number; + /** Null without an audio track. */ + loudness: LoudnessMeasurement | null; +} + +/** Highest video bitrate that keeps a file of this length under the platform's size limit. */ +export function sizeBudgetKbps( + profile: PlatformProfile, + durationMs: number, + headroom = SIZE_HEADROOM, +): number { + const totalKbps = (profile.video.maxFileSizeMb * BYTES_PER_MB * 8) / Math.max(1, durationMs); + return Math.max(MIN_VIDEO_KBPS, Math.floor(totalKbps * headroom - AUDIO_BITRATE_KBPS)); +} + +export function platformTarget( + profile: PlatformProfile, + fit: MediaFit, + durationMs: number, +): ExportTarget { + return { + width: profile.video.width, + height: profile.video.height, + fit, + maxFps: profile.video.fps, + maxBitrateKbps: Math.min(profile.video.maxBitrateKbps, sizeBudgetKbps(profile, durationMs)), + integratedLufs: profile.audio.targetLufs, + truePeakDb: profile.audio.truePeakDb, + }; +} + +/** Why a video of this length cannot go on the platform; empty when it can. */ +export function durationIssues(profile: PlatformProfile, durationMs: number): MediaCheckIssue[] { + const { hardMinMs, hardMaxMs } = profile.duration; + if (durationMs > hardMaxMs + DURATION_TOLERANCE_MS) { + return [{ code: "too_long", maxMs: hardMaxMs, actualMs: Math.round(durationMs) }]; + } + if (durationMs < hardMinMs - DURATION_TOLERANCE_MS) { + return [{ code: "too_short", minMs: hardMinMs, actualMs: Math.round(durationMs) }]; + } + return []; +} + +function audible(loudness: LoudnessMeasurement | null): loudness is LoudnessMeasurement { + return loudness !== null && loudness.integratedLufs > SILENCE_LUFS; +} + +/** Compares a rendered file with the platform's limits; failing issues come first. */ +export function checkExport(profile: PlatformProfile, measured: ExportMeasurement): MediaCheck { + const { video, audio } = profile; + const issues: MediaCheckIssue[] = [...durationIssues(profile, measured.durationMs)]; + const maxBytes = video.maxFileSizeMb * BYTES_PER_MB; + if (measured.sizeBytes > maxBytes) { + issues.push({ + code: "file_too_large", + maxMb: video.maxFileSizeMb, + actualMb: round(measured.sizeBytes / BYTES_PER_MB, 1), + }); + } + if (measured.codec !== video.codec) issues.push({ code: "codec_mismatch" }); + if (measured.width !== video.width || measured.height !== video.height) { + issues.push({ + code: "resolution_mismatch", + width: video.width, + height: video.height, + actualWidth: measured.width, + actualHeight: measured.height, + }); + } + if (measured.fps > video.fps * (1 + FPS_TOLERANCE)) { + issues.push({ code: "fps_mismatch", maxFps: video.fps, actualFps: round(measured.fps, 2) }); + } + if ( + measured.videoKbps !== undefined && + measured.videoKbps > video.maxBitrateKbps * (1 + BITRATE_TOLERANCE) + ) { + issues.push({ + code: "bitrate_high", + maxKbps: video.maxBitrateKbps, + actualKbps: Math.round(measured.videoKbps), + }); + } + if (audible(measured.loudness)) { + const { integratedLufs, truePeakDb } = measured.loudness; + if (Math.abs(integratedLufs - audio.targetLufs) > LOUDNESS_TOLERANCE_LU) { + issues.push({ + code: "loudness_off", + targetLufs: audio.targetLufs, + actualLufs: round(integratedLufs, 1), + }); + } + if (truePeakDb > audio.truePeakDb + TRUE_PEAK_TOLERANCE_DB) { + issues.push({ + code: "true_peak_high", + maxDb: audio.truePeakDb, + actualDb: round(truePeakDb, 1), + }); + } + } + const failing = issues.filter((issue) => FAILING_CODES.has(issue.code)); + const warning = issues.filter((issue) => !FAILING_CODES.has(issue.code)); + return { + status: failing.length ? "fail" : warning.length ? "warn" : "pass", + issues: [...failing, ...warning], + }; +} + +/** + * The lower bitrate cap for one re-encode of a file over the size limit, scaled from what the + * file actually averaged; null when it fits or a lower cap would not help. + */ +export function sizeFixKbps( + profile: PlatformProfile, + measured: ExportMeasurement, + currentKbps: number, +): number | null { + const maxBytes = profile.video.maxFileSizeMb * BYTES_PER_MB; + if (measured.sizeBytes <= maxBytes) return null; + const averageKbps = + measured.videoKbps ?? + (measured.sizeBytes * 8) / Math.max(1, measured.durationMs) - AUDIO_BITRATE_KBPS; + const scaled = Math.min(currentKbps, averageKbps) * (maxBytes / measured.sizeBytes); + const next = Math.max(MIN_VIDEO_KBPS, Math.floor(scaled * SIZE_FIX_HEADROOM)); + return next < currentKbps ? next : null; +} + +/** Whether one more loudness pass is worth it: the audio is audible and off the target. */ +export function needsLoudnessFix(profile: PlatformProfile, measured: ExportMeasurement): boolean { + return ( + audible(measured.loudness) && + Math.abs(measured.loudness.integratedLufs - profile.audio.targetLufs) > LOUDNESS_TOLERANCE_LU + ); +} + +/** An issue in a few English words for the bot's summary; the web words them itself. */ +export function describeIssue(issue: MediaCheckIssue): string { + switch (issue.code) { + case "too_long": + return `${seconds(issue.actualMs)} is longer than its ${seconds(issue.maxMs)} limit`; + case "too_short": + return `${seconds(issue.actualMs)} is shorter than its ${seconds(issue.minMs)} minimum`; + case "file_too_large": + return `${issue.actualMb} MB is larger than its ${issue.maxMb} MB limit`; + case "bitrate_high": + return `video bitrate ${issue.actualKbps} kbps is above its ${issue.maxKbps} kbps`; + case "loudness_off": + return `loudness ${issue.actualLufs} LUFS, target ${issue.targetLufs} LUFS`; + case "true_peak_high": + return `true peak ${issue.actualDb} dBTP is above its ${issue.maxDb} dBTP`; + case "resolution_mismatch": + return `frame ${issue.actualWidth}×${issue.actualHeight} instead of ${issue.width}×${issue.height}`; + case "fps_mismatch": + return `${issue.actualFps} fps is above its ${issue.maxFps} fps`; + case "codec_mismatch": + return "video is not H.264"; + } +} + +/** "Not exported for TikTok: 720 s is longer than its 600 s limit; trim the video first." */ +export function describeSkipped( + profile: Pick, + issues: readonly MediaCheckIssue[], +): string { + const advice = issues.some((issue) => issue.code === "too_short") + ? "use a longer video" + : "trim the video first"; + return `Not exported for ${profile.name}: ${issues.map(describeIssue).join("; ")}; ${advice}.`; +} + +function seconds(ms: number): string { + return `${Number((ms / 1000).toFixed(1))} s`; +} + +function round(value: number, digits: number): number { + return Number(value.toFixed(digits)); +} diff --git a/packages/adapters/src/media/transcription.test.ts b/packages/adapters/src/media/transcription.test.ts index 4b5f0ae..f47ac43 100644 --- a/packages/adapters/src/media/transcription.test.ts +++ b/packages/adapters/src/media/transcription.test.ts @@ -150,6 +150,7 @@ async function jobContext(source: Omit, input: unknow outputs.push({ ...file, body: await readFile(file.path, "utf8") }); return { artifactId: `out-${outputs.length}` } as MediaOutput; }, + addSkipped: async () => undefined, }; return { context, outputs, progress }; } diff --git a/packages/adapters/src/media/video-edit-plan.test.ts b/packages/adapters/src/media/video-edit-plan.test.ts index 4927f9e..ce64277 100644 --- a/packages/adapters/src/media/video-edit-plan.test.ts +++ b/packages/adapters/src/media/video-edit-plan.test.ts @@ -1,8 +1,10 @@ import type { EditVideoStep } from "@rakazo/contracts"; +import { MEDIA_PLATFORM_PROFILES } from "@rakazo/contracts"; import { describe, expect, it } from "vitest"; import type { MediaProbe } from "./ffmpeg.js"; +import { platformTarget } from "./platform-export.js"; import type { PlanInput } from "./video-edit-plan.js"; -import { atempoFilters, planVideoEdit, subtitleStyle } from "./video-edit-plan.js"; +import { atempoFilters, planLoudnessFix, planVideoEdit, subtitleStyle } from "./video-edit-plan.js"; function probe( overrides: Partial> & { @@ -298,3 +300,83 @@ describe("planVideoEdit", () => { expect(graphOf(evened.args)).toBe("[0:v:0]crop=720:404:0:0,format=yuv420p[v1]"); }); }); + +describe("platform export plans", () => { + const fast = { ...source, probe: probe({ fps: 120, durationMs: 20_000 }) }; + + it.each(Object.values(MEDIA_PLATFORM_PROFILES))( + "renders $name to its frame, frame rate, bitrate and loudness", + (profile) => { + const target = platformTarget(profile, "crop", 20_000); + const result = planVideoEdit({ + source: fast, + steps: [], + target, + outputPath: "/jobs/1/output.mp4", + }); + const { width, height, fps, maxBitrateKbps } = profile.video; + const graph = graphOf(result.args); + expect(graph).toContain(`fps=${fps}`); + expect(graph).toMatch(new RegExp(`scale=${width}:${height}[,:]`)); + expect(graph).toContain( + `loudnorm=I=${profile.audio.targetLufs}:TP=${profile.audio.truePeakDb}:LRA=11,atrim=end=20`, + ); + expect(target.maxBitrateKbps).toBe(maxBitrateKbps); + const maxrate = result.args.indexOf("-maxrate"); + expect(result.args.slice(maxrate, maxrate + 4)).toEqual([ + "-maxrate", + `${maxBitrateKbps}k`, + "-bufsize", + `${maxBitrateKbps * 2}k`, + ]); + expect(result).toMatchObject({ kind: "video", width, height, durationMs: 20_000 }); + }, + ); + + it("keeps a slower frame rate, fits by the export's fit, and skips loudness without audio", () => { + const target = platformTarget(MEDIA_PLATFORM_PROFILES.tiktok, "pad", 10_000); + const graph = graphOf( + planVideoEdit({ + source: { ...source, probe: probe({ fps: 24, hasAudio: false }) }, + steps: [], + target, + outputPath: "/jobs/1/output.mp4", + }).args, + ); + expect(graph).not.toContain("fps="); + expect(graph).toContain("pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black"); + expect(graph).not.toContain("loudnorm"); + // An unknown (variable) frame rate gets the platform's. + const variable = graphOf( + planVideoEdit({ + source: { ...source, probe: probe({ fps: 0 }) }, + steps: [], + target, + outputPath: "/jobs/1/output.mp4", + }).args, + ); + expect(variable).toContain("fps=30"); + }); + + it("plans a linear second loudness pass that copies the video", () => { + const args = planLoudnessFix({ + inputPath: "/jobs/1/tiktok.mp4", + outputPath: "/jobs/1/tiktok-loudness.mp4", + durationMs: 12_500, + target: { integratedLufs: -14, truePeakDb: -1 }, + measured: { + integratedLufs: -17.4, + truePeakDb: -4.2, + rangeLu: 5.1, + thresholdLufs: -27.6, + offsetLu: 0.3, + }, + }); + expect(args).toContain("file:/jobs/1/tiktok.mp4"); + expect(graphOf(args)).toBe( + "[0:a:0]loudnorm=I=-14:TP=-1:LRA=11:measured_I=-17.4:measured_TP=-4.2:measured_LRA=5.1:measured_thresh=-27.6:offset=0.3:linear=true,atrim=end=12.5,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a1]", + ); + expect(args.join(" ")).toContain("-map 0:v:0 -map [a1] -c:v copy -c:a aac -b:a 160k"); + expect(args.at(-1)).toBe("file:/jobs/1/tiktok-loudness.mp4"); + }); +}); diff --git a/packages/adapters/src/media/video-edit-plan.ts b/packages/adapters/src/media/video-edit-plan.ts index f39f8d5..68caa4d 100644 --- a/packages/adapters/src/media/video-edit-plan.ts +++ b/packages/adapters/src/media/video-edit-plan.ts @@ -1,7 +1,7 @@ import type { EditVideoStep, MediaFit } from "@rakazo/contracts"; import { MEDIA_PRESETS } from "@rakazo/contracts"; -import type { MediaProbe } from "./ffmpeg.js"; -import { ffmpegInputArgs, ffmpegSeconds } from "./ffmpeg.js"; +import type { LoudnessMeasurement, MediaProbe } from "./ffmpeg.js"; +import { ffmpegInputArgs, ffmpegSeconds, loudnormFilter } from "./ffmpeg.js"; /** * Turns an ordered list of edit steps into one ffmpeg invocation with a single filter_complex @@ -22,9 +22,22 @@ export interface VideoEditPlanRequest { clips?: ReadonlyMap; /** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */ subtitleFiles?: ReadonlyMap; + /** A platform's limits, applied after the steps. */ + target?: ExportTarget; outputPath: string; } +/** The frame, frame rate cap, bitrate cap and loudness a platform export is rendered to. */ +export interface ExportTarget { + width: number; + height: number; + fit: MediaFit; + maxFps: number; + maxBitrateKbps: number; + integratedLufs: number; + truePeakDb: number; +} + export interface VideoEditPlan { args: string[]; kind: "video" | "audio"; @@ -46,7 +59,8 @@ export const VIDEO_ENCODE_ARGS = [ "-movflags", "+faststart", ]; -const AUDIO_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "160k"]; +export const AUDIO_BITRATE_KBPS = 160; +const AUDIO_ENCODE_ARGS = ["-c:a", "aac", "-b:a", `${AUDIO_BITRATE_KBPS}k`]; const AUDIO_ONLY_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"]; /** Location and other camera metadata stay private; chapters do not survive cuts anyway. */ const STRIP_METADATA_ARGS = ["-map_metadata", "-1", "-map_chapters", "-1"]; @@ -229,6 +243,8 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { `poster at ${seconds(posterAt)} s is past the end of the ${seconds(timeline.seconds)} s result.`, ); } + const { target } = request; + if (target && !audioOnly) applyTarget(graph, timeline, target); const durationMs = Math.round(timeline.seconds * 1000); const output = `file:${request.outputPath}`; if (audioOnly || !timeline.video) { @@ -268,6 +284,7 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { `[${videoOut}]`, ...(audioOut ? ["-map", `[${audioOut}]`, ...AUDIO_ENCODE_ARGS] : []), ...VIDEO_ENCODE_ARGS, + ...(target && !audioOnly ? bitrateCapArgs(target.maxBitrateKbps) : []), ...STRIP_METADATA_ARGS, "-f", "mp4", @@ -281,6 +298,64 @@ export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan { }; } +/** + * Fits the result to a platform: frame rate cap, frame size and loudness. Loudness is one + * dynamic loudnorm pass, which lands within about 1 LU of the target on most material without + * decoding the source twice; the export measures the result and corrects it when it misses. + */ +function applyTarget(graph: FilterGraph, timeline: Timeline, target: ExportTarget) { + if (timeline.video) { + if (!timeline.fps || timeline.fps > target.maxFps + 0.01) { + timeline.video.filters.push(`fps=${number(target.maxFps)}`); + timeline.fps = target.maxFps; + } + timeline.video = fitFrame(graph, timeline.video, timeline, target, target.fit); + } + timeline.width = target.width; + timeline.height = target.height; + // loudnorm pads its last frame; trim the audio back to the video's length. + timeline.audio?.filters.push( + loudnormFilter(target), + `atrim=end=${ffmpegSeconds(timeline.seconds)}`, + ); +} + +/** CRF quality, capped so no stretch of the video runs above the platform's bitrate. */ +function bitrateCapArgs(maxBitrateKbps: number): string[] { + const kbps = Math.max(1, Math.round(maxBitrateKbps)); + return ["-maxrate", `${kbps}k`, "-bufsize", `${kbps * 2}k`]; +} + +/** + * Second loudness pass over a rendered MP4: linear loudnorm with the measured values on the + * audio, the video copied as it is. + */ +export function planLoudnessFix(request: { + inputPath: string; + outputPath: string; + durationMs: number; + target: { integratedLufs: number; truePeakDb: number }; + measured: LoudnessMeasurement; +}): string[] { + const filters = [ + loudnormFilter(request.target, request.measured), + `atrim=end=${ffmpegSeconds(request.durationMs / 1000)}`, + AUDIO_FORMAT, + ]; + return [ + ...ffmpegInputArgs(request.inputPath, { mimeType: "video/mp4" }), + "-filter_complex", + `[0:a:0]${filters.join(",")}[a1]`, + ...["-map", "0:v:0", "-map", "[a1]", "-c:v", "copy"], + ...AUDIO_ENCODE_ARGS, + ...["-movflags", "+faststart"], + ...STRIP_METADATA_ARGS, + "-f", + "mp4", + `file:${request.outputPath}`, + ]; +} + interface Segment { start: number; end: number; diff --git a/packages/adapters/src/media/video-operations.postgres.test.ts b/packages/adapters/src/media/video-operations.postgres.test.ts index 7f5592d..5cdd51d 100644 --- a/packages/adapters/src/media/video-operations.postgres.test.ts +++ b/packages/adapters/src/media/video-operations.postgres.test.ts @@ -176,6 +176,63 @@ describePostgres("video operations in the media runtime (PostgreSQL + ffmpeg)", ]); }); + it("exports for a platform with its check and shows a skipped platform on the card", async () => { + const { runtime, scope, store } = await setup(); + const source = await store(clips.landscape, "holiday.mp4", "video/mp4"); + const started = await runtime.start({ + ...scope, + operation: "export_video", + input: { artifact_id: source, platforms: ["tiktok", "snapchat-spotlight"] }, + }); + await runtime.runJob(started.jobId); + const job = await runtime.get(started.jobId); + expect(job).toMatchObject({ status: "done", title: "TikTok · Snapchat Spotlight" }); + expect(job?.outputs).toEqual([ + expect.objectContaining({ + name: "holiday-tiktok.mp4", + platform: "tiktok", + width: 1080, + height: 1920, + check: { status: "pass", issues: [] }, + }), + ]); + const stored = await prisma.artifact.findUniqueOrThrow({ + where: { id: job!.outputs[0]!.artifactId }, + }); + expect(stored).toMatchObject({ parentArtifactId: source, width: 1080, height: 1920 }); + expect(await card(started.messageId, started.jobId)).toMatchObject({ + status: "done", + outputs: job?.outputs, + skipped: [ + { + platform: "snapchat-spotlight", + issues: [{ code: "too_short", minMs: 5000, actualMs: 3000 }], + }, + ], + }); + }); + + it("fails an export that fits none of its platforms and keeps the reason on the card", async () => { + const { runtime, scope, store } = await setup(); + const source = await store(clips.landscape, "short.mp4", "video/mp4"); + const started = await runtime.start({ + ...scope, + operation: "export_video", + input: { artifact_id: source, platforms: ["snapchat-spotlight"] }, + }); + await runtime.runJob(started.jobId); + expect(await runtime.get(started.jobId)).toMatchObject({ + status: "failed", + error: + "Not exported for Snapchat Spotlight: 3 s is shorter than its 5 s minimum; use a longer video.", + outputs: [], + }); + expect(await card(started.messageId, started.jobId)).toMatchObject({ + status: "failed", + skipped: [{ platform: "snapchat-spotlight", issues: [{ code: "too_short" }] }], + }); + }); + it("refuses appended clips and subtitles the user cannot read", async () => { const { runtime, scope, store } = await setup(); const other = await setup(); diff --git a/packages/adapters/src/media/video-operations.test.ts b/packages/adapters/src/media/video-operations.test.ts index 1d8d6f9..ddd3ad0 100644 --- a/packages/adapters/src/media/video-operations.test.ts +++ b/packages/adapters/src/media/video-operations.test.ts @@ -1,13 +1,15 @@ import { rm } from "node:fs/promises"; import { join } from "node:path"; +import { MEDIA_PLATFORM_PROFILES, MediaPlatformIdSchema } from "@rakazo/contracts"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { builtinAgentTools } from "../builtin-tools.js"; import { createMediaOperations } from "../media-operations.js"; import { MEDIA_CONTINUE_AFTER_PROPERTY } from "../media-tools.js"; import { parametersFor } from "../pi-runtime.js"; -import { detectFfmpeg, probeMedia, runFfmpeg } from "./ffmpeg.js"; +import { detectFfmpeg, measureLoudness, probeMedia, runFfmpeg } from "./ffmpeg.js"; import type { TestArtifact, TestClips } from "./media-test-support.js"; import { createTestClips, fakeMediaJobContext } from "./media-test-support.js"; +import { LOUDNESS_TOLERANCE_LU } from "./platform-export.js"; import { cleanFileName, createVideoOperations, subtitleFormat } from "./video-operations.js"; const ffmpeg = await detectFfmpeg(); @@ -48,13 +50,22 @@ describe("video operation registrations", () => { expect(exported.artifactIds?.(exportInput)).toEqual(["src"]); }); - it("titles exports by aspect ratio unless the bot gave a title", () => { + it("titles exports by platform and aspect ratio unless the bot gave a title", () => { const exported = operation("export_video"); expect( exported.title( exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical", "square"] }), ), ).toBe("9:16 · 1:1"); + expect( + exported.title( + exported.inputSchema.parse({ + artifact_id: "a", + platforms: ["tiktok", "youtube-shorts"], + presets: ["square"], + }), + ), + ).toBe("TikTok · YouTube Shorts · 1:1"); expect( exported.title( exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical"], title: "Reel" }), @@ -70,6 +81,18 @@ describe("video operation registrations", () => { expect(() => parametersFor(tool!)).not.toThrow(); } }); + + it("tells the model every platform id with its longest video", () => { + const tool = builtinAgentTools.find((entry) => entry.name === "export_video")!; + const properties = tool.inputSchema.properties as Record< + string, + { items?: { enum?: string[] } } + >; + expect(properties.platforms?.items?.enum).toEqual(MediaPlatformIdSchema.options); + for (const profile of Object.values(MEDIA_PLATFORM_PROFILES)) { + expect(tool.description).toContain(`${profile.id} ${profile.duration.hardMaxMs / 1000} s`); + } + }); }); describe("subtitle and file name helpers", () => { @@ -251,6 +274,74 @@ describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => { expect(progress.at(-1)).toBe(1); }); + it("exports for a platform at its frame, frame rate and loudness, and skips one it is too short for", async () => { + const fast = join(clips.dir, "fast.mp4"); + await runFfmpeg([ + ...["-f", "lavfi", "-i", "testsrc=size=320x180:rate=60:duration=4"], + ...["-f", "lavfi", "-i", "sine=frequency=330:sample_rate=48000:duration=4"], + ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"], + ...["-c:a", "aac", "-shortest", `file:${fast}`], + ]); + const registration = operation("export_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ + artifact_id: "fast", + platforms: ["tiktok", "snapchat-spotlight"], + }), + { fast: { path: fast, name: "Fast clip.mp4", mimeType: "video/mp4" } }, + ); + try { + const result = await registration.handler.run(job.context); + expect(job.skipped).toEqual([ + { + platform: "snapchat-spotlight", + issues: [{ code: "too_short", minMs: 5000, actualMs: expect.any(Number) }], + }, + ]); + expect(job.outputs).toHaveLength(1); + const [output] = job.outputs; + expect(output).toMatchObject({ + name: "Fast clip-tiktok.mp4", + platform: "tiktok", + width: 1080, + height: 1920, + parentArtifactId: "fast", + check: { status: "pass", issues: [] }, + }); + expect(output!.posterSize).toBeGreaterThan(0); + const probe = await probeMedia(output!.path, { mimeType: "video/mp4" }); + expect(probe.video).toMatchObject({ codec: "h264", width: 1080, height: 1920, fps: 30 }); + expect(Math.abs(probe.durationMs - 4000)).toBeLessThan(100); + const loudness = await measureLoudness( + output!.path, + { integratedLufs: -14, truePeakDb: -1 }, + { mimeType: "video/mp4" }, + ); + expect(Math.abs(loudness.integratedLufs + 14)).toBeLessThanOrEqual(LOUDNESS_TOLERANCE_LU); + expect(result.summary).toContain("Fast clip-tiktok.mp4 for TikTok (1080×1920, 4 s)"); + expect(result.summary).toContain( + "Not exported for Snapchat Spotlight: 4 s is shorter than its 5 s minimum; use a longer video.", + ); + expect(job.progress.at(-1)).toBe(1); + } finally { + await rm(job.tempDir, { recursive: true, force: true }); + } + }); + + it("fails without rendering when the video fits none of the platforms", async () => { + const registration = operation("export_video"); + const job = await fakeMediaJobContext( + registration.inputSchema.parse({ artifact_id: "land", platforms: ["snapchat-spotlight"] }), + artifacts, + ); + await expect(registration.handler.run(job.context)).rejects.toThrow( + "Not exported for Snapchat Spotlight: 3 s is shorter than its 5 s minimum; use a longer video.", + ); + expect(job.skipped.map((target) => target.platform)).toEqual(["snapchat-spotlight"]); + expect(job.outputs).toEqual([]); + await rm(job.tempDir, { recursive: true, force: true }); + }); + it("fails clearly on ranges past the end, non-subtitle files and non-videos", async () => { await expect( run("edit_video", { diff --git a/packages/adapters/src/media/video-operations.ts b/packages/adapters/src/media/video-operations.ts index e4e1e6d..180bcff 100644 --- a/packages/adapters/src/media/video-operations.ts +++ b/packages/adapters/src/media/video-operations.ts @@ -1,24 +1,49 @@ -import { copyFile, mkdtemp, open, stat } from "node:fs/promises"; +import { copyFile, mkdtemp, open, rename, stat } from "node:fs/promises"; import { join } from "node:path"; -import type { MediaJobContext, MediaJobInputFile, MediaOperationResult } from "@rakazo/adapter-kit"; -import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts"; +import type { + MediaJobContext, + MediaJobInputFile, + MediaJobOutputFile, + MediaOperationResult, +} from "@rakazo/adapter-kit"; +import type { + EditVideoInput, + ExportVideoInput, + MediaFit, + MediaOutput, + MediaPlatformId, + MediaSkippedTarget, + PlatformProfile, +} from "@rakazo/contracts"; import { ARTIFACT_MAX_BYTES_BY_KIND, EditVideoInputSchema, ExportVideoInputSchema, + MEDIA_PLATFORM_PROFILES, MEDIA_PRESETS, } from "@rakazo/contracts"; import type { MediaOperationRegistration } from "../media-runtime.js"; -import type { FfmpegInfo } from "./ffmpeg.js"; +import type { FfmpegInfo, MediaProbe } from "./ffmpeg.js"; import { detectFfmpeg, + measureLoudness, outputTooLargeMessage, probeMedia, renderPoster, runFfmpeg, } from "./ffmpeg.js"; +import type { ExportMeasurement } from "./platform-export.js"; +import { + checkExport, + describeIssue, + describeSkipped, + durationIssues, + needsLoudnessFix, + platformTarget, + sizeFixKbps, +} from "./platform-export.js"; import type { PlanInput, VideoEditPlan } from "./video-edit-plan.js"; -import { planVideoEdit } from "./video-edit-plan.js"; +import { planLoudnessFix, planVideoEdit } from "./video-edit-plan.js"; export interface VideoOperationsDeps { /** Finds ffmpeg; defaults to PATH lookup. */ @@ -66,7 +91,11 @@ export function createVideoOperations( operation: "export_video", inputSchema: ExportVideoInputSchema, title: (input) => - input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "), + input.title ?? + [ + ...(input.platforms ?? []).map((platform) => MEDIA_PLATFORM_PROFILES[platform].name), + ...(input.presets ?? []).map((preset) => MEDIA_PRESETS[preset].aspect), + ].join(" · "), available, artifactIds: (input) => [input.artifact_id], handler: { @@ -142,39 +171,160 @@ async function runEditVideo(context: MediaJobContext): Promise { const input = ExportVideoInputSchema.parse(context.input); - const presets = [...new Set(input.presets)]; + const platforms = [...new Set(input.platforms ?? [])]; + const presets = [...new Set(input.presets ?? [])]; const workDir = await mkdtemp(join(context.tempDir, "export-")); const progress = createProgressReporter(context); const source = await loadVideo(context, input.artifact_id); const base = cleanFileName(withoutExtension(source.name), "video"); + // A platform the video is too long or too short for is not rendered; the bot trims first. + const skipped: MediaSkippedTarget[] = []; + const rendered: MediaPlatformId[] = []; + for (const platform of platforms) { + const issues = durationIssues(MEDIA_PLATFORM_PROFILES[platform], source.probe.durationMs); + if (issues.length) skipped.push({ platform, issues }); + else rendered.push(platform); + } + for (const target of skipped) await context.addSkipped(target); + const notExported = skipped.map(({ platform, issues }) => + describeSkipped(MEDIA_PLATFORM_PROFILES[platform], issues), + ); + const count = rendered.length + presets.length; + if (count === 0) throw new Error(notExported.join(" ")); + const share = 1 / count; + const report = (index: number) => progress.range(index * share, (index + 1) * share); const outputs: MediaOutput[] = []; - for (const [index, preset] of presets.entries()) { + for (const platform of rendered) { + outputs.push( + await exportForPlatform(context, source, MEDIA_PLATFORM_PROFILES[platform], { + fit: input.fit, + name: `${base}-${platform}.mp4`, + workDir, + report: report(outputs.length), + }), + ); + } + for (const preset of presets) { const outputPath = join(workDir, `${preset}.mp4`); const plan = planVideoEdit({ source, steps: [{ op: "reframe", preset, fit: input.fit }], outputPath, }); - const share = 1 / presets.length; outputs.push( - await encode( - context, - plan, - outputPath, - workDir, - progress.range(index * share, (index + 1) * share), - { - name: `${base}-${MEDIA_PRESETS[preset].aspect.replace(":", "x")}.mp4`, - parentArtifactId: source.artifactId, - preset, - }, - ), + await encode(context, plan, outputPath, workDir, report(outputs.length), { + name: `${base}-${MEDIA_PRESETS[preset].aspect.replace(":", "x")}.mp4`, + parentArtifactId: source.artifactId, + preset, + }), ); } await progress.done(); - return { summary: `Exported ${source.name} as ${outputs.map(describe).join(", ")}.` }; + const checks = outputs.flatMap((output) => + output.platform && output.check && output.check.status !== "pass" + ? [ + `${MEDIA_PLATFORM_PROFILES[output.platform].name} check ${output.check.status}: ${output.check.issues.map(describeIssue).join("; ")}.`, + ] + : [], + ); + return { + summary: [ + `Exported ${source.name} as ${outputs.map(describe).join(", ")}.`, + ...checks, + ...notExported, + ].join(" "), + }; } +/** + * Renders a video to a platform's frame, frame rate, bitrate and loudness, measures the file + * and stores it with the check. A file over the size limit is encoded once more at a lower + * bitrate, and audio off the loudness target gets one more, linear loudness pass. + */ +async function exportForPlatform( + context: MediaJobContext, + source: PlanInput & MediaJobInputFile, + profile: PlatformProfile, + options: { fit: MediaFit; name: string; workDir: string; report: (fraction: number) => void }, +): Promise { + const { workDir, report } = options; + const outputPath = join(workDir, `${profile.id}.mp4`); + let target = platformTarget(profile, options.fit, source.probe.durationMs); + const renderTarget = async () => { + const plan = planVideoEdit({ source, steps: [], target, outputPath }); + await render(context, plan, outputPath, workDir, report); + return plan; + }; + let plan = await renderTarget(); + let measured = await measureExport(context, outputPath, profile); + const lowerKbps = sizeFixKbps(profile, measured.result, target.maxBitrateKbps); + if (lowerKbps !== null) { + target = { ...target, maxBitrateKbps: lowerKbps }; + plan = await renderTarget(); + measured = await measureExport(context, outputPath, profile); + } + if (needsLoudnessFix(profile, measured.result) && measured.result.loudness) { + const fixedPath = join(workDir, `${profile.id}-loudness.mp4`); + await runFfmpeg( + planLoudnessFix({ + inputPath: outputPath, + outputPath: fixedPath, + durationMs: measured.result.durationMs, + target, + measured: measured.result.loudness, + }), + { + signal: context.signal, + cwd: workDir, + maxOutputBytes: ARTIFACT_MAX_BYTES_BY_KIND.video, + outputs: [fixedPath], + }, + ); + await rename(fixedPath, outputPath); + measured = await measureExport(context, outputPath, profile); + } + return store(context, plan, outputPath, measured.probe, report, { + name: options.name, + parentArtifactId: source.artifactId, + platform: profile.id, + check: checkExport(profile, measured.result), + }); +} + +async function measureExport( + context: MediaJobContext, + path: string, + profile: PlatformProfile, +): Promise<{ probe: MediaProbe; result: ExportMeasurement }> { + const options = { signal: context.signal, mimeType: "video/mp4" }; + const probe = await probeMedia(path, options); + const loudness = probe.hasAudio + ? await measureLoudness( + path, + { integratedLufs: profile.audio.targetLufs, truePeakDb: profile.audio.truePeakDb }, + options, + ) + : null; + return { + probe, + result: { + width: probe.video?.width ?? 0, + height: probe.video?.height ?? 0, + fps: probe.video?.fps ?? 0, + codec: probe.video?.codec ?? "", + durationMs: probe.durationMs, + sizeBytes: (await stat(path)).size, + ...(probe.video?.bitRateKbps !== undefined ? { videoKbps: probe.video.bitRateKbps } : {}), + loudness, + }, + }; +} + +type OutputDetails = Pick< + MediaJobOutputFile, + "name" | "parentArtifactId" | "preset" | "platform" | "check" +>; + /** Runs a plan, checks the result, renders its poster and stores it as an output. */ async function encode( context: MediaJobContext, @@ -182,8 +332,24 @@ async function encode( outputPath: string, workDir: string, report: (fraction: number) => void, - file: Pick[0], "name" | "parentArtifactId" | "preset">, + file: OutputDetails, ): Promise { + await render(context, plan, outputPath, workDir, report); + const probe = await probeMedia(outputPath, { + signal: context.signal, + mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4", + }); + return store(context, plan, outputPath, probe, report, file); +} + +/** Runs a plan's ffmpeg, refusing a result larger than an artifact may be. */ +async function render( + context: MediaJobContext, + plan: VideoEditPlan, + outputPath: string, + workDir: string, + report: (fraction: number) => void, +): Promise { const maxBytes = ARTIFACT_MAX_BYTES_BY_KIND[plan.kind]; await runFfmpeg(plan.args, { signal: context.signal, @@ -194,10 +360,17 @@ async function encode( outputs: [outputPath], }); if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes)); - const probe = await probeMedia(outputPath, { - signal: context.signal, - mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4", - }); +} + +/** Renders a video's poster and stores the file as an output. */ +async function store( + context: MediaJobContext, + plan: VideoEditPlan, + outputPath: string, + probe: MediaProbe, + report: (fraction: number) => void, + file: OutputDetails, +): Promise { if (plan.kind === "audio") { const output = await context.addOutput({ ...file, @@ -294,13 +467,16 @@ function withoutExtension(name: string): string { return name.replace(/\.[A-Za-z0-9]{1,5}$/, ""); } -/** "name (1080×1920, 12.5 s)" for the bot's summary; the runtime adds the artifact ids. */ +/** "name for TikTok (1080×1920, 12.5 s)" for the bot's summary; the runtime adds the ids. */ function describe(output: MediaOutput): string { + const name = output.platform + ? `${output.name} for ${MEDIA_PLATFORM_PROFILES[output.platform].name}` + : output.name; const details = [ output.width && output.height ? `${output.width}×${output.height}` : "", output.durationMs !== undefined ? `${Number((output.durationMs / 1000).toFixed(1))} s` : "", ].filter(Boolean); - return details.length ? `${output.name} (${details.join(", ")})` : output.name; + return details.length ? `${name} (${details.join(", ")})` : name; } /** Maps ffmpeg progress onto the card: monotonic, written in order, never failing the job. */ diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 5f265fe..928f122 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -18,6 +18,8 @@ export * from "./media.js"; export * from "./media-generation.js"; export * from "./media-jobs.js"; export * from "./media-library.js"; +export * from "./media-platforms.js"; +export * from "./media-presets.js"; export * from "./media-settings.js"; export * from "./media-transcription.js"; export * from "./media-video.js"; diff --git a/packages/contracts/src/media-generation.ts b/packages/contracts/src/media-generation.ts index b834196..959f027 100644 --- a/packages/contracts/src/media-generation.ts +++ b/packages/contracts/src/media-generation.ts @@ -1,6 +1,6 @@ import * as z from "zod"; import { Id } from "./ids.js"; -import { MediaFitSchema, MediaPresetIdSchema } from "./media.js"; +import { MediaFitSchema, MediaPresetIdSchema } from "./media-presets.js"; import { MediaTitleSchema } from "./media-video.js"; export const MEDIA_IMAGE_MAX_COUNT = 4; diff --git a/packages/contracts/src/media-platforms.test.ts b/packages/contracts/src/media-platforms.test.ts new file mode 100644 index 0000000..881250d --- /dev/null +++ b/packages/contracts/src/media-platforms.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { + MEDIA_PLATFORM_PROFILES, + MediaCheckSchema, + MediaPlatformIdSchema, + PlatformProfileSchema, +} from "./media-platforms.js"; +import { MEDIA_PRESETS } from "./media-presets.js"; + +describe("platform profiles", () => { + it("ships one valid profile per platform id, keyed by its id", () => { + expect(Object.keys(MEDIA_PLATFORM_PROFILES)).toEqual(MediaPlatformIdSchema.options); + for (const [id, profile] of Object.entries(MEDIA_PLATFORM_PROFILES)) { + expect(PlatformProfileSchema.parse(profile)).toEqual(profile); + expect(profile.id).toBe(id); + expect(profile.sourceUrls.every((url) => url.startsWith("https://"))).toBe(true); + } + }); + + it("uses only the aspect ratios the presets know, at a size of that ratio", () => { + const aspects = new Set(Object.values(MEDIA_PRESETS).map((preset) => preset.aspect)); + for (const profile of Object.values(MEDIA_PLATFORM_PROFILES)) { + expect(aspects.has(profile.video.aspect)).toBe(true); + } + const tiktok = MEDIA_PLATFORM_PROFILES.tiktok; + expect( + PlatformProfileSchema.safeParse({ ...tiktok, video: { ...tiktok.video, aspect: "3:2" } }) + .success, + ).toBe(false); + expect( + PlatformProfileSchema.safeParse({ ...tiktok, video: { ...tiktok.video, width: 1920 } }) + .success, + ).toBe(false); + expect( + PlatformProfileSchema.safeParse({ + ...tiktok, + duration: { hardMinMs: 5000, hardMaxMs: 4000 }, + }).success, + ).toBe(false); + }); + + it("keeps the research provenance on every profile", () => { + for (const profile of Object.values(MEDIA_PLATFORM_PROFILES)) { + expect(profile.reviewedAt).toMatch(/^\d{4}-\d{2}-\d{2}$/); + expect(profile.unverified).toBe(true); + } + }); +}); + +describe("platform check", () => { + it("carries language-neutral issue codes with their numbers", () => { + expect( + MediaCheckSchema.parse({ + status: "fail", + issues: [ + { code: "too_long", maxMs: 600_000, actualMs: 720_000 }, + { code: "codec_mismatch" }, + ], + }).issues, + ).toHaveLength(2); + expect( + MediaCheckSchema.safeParse({ status: "warn", issues: [{ code: "too_blurry" }] }).success, + ).toBe(false); + expect( + MediaCheckSchema.safeParse({ status: "fail", issues: [{ code: "too_long", maxMs: 1 }] }) + .success, + ).toBe(false); + }); +}); diff --git a/packages/contracts/src/media-platforms.ts b/packages/contracts/src/media-platforms.ts new file mode 100644 index 0000000..568c21c --- /dev/null +++ b/packages/contracts/src/media-platforms.ts @@ -0,0 +1,282 @@ +import * as z from "zod"; +import type { MediaPreset } from "./media-presets.js"; +import { MEDIA_PRESETS } from "./media-presets.js"; + +/** + * Platforms a video can be exported for. An export renders each one to the platform's + * technical limits and checks the file against them, so an upload is not rejected or degraded. + */ +export const MediaPlatformIdSchema = z.enum([ + "tiktok", + "instagram-reels", + "instagram-feed-4x5", + "youtube-shorts", + "youtube", + "facebook-reels", + "linkedin", + "x-twitter", + "pinterest-video-pin", + "snapchat-spotlight", +]); +export type MediaPlatformId = z.infer; + +const PRESET_ASPECTS = Object.values(MEDIA_PRESETS).map((preset) => preset.aspect) as [ + MediaPreset["aspect"], + ...MediaPreset["aspect"][], +]; + +const Percent = z.number().min(0).max(50); + +/** + * The technical limits of a platform. Content, disclosure and caption rules need a review flow + * and are not part of a profile. + */ +export const PlatformProfileSchema = z + .object({ + id: MediaPlatformIdSchema, + /** Brand name, shown untranslated. */ + name: z.string().min(1).max(40), + /** Where the limits come from. */ + sourceUrls: z.array(z.httpUrl()).min(1), + reviewedAt: z.iso.date(), + /** True until someone checked the limits against the platform's current documentation. */ + unverified: z.boolean(), + video: z.object({ + aspect: z.enum(PRESET_ASPECTS), + width: z.number().int().positive().multipleOf(2), + height: z.number().int().positive().multipleOf(2), + /** Highest frame rate; slower sources keep theirs. */ + fps: z.number().positive().max(120), + codec: z.literal("h264"), + maxBitrateKbps: z.number().int().positive(), + maxFileSizeMb: z.number().positive(), + }), + duration: z + .object({ + hardMinMs: z.number().int().nonnegative(), + hardMaxMs: z.number().int().positive(), + }) + .refine((duration) => duration.hardMinMs < duration.hardMaxMs, "hardMinMs must be lower"), + /** Edges the platform covers with its own buttons and captions, in percent of the frame. */ + safeZones: z.object({ + topPct: Percent, + bottomPct: Percent, + leftPct: Percent, + rightPct: Percent, + }), + audio: z.object({ + targetLufs: z.number().min(-40).max(0), + truePeakDb: z.number().min(-10).max(0), + }), + }) + .refine((profile) => { + const [across, down] = profile.video.aspect.split(":").map(Number); + return Math.abs(profile.video.width / profile.video.height - across! / down!) < 0.01; + }, "width and height must match the aspect ratio"); +export type PlatformProfile = z.infer; + +/** + * What a platform check found, with the numbers the UI words it with. Durations in ms, sizes in + * MB (10^6 bytes), bitrates in kbps, loudness in LUFS, peaks in dBTP. + */ +export const MediaCheckIssueSchema = z.discriminatedUnion("code", [ + z.object({ code: z.literal("too_long"), maxMs: z.number(), actualMs: z.number() }), + z.object({ code: z.literal("too_short"), minMs: z.number(), actualMs: z.number() }), + z.object({ code: z.literal("file_too_large"), maxMb: z.number(), actualMb: z.number() }), + z.object({ code: z.literal("bitrate_high"), maxKbps: z.number(), actualKbps: z.number() }), + z.object({ code: z.literal("loudness_off"), targetLufs: z.number(), actualLufs: z.number() }), + z.object({ code: z.literal("true_peak_high"), maxDb: z.number(), actualDb: z.number() }), + z.object({ + code: z.literal("resolution_mismatch"), + width: z.number(), + height: z.number(), + actualWidth: z.number(), + actualHeight: z.number(), + }), + z.object({ code: z.literal("fps_mismatch"), maxFps: z.number(), actualFps: z.number() }), + z.object({ code: z.literal("codec_mismatch") }), +]); +export type MediaCheckIssue = z.infer; + +/** `fail`: the platform would reject the file. `warn`: it would accept and degrade it. */ +export const MediaCheckSchema = z.object({ + status: z.enum(["pass", "warn", "fail"]), + /** Most severe first. */ + issues: z.array(MediaCheckIssueSchema).max(20), +}); +export type MediaCheck = z.infer; + +/** A platform an export did not render, e.g. because the video is too long for it. */ +export const MediaSkippedTargetSchema = z.object({ + platform: MediaPlatformIdSchema, + issues: z.array(MediaCheckIssueSchema).min(1).max(20), +}); +export type MediaSkippedTarget = z.infer; + +const VERTICAL_SAFE_ZONES = { topPct: 10, bottomPct: 20, leftPct: 5, rightPct: 15 }; +const FRAMED_SAFE_ZONES = { topPct: 5, bottomPct: 10, leftPct: 5, rightPct: 5 }; +const STREAMING_LOUDNESS = { targetLufs: -14, truePeakDb: -1 }; +const VERTICAL_1080 = { aspect: "9:16", width: 1080, height: 1920 } as const; +const REVIEWED_AT = "2026-10-03"; + +/** + * Researched limits, all unverified: each needs a check against the linked documentation before + * it is trusted. YouTube allows 60 fps per its encoding guide; the others are capped at 30. + */ +export const MEDIA_PLATFORM_PROFILES: Readonly> = { + tiktok: { + id: "tiktok", + name: "TikTok", + sourceUrls: ["https://support.tiktok.com/en/using-tiktok/creating-videos/upload-videos"], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { ...VERTICAL_1080, fps: 30, codec: "h264", maxBitrateKbps: 8000, maxFileSizeMb: 287 }, + duration: { hardMinMs: 3000, hardMaxMs: 600_000 }, + safeZones: VERTICAL_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, + "instagram-reels": { + id: "instagram-reels", + name: "Instagram Reels", + sourceUrls: ["https://help.instagram.com/", "https://www.facebook.com/business/ads-guide/"], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { ...VERTICAL_1080, fps: 30, codec: "h264", maxBitrateKbps: 5000, maxFileSizeMb: 1024 }, + duration: { hardMinMs: 3000, hardMaxMs: 90_000 }, + safeZones: VERTICAL_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, + "instagram-feed-4x5": { + id: "instagram-feed-4x5", + name: "Instagram Feed", + sourceUrls: ["https://help.instagram.com/", "https://www.facebook.com/business/ads-guide/"], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { + aspect: "4:5", + width: 1080, + height: 1350, + fps: 30, + codec: "h264", + maxBitrateKbps: 5000, + maxFileSizeMb: 1024, + }, + duration: { hardMinMs: 3000, hardMaxMs: 60_000 }, + safeZones: FRAMED_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, + "youtube-shorts": { + id: "youtube-shorts", + name: "YouTube Shorts", + sourceUrls: [ + "https://support.google.com/youtube/answer/15424877", + "https://support.google.com/youtube/answer/10059070", + "https://support.google.com/youtube/answer/1722171", + ], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { + ...VERTICAL_1080, + fps: 60, + codec: "h264", + maxBitrateKbps: 12_000, + maxFileSizeMb: 2048, + }, + duration: { hardMinMs: 1000, hardMaxMs: 180_000 }, + safeZones: VERTICAL_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, + youtube: { + id: "youtube", + name: "YouTube", + sourceUrls: [ + "https://support.google.com/youtube/answer/1722171", + "https://support.google.com/youtube/answer/71673", + ], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { + aspect: "16:9", + width: 1920, + height: 1080, + fps: 60, + codec: "h264", + maxBitrateKbps: 8000, + maxFileSizeMb: 4096, + }, + duration: { hardMinMs: 1000, hardMaxMs: 900_000 }, + safeZones: FRAMED_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, + "facebook-reels": { + id: "facebook-reels", + name: "Facebook Reels", + sourceUrls: ["https://www.facebook.com/help/", "https://www.facebook.com/business/ads-guide/"], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { ...VERTICAL_1080, fps: 30, codec: "h264", maxBitrateKbps: 5000, maxFileSizeMb: 1024 }, + duration: { hardMinMs: 3000, hardMaxMs: 90_000 }, + safeZones: VERTICAL_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, + linkedin: { + id: "linkedin", + name: "LinkedIn", + sourceUrls: ["https://www.linkedin.com/help/linkedin/"], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { + aspect: "16:9", + width: 1920, + height: 1080, + fps: 30, + codec: "h264", + maxBitrateKbps: 8000, + maxFileSizeMb: 5000, + }, + duration: { hardMinMs: 3000, hardMaxMs: 600_000 }, + safeZones: FRAMED_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, + "x-twitter": { + id: "x-twitter", + name: "X", + sourceUrls: ["https://help.x.com/en/using-x/x-videos"], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { + aspect: "16:9", + width: 1280, + height: 720, + fps: 30, + codec: "h264", + maxBitrateKbps: 5000, + maxFileSizeMb: 512, + }, + duration: { hardMinMs: 1000, hardMaxMs: 140_000 }, + safeZones: FRAMED_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, + "pinterest-video-pin": { + id: "pinterest-video-pin", + name: "Pinterest", + sourceUrls: ["https://help.pinterest.com/en/business"], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { ...VERTICAL_1080, fps: 30, codec: "h264", maxBitrateKbps: 8000, maxFileSizeMb: 2048 }, + duration: { hardMinMs: 4000, hardMaxMs: 300_000 }, + safeZones: VERTICAL_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, + "snapchat-spotlight": { + id: "snapchat-spotlight", + name: "Snapchat Spotlight", + sourceUrls: ["https://help.snapchat.com/hc/en-us"], + reviewedAt: REVIEWED_AT, + unverified: true, + video: { ...VERTICAL_1080, fps: 30, codec: "h264", maxBitrateKbps: 5000, maxFileSizeMb: 250 }, + duration: { hardMinMs: 5000, hardMaxMs: 60_000 }, + safeZones: VERTICAL_SAFE_ZONES, + audio: STREAMING_LOUDNESS, + }, +}; diff --git a/packages/contracts/src/media-presets.ts b/packages/contracts/src/media-presets.ts new file mode 100644 index 0000000..9f0949c --- /dev/null +++ b/packages/contracts/src/media-presets.ts @@ -0,0 +1,28 @@ +import * as z from "zod"; + +/** + * Output formats for video cuts and exports. Width and height are the encoded frame; every + * preset encodes H.264 + AAC in MP4 so the result plays everywhere it is posted. + */ +export const MEDIA_PRESETS = { + vertical: { aspect: "9:16", width: 1080, height: 1920 }, + landscape: { aspect: "16:9", width: 1920, height: 1080 }, + square: { aspect: "1:1", width: 1080, height: 1080 }, + portrait: { aspect: "4:5", width: 1080, height: 1350 }, + classic: { aspect: "4:3", width: 1440, height: 1080 }, + cinema: { aspect: "21:9", width: 2560, height: 1080 }, +} as const; +export const MediaPresetIdSchema = z.enum([ + "vertical", + "landscape", + "square", + "portrait", + "classic", + "cinema", +]); +export type MediaPresetId = z.infer; +export type MediaPreset = (typeof MEDIA_PRESETS)[MediaPresetId]; + +/** How a frame of another aspect ratio fills a preset. */ +export const MediaFitSchema = z.enum(["crop", "pad", "blur"]); +export type MediaFit = z.infer; diff --git a/packages/contracts/src/media-video.test.ts b/packages/contracts/src/media-video.test.ts index eb3fc17..0499442 100644 --- a/packages/contracts/src/media-video.test.ts +++ b/packages/contracts/src/media-video.test.ts @@ -42,15 +42,42 @@ describe("edit_video input", () => { }); describe("export_video input", () => { - it("needs at least one preset and defaults to cropping", () => { + it("needs at least one preset or platform and defaults to cropping", () => { expect(ExportVideoInputSchema.parse({ artifact_id: "a", presets: ["square"] })).toEqual({ artifact_id: "a", presets: ["square"], fit: "crop", }); + expect(ExportVideoInputSchema.parse({ artifact_id: "a", platforms: ["tiktok"] })).toEqual({ + artifact_id: "a", + platforms: ["tiktok"], + fit: "crop", + }); expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: [] }).success).toBe(false); + expect(ExportVideoInputSchema.safeParse({ artifact_id: "a" }).success).toBe(false); expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: ["imax"] }).success).toBe( false, ); + expect( + ExportVideoInputSchema.safeParse({ artifact_id: "a", platforms: ["myspace"] }).success, + ).toBe(false); + }); + + it("takes at most six targets across platforms and presets", () => { + const platforms = ["tiktok", "youtube-shorts", "instagram-reels", "linkedin"]; + expect( + ExportVideoInputSchema.safeParse({ + artifact_id: "a", + platforms, + presets: ["square", "portrait"], + }).success, + ).toBe(true); + const tooMany = ExportVideoInputSchema.safeParse({ + artifact_id: "a", + platforms, + presets: ["square", "portrait", "classic"], + }); + expect(tooMany.success).toBe(false); + expect(tooMany.error?.issues[0]?.path).toEqual(["platforms"]); }); }); diff --git a/packages/contracts/src/media-video.ts b/packages/contracts/src/media-video.ts index a12bf51..29fb141 100644 --- a/packages/contracts/src/media-video.ts +++ b/packages/contracts/src/media-video.ts @@ -1,6 +1,7 @@ import * as z from "zod"; import { Id } from "./ids.js"; -import { MediaFitSchema, MediaPresetIdSchema } from "./media.js"; +import { MediaPlatformIdSchema } from "./media-platforms.js"; +import { MediaFitSchema, MediaPresetIdSchema } from "./media-presets.js"; /** * Inputs of the `edit_video` and `export_video` media operations, which are also the arguments @@ -71,10 +72,25 @@ export const EditVideoInputSchema = z }); export type EditVideoInput = z.infer; -export const ExportVideoInputSchema = z.object({ - artifact_id: Id, - presets: z.array(MediaPresetIdSchema).min(1).max(6), - fit: MediaFitSchema.default("crop"), - title: MediaTitleSchema.optional(), -}); +export const MAX_EXPORT_TARGETS = 6; + +/** One output per platform and per preset: at least one, at most MAX_EXPORT_TARGETS in all. */ +export const ExportVideoInputSchema = z + .object({ + artifact_id: Id, + platforms: z.array(MediaPlatformIdSchema).max(MAX_EXPORT_TARGETS).optional(), + presets: z.array(MediaPresetIdSchema).max(MAX_EXPORT_TARGETS).optional(), + fit: MediaFitSchema.default("crop"), + title: MediaTitleSchema.optional(), + }) + .refine( + (input) => { + const targets = (input.platforms?.length ?? 0) + (input.presets?.length ?? 0); + return targets >= 1 && targets <= MAX_EXPORT_TARGETS; + }, + { + path: ["platforms"], + message: `name 1 to ${MAX_EXPORT_TARGETS} platforms and presets in all`, + }, + ); export type ExportVideoInput = z.infer; diff --git a/packages/contracts/src/media.ts b/packages/contracts/src/media.ts index 3654e9e..ad27f51 100644 --- a/packages/contracts/src/media.ts +++ b/packages/contracts/src/media.ts @@ -1,5 +1,11 @@ import * as z from "zod"; import { Id } from "./ids.js"; +import { + MediaCheckSchema, + MediaPlatformIdSchema, + MediaSkippedTargetSchema, +} from "./media-platforms.js"; +import { MediaPresetIdSchema } from "./media-presets.js"; /** What a media job does. Generation runs on the configured image/video backend, edits run locally. */ export const MediaOperationSchema = z.enum([ @@ -32,33 +38,6 @@ export const MEDIA_JOB_ACTIVE_STATUSES = [ export const MediaOutputKindSchema = z.enum(["image", "video", "audio", "file"]); export type MediaOutputKind = z.infer; -/** - * Output formats for video cuts and exports. Width and height are the encoded frame; every - * preset encodes H.264 + AAC in MP4 so the result plays everywhere it is posted. - */ -export const MEDIA_PRESETS = { - vertical: { aspect: "9:16", width: 1080, height: 1920 }, - landscape: { aspect: "16:9", width: 1920, height: 1080 }, - square: { aspect: "1:1", width: 1080, height: 1080 }, - portrait: { aspect: "4:5", width: 1080, height: 1350 }, - classic: { aspect: "4:3", width: 1440, height: 1080 }, - cinema: { aspect: "21:9", width: 2560, height: 1080 }, -} as const; -export const MediaPresetIdSchema = z.enum([ - "vertical", - "landscape", - "square", - "portrait", - "classic", - "cinema", -]); -export type MediaPresetId = z.infer; -export type MediaPreset = (typeof MEDIA_PRESETS)[MediaPresetId]; - -/** How a frame of another aspect ratio fills a preset. */ -export const MediaFitSchema = z.enum(["crop", "pad", "blur"]); -export type MediaFit = z.infer; - export const MediaOutputSchema = z.object({ artifactId: Id, kind: MediaOutputKindSchema, @@ -72,6 +51,9 @@ export const MediaOutputSchema = z.object({ /** The artifact this output was made from, for version history. */ parentArtifactId: Id.optional(), preset: MediaPresetIdSchema.optional(), + /** The platform it was exported for, and how it measured against that platform's limits. */ + platform: MediaPlatformIdSchema.optional(), + check: MediaCheckSchema.optional(), }); export type MediaOutput = z.infer; @@ -86,6 +68,8 @@ export const MediaJobBlockSchema = z.object({ progress: z.number().min(0).max(1).optional(), error: z.string().optional(), outputs: z.array(MediaOutputSchema), + /** Platforms an export did not render, each with the reason. */ + skipped: z.array(MediaSkippedTargetSchema).optional(), }); export type MediaJobBlock = z.infer;