Merge claude/phase3-media into p3/b5-transcribe-studio
Brings in ffmpeg (probing, extractAudio) and the video operations. The transcribe registration now gets the ffmpeg audio tools, so video is transcribed through extractAudio with the demuxer forced from the MIME type, and long recordings are converted and uploaded in ranges. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MRtGG9ARfqqhma3t85Ro2u
This commit is contained in:
Claude
commit
a02cbb5374
@ -29,6 +29,7 @@ function setup() {
|
||||
},
|
||||
spaceMember: { findUnique: vi.fn(async () => ({ role: "owner" })) },
|
||||
deploymentSettings: { findUnique: vi.fn(async () => null) },
|
||||
integrationProviderConfig: { findUnique: vi.fn(async () => null) },
|
||||
aiDataConsent: { findMany: vi.fn(async () => []), upsert },
|
||||
$transaction: vi.fn(async (queries) => Promise.all(queries)),
|
||||
},
|
||||
@ -134,6 +135,28 @@ describe("consent grants", () => {
|
||||
expect(JSON.stringify(status.recipients)).not.toContain("supermemory");
|
||||
expect(deps.prisma.spaceMemoryConfig.findUnique).not.toHaveBeenCalled();
|
||||
});
|
||||
it("discloses the deployment's media servers before model use, keyed by server", async () => {
|
||||
const { deps, actor } = setup();
|
||||
const settings = {
|
||||
comfyui: { baseUrl: "http://127.0.0.1:8188", apiKey: "comfy-key", imageModel: "sdxl" },
|
||||
transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "whisper-1" },
|
||||
};
|
||||
vi.mocked(deps.prisma.integrationProviderConfig.findUnique).mockResolvedValue({
|
||||
id: "media",
|
||||
ciphertext: "fake-media-ciphertext",
|
||||
} as never);
|
||||
deps.secrets = {
|
||||
load: vi.fn(() => JSON.stringify(settings)),
|
||||
} as unknown as RouterDeps["secrets"];
|
||||
const status = await aiConsentStatus(deps, actor, { botId: "bot", uses: ["model"] });
|
||||
expect(status.recipients.map((recipient) => recipient.name)).toEqual([
|
||||
"ComfyUI (http://127.0.0.1:8188)",
|
||||
"Transcription (http://127.0.0.1:8080)",
|
||||
]);
|
||||
expect(status.recipients[0]?.detail).toBe("Model: sdxl.");
|
||||
expect(JSON.stringify(status)).not.toContain("comfy-key");
|
||||
expect((await aiConsentStatus(deps, actor, { uses: ["voice"] })).recipients).toHaveLength(1);
|
||||
});
|
||||
it("returns an operator policy without requiring a hosted provider", async () => {
|
||||
const { deps, actor } = setup();
|
||||
deps.env.privacyPolicyUrl = "https://example.com/privacy";
|
||||
|
||||
@ -3,6 +3,9 @@ import { ORPCError } from "@orpc/server";
|
||||
import {
|
||||
aiRecipient,
|
||||
cloudAgentsEnabled,
|
||||
loadMediaSettings,
|
||||
MEDIA_COMFYUI_PROVIDER_ID,
|
||||
MEDIA_TRANSCRIPTION_PROVIDER_ID,
|
||||
parseModelSecret,
|
||||
selectConfiguredModel,
|
||||
toStringRecord,
|
||||
@ -210,6 +213,30 @@ export async function aiConsentStatus(
|
||||
) {
|
||||
add(aiRecipient({ provider: deps.cloudAgent.provider.describe().id, use: "model" }));
|
||||
}
|
||||
if (uses.includes("model")) {
|
||||
// Media tools and card actions send prompts, images, video and audio to the owner's servers.
|
||||
const media = await loadMediaSettings(deps);
|
||||
if (media.comfyui) {
|
||||
add(
|
||||
aiRecipient({
|
||||
provider: MEDIA_COMFYUI_PROVIDER_ID,
|
||||
use: "model",
|
||||
baseUrl: media.comfyui.baseUrl,
|
||||
modelId: media.comfyui.imageModel,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (media.transcription) {
|
||||
add(
|
||||
aiRecipient({
|
||||
provider: MEDIA_TRANSCRIPTION_PROVIDER_ID,
|
||||
use: "model",
|
||||
baseUrl: media.transcription.baseUrl,
|
||||
modelId: media.transcription.model,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
return {
|
||||
scope: createHash("sha256")
|
||||
.update(JSON.stringify([actor.userId, actor.spaceId]))
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { ORPCError } from "@orpc/server";
|
||||
import type { MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit";
|
||||
import type { EncryptedSecretStore } from "@rakazo/adapters";
|
||||
import {
|
||||
type EncryptedSecretStore,
|
||||
loadMediaSettings,
|
||||
MediaJobRejectedError,
|
||||
MediaSettingsInvalidError,
|
||||
|
||||
278
apps/web/src/components/MediaSettings.tsx
Normal file
278
apps/web/src/components/MediaSettings.tsx
Normal file
@ -0,0 +1,278 @@
|
||||
import { Trans, useLingui } from "@lingui/react/macro";
|
||||
import type { MediaSettingsUpdate, MediaSettingsView } from "@rakazo/contracts";
|
||||
import { Button, Field, FieldLabel, Input, Textarea } from "@rakazo/ui-web";
|
||||
import { useEffect, useId, useState } from "react";
|
||||
import { rpc } from "../lib/rpc";
|
||||
import { SuccessPop } from "./ai/primitives";
|
||||
|
||||
type KeyDraft = { value: string; remove: boolean };
|
||||
|
||||
const emptyKey: KeyDraft = { value: "", remove: false };
|
||||
|
||||
/** Omitted keeps the saved key, empty removes it, anything else replaces it. */
|
||||
function keyUpdate(draft: KeyDraft): string | undefined {
|
||||
if (draft.value.trim()) return draft.value.trim();
|
||||
return draft.remove ? "" : undefined;
|
||||
}
|
||||
|
||||
function ApiKeyField({
|
||||
saved,
|
||||
draft,
|
||||
disabled,
|
||||
onChange,
|
||||
}: {
|
||||
saved: boolean;
|
||||
draft: KeyDraft;
|
||||
disabled: boolean;
|
||||
onChange: (draft: KeyDraft) => void;
|
||||
}) {
|
||||
const { t } = useLingui();
|
||||
const id = useId();
|
||||
const showSaved = saved && !draft.remove;
|
||||
return (
|
||||
<Field className="mt-4">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<FieldLabel htmlFor={id}>
|
||||
<Trans>API key</Trans>
|
||||
</FieldLabel>
|
||||
{showSaved && !draft.value ? (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
disabled={disabled}
|
||||
onClick={() => onChange({ value: "", remove: true })}
|
||||
>
|
||||
<Trans>Remove</Trans>
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
<Input
|
||||
id={id}
|
||||
type="password"
|
||||
autoComplete="new-password"
|
||||
value={draft.value}
|
||||
disabled={disabled}
|
||||
placeholder={showSaved ? t`Saved` : t`Optional`}
|
||||
onChange={(event) => onChange({ ...draft, value: event.target.value })}
|
||||
/>
|
||||
</Field>
|
||||
);
|
||||
}
|
||||
|
||||
/** The deployment owner's media servers: ComfyUI for generation, a transcription server. */
|
||||
export function MediaSettings() {
|
||||
const { t } = useLingui();
|
||||
const ids = useId();
|
||||
const [view, setView] = useState<MediaSettingsView | null>(null);
|
||||
const [comfyUrl, setComfyUrl] = useState("");
|
||||
const [comfyKey, setComfyKey] = useState<KeyDraft>(emptyKey);
|
||||
const [imageModel, setImageModel] = useState("");
|
||||
const [imageWorkflow, setImageWorkflow] = useState("");
|
||||
const [videoWorkflow, setVideoWorkflow] = useState("");
|
||||
const [transcriptionUrl, setTranscriptionUrl] = useState("");
|
||||
const [transcriptionModel, setTranscriptionModel] = useState("");
|
||||
const [transcriptionKey, setTranscriptionKey] = useState<KeyDraft>(emptyKey);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [saved, setSaved] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
function apply(next: MediaSettingsView) {
|
||||
setView(next);
|
||||
setComfyUrl(next.comfyui?.baseUrl ?? "");
|
||||
setImageModel(next.comfyui?.imageModel ?? "");
|
||||
setImageWorkflow(next.comfyui?.imageWorkflow ?? "");
|
||||
setVideoWorkflow(next.comfyui?.videoWorkflow ?? "");
|
||||
setTranscriptionUrl(next.transcription?.baseUrl ?? "");
|
||||
setTranscriptionModel(next.transcription?.model ?? "");
|
||||
setComfyKey(emptyKey);
|
||||
setTranscriptionKey(emptyKey);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
rpc.media.settings
|
||||
.get()
|
||||
.then((next) => {
|
||||
if (!cancelled) apply(next);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (!cancelled) setError(err instanceof Error ? err.message : t`Couldn't load`);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [t]);
|
||||
|
||||
async function save() {
|
||||
if (busy) return;
|
||||
setBusy(true);
|
||||
setSaved(false);
|
||||
setError(null);
|
||||
const update: MediaSettingsUpdate = {
|
||||
comfyui: comfyUrl.trim()
|
||||
? {
|
||||
baseUrl: comfyUrl.trim(),
|
||||
apiKey: keyUpdate(comfyKey),
|
||||
imageModel,
|
||||
imageWorkflow,
|
||||
videoWorkflow,
|
||||
}
|
||||
: null,
|
||||
transcription: transcriptionUrl.trim()
|
||||
? {
|
||||
baseUrl: transcriptionUrl.trim(),
|
||||
model: transcriptionModel,
|
||||
apiKey: keyUpdate(transcriptionKey),
|
||||
}
|
||||
: null,
|
||||
};
|
||||
try {
|
||||
apply(await rpc.media.settings.update(update));
|
||||
setSaved(true);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t`Couldn't save`);
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
const disabled = busy || view === null;
|
||||
const advancedSaved = Boolean(
|
||||
view?.comfyui?.imageModel || view?.comfyui?.imageWorkflow || view?.comfyui?.videoWorkflow,
|
||||
);
|
||||
const workflowClass = "max-h-48 overflow-auto font-mono text-xs";
|
||||
|
||||
return (
|
||||
<div data-testid="media-settings" className="space-y-5">
|
||||
<section className="rounded-xl border border-border px-4 py-4">
|
||||
<h3 className="text-ui-body font-medium text-foreground">ComfyUI</h3>
|
||||
<Field className="mt-3">
|
||||
<FieldLabel htmlFor={`${ids}-comfy-url`}>
|
||||
<Trans>Server URL</Trans>
|
||||
</FieldLabel>
|
||||
<Input
|
||||
id={`${ids}-comfy-url`}
|
||||
value={comfyUrl}
|
||||
disabled={disabled}
|
||||
placeholder="http://127.0.0.1:8188"
|
||||
autoComplete="off"
|
||||
onChange={(event) => setComfyUrl(event.target.value)}
|
||||
/>
|
||||
</Field>
|
||||
<ApiKeyField
|
||||
saved={Boolean(view?.comfyui?.hasApiKey)}
|
||||
draft={comfyKey}
|
||||
disabled={disabled}
|
||||
onChange={setComfyKey}
|
||||
/>
|
||||
<details
|
||||
key={advancedSaved ? "saved" : "empty"}
|
||||
open={advancedSaved}
|
||||
data-testid="media-settings-advanced"
|
||||
className="group mt-4"
|
||||
>
|
||||
<summary className="flex cursor-pointer list-none items-center justify-between gap-3 text-ui-body text-muted-foreground">
|
||||
<span>
|
||||
<Trans>Advanced</Trans>
|
||||
</span>
|
||||
<span aria-hidden="true" className="transition-transform group-open:rotate-90">
|
||||
›
|
||||
</span>
|
||||
</summary>
|
||||
<Field className="mt-3">
|
||||
<FieldLabel htmlFor={`${ids}-image-model`}>
|
||||
<Trans>Image model</Trans>
|
||||
</FieldLabel>
|
||||
<Input
|
||||
id={`${ids}-image-model`}
|
||||
value={imageModel}
|
||||
disabled={disabled}
|
||||
placeholder={t`Optional`}
|
||||
autoComplete="off"
|
||||
onChange={(event) => setImageModel(event.target.value)}
|
||||
/>
|
||||
</Field>
|
||||
<Field className="mt-4">
|
||||
<FieldLabel htmlFor={`${ids}-image-workflow`}>
|
||||
<Trans>Image workflow</Trans>
|
||||
</FieldLabel>
|
||||
<Textarea
|
||||
id={`${ids}-image-workflow`}
|
||||
className={workflowClass}
|
||||
value={imageWorkflow}
|
||||
disabled={disabled}
|
||||
placeholder={t`Optional`}
|
||||
spellCheck={false}
|
||||
onChange={(event) => setImageWorkflow(event.target.value)}
|
||||
/>
|
||||
</Field>
|
||||
<Field className="mt-4">
|
||||
<FieldLabel htmlFor={`${ids}-video-workflow`}>
|
||||
<Trans>Video workflow</Trans>
|
||||
</FieldLabel>
|
||||
<Textarea
|
||||
id={`${ids}-video-workflow`}
|
||||
className={workflowClass}
|
||||
value={videoWorkflow}
|
||||
disabled={disabled}
|
||||
placeholder={t`Optional`}
|
||||
spellCheck={false}
|
||||
onChange={(event) => setVideoWorkflow(event.target.value)}
|
||||
/>
|
||||
</Field>
|
||||
</details>
|
||||
</section>
|
||||
|
||||
<section className="rounded-xl border border-border px-4 py-4">
|
||||
<h3 className="text-ui-body font-medium text-foreground">
|
||||
<Trans>Transcription</Trans>
|
||||
</h3>
|
||||
<Field className="mt-3">
|
||||
<FieldLabel htmlFor={`${ids}-transcription-url`}>
|
||||
<Trans>Server URL</Trans>
|
||||
</FieldLabel>
|
||||
<Input
|
||||
id={`${ids}-transcription-url`}
|
||||
value={transcriptionUrl}
|
||||
disabled={disabled}
|
||||
placeholder="http://127.0.0.1:8080/v1"
|
||||
autoComplete="off"
|
||||
onChange={(event) => setTranscriptionUrl(event.target.value)}
|
||||
/>
|
||||
</Field>
|
||||
<Field className="mt-4">
|
||||
<FieldLabel htmlFor={`${ids}-transcription-model`}>
|
||||
<Trans>Model</Trans>
|
||||
</FieldLabel>
|
||||
<Input
|
||||
id={`${ids}-transcription-model`}
|
||||
value={transcriptionModel}
|
||||
disabled={disabled}
|
||||
placeholder={t`Optional`}
|
||||
autoComplete="off"
|
||||
onChange={(event) => setTranscriptionModel(event.target.value)}
|
||||
/>
|
||||
</Field>
|
||||
<ApiKeyField
|
||||
saved={Boolean(view?.transcription?.hasApiKey)}
|
||||
draft={transcriptionKey}
|
||||
disabled={disabled}
|
||||
onChange={setTranscriptionKey}
|
||||
/>
|
||||
</section>
|
||||
|
||||
{error ? (
|
||||
<p role="alert" className="text-ui-body-sm text-destructive">
|
||||
{error}
|
||||
</p>
|
||||
) : null}
|
||||
<div className="flex items-center gap-3">
|
||||
<Button className="rounded-full" disabled={disabled} onClick={() => void save()}>
|
||||
{busy ? <Trans>Saving…</Trans> : <Trans>Save</Trans>}
|
||||
</Button>
|
||||
{saved ? <SuccessPop label={t`Saved`} /> : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -3,6 +3,7 @@ import type { AvatarStyle, SpaceMemoryConfig } from "@rakazo/contracts";
|
||||
import { Button, Dialog, DialogClose, DialogContent, DialogTitle } from "@rakazo/ui-web";
|
||||
import {
|
||||
Brain,
|
||||
Clapperboard,
|
||||
CloudDownload,
|
||||
Cpu,
|
||||
Gauge,
|
||||
@ -16,6 +17,7 @@ import {
|
||||
} from "lucide-react";
|
||||
import { type ComponentType, useEffect, useRef, useState } from "react";
|
||||
import { computersAreUnavailable } from "../components/ComputersUnavailableHint";
|
||||
import { MediaSettings } from "../components/MediaSettings";
|
||||
import { UserProfileSettings } from "../components/UserProfileSettings";
|
||||
import type { TradingBotUiState } from "../lib/trading-bot-status";
|
||||
import {
|
||||
@ -37,6 +39,7 @@ export type SettingsSection =
|
||||
| "memory"
|
||||
| "profile"
|
||||
| "voice"
|
||||
| "media"
|
||||
| "usage"
|
||||
| "computer"
|
||||
| "updates"
|
||||
@ -111,6 +114,7 @@ export function SettingsOverlay({
|
||||
{ id: "memory", label: t`Memory`, icon: Brain },
|
||||
{ id: "profile", label: t`About you`, icon: UserRound },
|
||||
{ id: "voice", label: t`Voice`, icon: Volume2 },
|
||||
...(isDeploymentOwner ? [{ id: "media" as const, label: t`Media`, icon: Clapperboard }] : []),
|
||||
{ id: "usage", label: t`Usage`, icon: Gauge },
|
||||
...(showComputer ? [{ id: "computer" as const, label: t`Computer`, icon: Monitor }] : []),
|
||||
{ id: "updates", label: t`Updates`, icon: CloudDownload },
|
||||
@ -250,6 +254,7 @@ export function SettingsOverlay({
|
||||
/>
|
||||
) : null}
|
||||
{section === "profile" ? <UserProfileSettings /> : null}
|
||||
{section === "media" && isDeploymentOwner ? <MediaSettings /> : null}
|
||||
{section === "usage" ? (
|
||||
<UsageSettingsPanel usage={usage} panelRef={usageRef} />
|
||||
) : null}
|
||||
|
||||
@ -1,4 +1,8 @@
|
||||
FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a
|
||||
# ffmpeg/ffprobe cut, reframe and export videos; DejaVu is the burned-in subtitle font.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ffmpeg fonts-dejavu-core \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN corepack enable
|
||||
WORKDIR /app
|
||||
# Normalize modes: a restrictive host umask (e.g. 077 on hardened hosts) would otherwise
|
||||
|
||||
@ -41,6 +41,18 @@ describe("mobile AI recipient disclosure", () => {
|
||||
expect(recipient.name).toBe("Embeddings (http://192.168.1.20:8080)");
|
||||
expect(recipient.privacyUrl).toBeUndefined();
|
||||
});
|
||||
it("names the media servers that media jobs send files to", () => {
|
||||
expect(
|
||||
aiRecipient({ provider: "comfyui", use: "model", baseUrl: "http://127.0.0.1:8188" })?.name,
|
||||
).toBe("ComfyUI (http://127.0.0.1:8188)");
|
||||
expect(
|
||||
aiRecipient({
|
||||
provider: "media-transcription",
|
||||
use: "model",
|
||||
baseUrl: "http://127.0.0.1:8080/v1",
|
||||
})?.name,
|
||||
).toBe("Transcription (http://127.0.0.1:8080)");
|
||||
});
|
||||
it("supports local models without a hosted dependency", () => {
|
||||
const fetch = vi.fn();
|
||||
vi.stubGlobal("fetch", fetch);
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { AiDataUse, AiRecipient } from "@rakazo/contracts";
|
||||
import { BUILTIN_MEMORY_PROVIDER_ID } from "./builtin-memory-provider.js";
|
||||
import { MEDIA_COMFYUI_PROVIDER_ID, MEDIA_TRANSCRIPTION_PROVIDER_ID } from "./media-settings.js";
|
||||
import { localBaseUrl } from "./pi-local-provider.js";
|
||||
import { listPiCatalog } from "./pi-models.js";
|
||||
import { voiceCatalogEntry } from "./voice-factory.js";
|
||||
@ -59,5 +60,7 @@ function recipientName(provider: string, use: AiDataUse) {
|
||||
return provider;
|
||||
}
|
||||
if (provider === "cursor") return "Cursor";
|
||||
if (provider === MEDIA_COMFYUI_PROVIDER_ID) return "ComfyUI";
|
||||
if (provider === MEDIA_TRANSCRIPTION_PROVIDER_ID) return "Transcription";
|
||||
return listPiCatalog().find((entry) => entry.provider === provider)?.providerName ?? provider;
|
||||
}
|
||||
|
||||
@ -3,6 +3,8 @@ import {
|
||||
BotSecretDestination,
|
||||
BotSecretName,
|
||||
EditImageInputSchema,
|
||||
EditVideoInputSchema,
|
||||
ExportVideoInputSchema,
|
||||
GenerateImageInputSchema,
|
||||
GenerateVideoInputSchema,
|
||||
MODEL_TIERS,
|
||||
@ -26,6 +28,9 @@ function mediaParameters(schema: z.ZodType) {
|
||||
export const SUBAGENT_MODEL_TIER_DESCRIPTION =
|
||||
"Optional model tier: fast for quick simple work, code for programming, standard otherwise. Uses the model this space maps to the tier, or your own model when it maps none. Ignored when model_provider and model_id are set.";
|
||||
|
||||
const VIDEO_PRESETS_AND_FITS =
|
||||
"Presets: vertical 9:16, landscape 16:9, square 1:1, portrait 4:5, classic 4:3, cinema 21:9. fit: crop (default) fills the frame and cuts the edges, pad adds bars, blur fills the bars with a blurred copy.";
|
||||
|
||||
export const DELEGATION_TOOL_NAMES = new Set([
|
||||
"run_subagent",
|
||||
"spawn_bot",
|
||||
@ -358,6 +363,16 @@ export const builtinAgentTools: ConnectorTool[] = [
|
||||
"Edit an image artifact into a new one: crop, rotate, flip, resize (crop, pad, or blur to fit a format), add borders, convert the format, or repaint it from a prompt (only when generate_image is available). Edits apply in that order.",
|
||||
inputSchema: mediaParameters(EditImageInputSchema),
|
||||
},
|
||||
{
|
||||
name: "edit_video",
|
||||
description: `Edit a video artifact into a new one. steps run in order, times in seconds on the result of the steps before: trim {start, end}; keep {segments: [{start, end}]} keeps those parts and cuts the rest; concat {artifact_ids} appends videos fitted to the current frame; reframe {preset, fit}; speed {factor 0.25–4}; volume {gain 0–8, 1 unchanged}; mute; subtitles {artifact_id} burns in an SRT or VTT file; extract_audio (last step) makes the result M4A audio; poster {at} picks the cover frame, or as the only step saves that frame as an image. ${VIDEO_PRESETS_AND_FITS} Videos come out as H.264/AAC MP4. title: a few words in the user's language for the card and file name.`,
|
||||
inputSchema: mediaParameters(EditVideoInputSchema),
|
||||
},
|
||||
{
|
||||
name: "export_video",
|
||||
description: `Export a video artifact as one H.264/AAC MP4 per preset. ${VIDEO_PRESETS_AND_FITS} For cuts, speed, audio or subtitles use edit_video.`,
|
||||
inputSchema: mediaParameters(ExportVideoInputSchema),
|
||||
},
|
||||
{
|
||||
name: "transcribe",
|
||||
description:
|
||||
|
||||
@ -78,6 +78,16 @@ export * from "./mcp-emulator.js";
|
||||
export * from "./mcp-oauth.js";
|
||||
export * from "./mcp-server-tool.js";
|
||||
export * from "./mcp-transport.js";
|
||||
export type { MediaProbe, RunFfmpegOptions } from "./media/ffmpeg.js";
|
||||
export {
|
||||
detectFfmpeg,
|
||||
extractAudio,
|
||||
FfmpegError,
|
||||
probeMedia,
|
||||
renderPoster,
|
||||
runFfmpeg,
|
||||
} from "./media/ffmpeg.js";
|
||||
export { createVideoOperations } from "./media/video-operations.js";
|
||||
export * from "./media-operations.js";
|
||||
export * from "./media-runtime.js";
|
||||
export * from "./media-settings.js";
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import type { MediaSettings } from "@rakazo/contracts";
|
||||
import { createImageOperations } from "./media/image-operations.js";
|
||||
import { createTranscriptionOperations } from "./media/transcription.js";
|
||||
import { createTranscriptionOperations, ffmpegTranscriptionAudio } from "./media/transcription.js";
|
||||
import { createVideoOperations } from "./media/video-operations.js";
|
||||
import type { MediaOperationRegistration } from "./media-runtime.js";
|
||||
|
||||
export interface MediaOperationDeps {
|
||||
@ -15,6 +16,10 @@ export interface MediaOperationDeps {
|
||||
export function createMediaOperations(deps: MediaOperationDeps): MediaOperationRegistration[] {
|
||||
return [
|
||||
...createImageOperations(deps),
|
||||
...createTranscriptionOperations({ loadSettings: deps.loadSettings }),
|
||||
...createVideoOperations(),
|
||||
...createTranscriptionOperations({
|
||||
loadSettings: deps.loadSettings,
|
||||
audio: ffmpegTranscriptionAudio,
|
||||
}),
|
||||
];
|
||||
}
|
||||
|
||||
@ -11,18 +11,14 @@ import type {
|
||||
} from "@rakazo/adapter-kit";
|
||||
import { MediaJobDeferredError, mediaRunJobKey } from "@rakazo/adapter-kit";
|
||||
import type { MediaJobBlock, MessageBlock } from "@rakazo/contracts";
|
||||
import { clearThread, createDb, type PrismaClient } from "@rakazo/db";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { clearThread, createDb } from "@rakazo/db";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { storeArtifactFromFile } from "./artifact-files.js";
|
||||
import { LocalArtifactStore } from "./artifacts.js";
|
||||
import {
|
||||
createMediaRuntime,
|
||||
MEDIA_WAKE_TRIGGER,
|
||||
MediaJobRejectedError,
|
||||
type MediaOperationRegistration,
|
||||
type MediaRuntimeOptions,
|
||||
} from "./media-runtime.js";
|
||||
import type { MediaOperationRegistration, MediaRuntimeOptions } from "./media-runtime.js";
|
||||
import { createMediaRuntime, MEDIA_WAKE_TRIGGER, MediaJobRejectedError } from "./media-runtime.js";
|
||||
|
||||
const describePostgres =
|
||||
process.env.VERIFY_DATABASE && process.env.DATABASE_URL ? describe.sequential : describe.skip;
|
||||
@ -424,6 +420,38 @@ describePostgres("media job runtime (PostgreSQL, fake operations)", () => {
|
||||
expect((await ctx.job(jobId)).status).toBe("done");
|
||||
});
|
||||
|
||||
it("runs on a card smaller than the floor once it is nearly free, and fails on far too small", async () => {
|
||||
let free = 5_000;
|
||||
const ctx = await setup({
|
||||
gpu: {
|
||||
minVramMb: 12_000,
|
||||
status: async () => ({ vramFreeMb: free, vramTotalMb: 8_192 }),
|
||||
},
|
||||
});
|
||||
const { jobId } = await ctx.start({ prompt: "fox" });
|
||||
await ctx.runtime.runJob(jobId);
|
||||
expect(await ctx.job(jobId)).toMatchObject({ status: "waiting", deferrals: 1 });
|
||||
free = 7_600;
|
||||
await prisma.mediaJob.update({ where: { id: jobId }, data: { nextAttemptAt: new Date() } });
|
||||
await ctx.runtime.runJob(jobId);
|
||||
expect((await ctx.job(jobId)).status).toBe("done");
|
||||
|
||||
const tiny = await setup({
|
||||
gpu: {
|
||||
minVramMb: 12_000,
|
||||
status: async () => ({ vramFreeMb: 4_000, vramTotalMb: 4_096 }),
|
||||
},
|
||||
});
|
||||
const small = await tiny.start({ prompt: "fox" });
|
||||
await tiny.runtime.runJob(small.jobId);
|
||||
expect(tiny.handlerCalls).toHaveLength(0);
|
||||
expect(await tiny.job(small.jobId)).toMatchObject({
|
||||
status: "failed",
|
||||
error: "The GPU does not have enough memory for this job.",
|
||||
deferrals: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels a queued job at once and a running job through the lease heartbeat", async () => {
|
||||
const ctx = await setup({
|
||||
run: async (context) => {
|
||||
|
||||
@ -27,24 +27,24 @@ import {
|
||||
mediaRunJobKey,
|
||||
runContinueJob,
|
||||
} from "@rakazo/adapter-kit";
|
||||
import type {
|
||||
MediaJobBlock,
|
||||
MediaJobStatus,
|
||||
MediaOperation,
|
||||
MediaOutput,
|
||||
MessageBlock,
|
||||
} from "@rakazo/contracts";
|
||||
import {
|
||||
attachmentKindOf,
|
||||
MEDIA_JOB_ACTIVE_STATUSES,
|
||||
type MediaJobBlock,
|
||||
type MediaJobStatus,
|
||||
type MediaOperation,
|
||||
MediaOperationSchema,
|
||||
type MediaOutput,
|
||||
type MessageBlock,
|
||||
} from "@rakazo/contracts";
|
||||
import { attachmentExtensionForMimeType, inferAttachmentMimeType } from "@rakazo/core";
|
||||
import type { MediaJob, PrismaClient, ThreadEvents } from "@rakazo/db";
|
||||
import {
|
||||
appendEventInTransaction,
|
||||
createThreadMessageInTransaction,
|
||||
type MediaJob,
|
||||
Prisma,
|
||||
type PrismaClient,
|
||||
type ThreadEvents,
|
||||
withTransactionRetry,
|
||||
} from "@rakazo/db";
|
||||
import { getLogger } from "@rakazo/logging";
|
||||
@ -143,6 +143,7 @@ const TIMEOUT_ERROR = "The job took too long.";
|
||||
const FAILED_ERROR = "The job failed.";
|
||||
const MISSING_SOURCE_ERROR = "A source file is no longer available.";
|
||||
const TOO_LARGE_ERROR = "The result is too large to keep.";
|
||||
const GPU_TOO_SMALL_ERROR = "The GPU does not have enough memory for this job.";
|
||||
|
||||
type AbortReason = "cancel" | "lease" | "abandoned" | "timeout";
|
||||
|
||||
@ -191,6 +192,17 @@ function nonNegativeInt(value: number | undefined): number | undefined {
|
||||
return rounded >= 0 ? rounded : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Free VRAM a job waits for: the operation's floor, capped below the card's total so a smaller
|
||||
* card that can still run it (with offloading) does not wait forever. Null when the card is far
|
||||
* too small to try.
|
||||
*/
|
||||
export function requiredVramMb(minVramMb: number, gpu: GpuStatus): number | null {
|
||||
if (gpu.vramTotalMb > 0 && gpu.vramTotalMb * 2 < minVramMb) return null;
|
||||
if (gpu.vramTotalMb <= 0) return minVramMb;
|
||||
return Math.min(minVramMb, Math.floor(gpu.vramTotalMb * 0.9));
|
||||
}
|
||||
|
||||
function insideDir(dir: string, file: string): string {
|
||||
const resolved = path.resolve(dir, file);
|
||||
if (!resolved.startsWith(`${path.resolve(dir)}${path.sep}`)) {
|
||||
@ -754,7 +766,9 @@ export function createMediaRuntime(
|
||||
const gpu = await registration.gpu
|
||||
.status(AbortSignal.any([signal, AbortSignal.timeout(GPU_STATUS_TIMEOUT_MS)]))
|
||||
.catch(() => undefined);
|
||||
if (gpu && gpu.vramFreeMb < registration.gpu.minVramMb) {
|
||||
const needed = gpu ? requiredVramMb(registration.gpu.minVramMb, gpu) : undefined;
|
||||
if (needed === null) throw new MediaJobRejectedError(GPU_TOO_SMALL_ERROR);
|
||||
if (gpu && needed !== undefined && gpu.vramFreeMb < needed) {
|
||||
throw new MediaJobDeferredError("Waiting for free GPU memory", GPU_RETRY_MS);
|
||||
}
|
||||
}
|
||||
|
||||
136
packages/adapters/src/media-settings.test.ts
Normal file
136
packages/adapters/src/media-settings.test.ts
Normal file
@ -0,0 +1,136 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
loadMediaSettings,
|
||||
MEDIA_SETTINGS_ID,
|
||||
MediaSettingsInvalidError,
|
||||
mediaSettingsView,
|
||||
normalizeComfyUiBaseUrl,
|
||||
normalizeTranscriptionBaseUrl,
|
||||
saveMediaSettings,
|
||||
} from "./media-settings.js";
|
||||
import { EncryptedSecretStore } from "./secrets.js";
|
||||
|
||||
function store() {
|
||||
const rows = new Map<string, { id: string; ciphertext: string }>();
|
||||
const prisma = {
|
||||
integrationProviderConfig: {
|
||||
findUnique: vi.fn(async ({ where }: { where: { id: string } }) => rows.get(where.id) ?? null),
|
||||
upsert: vi.fn(
|
||||
async ({ where, create }: { where: { id: string }; create: { ciphertext: string } }) => {
|
||||
const row = { id: where.id, ciphertext: create.ciphertext };
|
||||
rows.set(where.id, row);
|
||||
return row;
|
||||
},
|
||||
),
|
||||
deleteMany: vi.fn(async ({ where }: { where: { id: string } }) => {
|
||||
rows.delete(where.id);
|
||||
return { count: 1 };
|
||||
}),
|
||||
},
|
||||
};
|
||||
const secrets = new EncryptedSecretStore("test-encryption-key-for-media-settings");
|
||||
return { deps: { prisma: prisma as never, secrets }, rows, prisma, secrets };
|
||||
}
|
||||
|
||||
describe("media settings", () => {
|
||||
it("normalizes server URLs and refuses ones that are not allowed", () => {
|
||||
expect(normalizeComfyUiBaseUrl(" http://127.0.0.1:8188/ ")).toBe("http://127.0.0.1:8188");
|
||||
expect(normalizeComfyUiBaseUrl("http://192.168.1.20:8188/comfy/?x=1#y")).toBe(
|
||||
"http://192.168.1.20:8188/comfy",
|
||||
);
|
||||
expect(normalizeTranscriptionBaseUrl("http://127.0.0.1:8080")).toBe("http://127.0.0.1:8080/v1");
|
||||
expect(normalizeTranscriptionBaseUrl("http://127.0.0.1:8080/v1/")).toBe(
|
||||
"http://127.0.0.1:8080/v1",
|
||||
);
|
||||
for (const url of ["ftp://127.0.0.1", "http://169.254.169.254", "not a url"]) {
|
||||
expect(() => normalizeComfyUiBaseUrl(url), url).toThrow();
|
||||
}
|
||||
expect(() => normalizeComfyUiBaseUrl("http://user:pass@127.0.0.1:8188")).toThrow("credentials");
|
||||
});
|
||||
|
||||
it("stores the settings encrypted and returns a view without keys", async () => {
|
||||
const { deps, rows } = store();
|
||||
const view = await saveMediaSettings(deps, {
|
||||
comfyui: { baseUrl: "http://127.0.0.1:8188", apiKey: "comfy-key" },
|
||||
transcription: null,
|
||||
});
|
||||
expect(view).toEqual({
|
||||
comfyui: { baseUrl: "http://127.0.0.1:8188", hasApiKey: true },
|
||||
transcription: null,
|
||||
});
|
||||
expect(rows.get(MEDIA_SETTINGS_ID)?.ciphertext).not.toContain("comfy-key");
|
||||
expect(await loadMediaSettings(deps)).toEqual({
|
||||
comfyui: { baseUrl: "http://127.0.0.1:8188", apiKey: "comfy-key" },
|
||||
transcription: null,
|
||||
});
|
||||
expect(
|
||||
mediaSettingsView({
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "http://127.0.0.1:8080/v1", apiKey: "k", model: "m" },
|
||||
}),
|
||||
).toEqual({
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "m", hasApiKey: true },
|
||||
});
|
||||
});
|
||||
|
||||
it("decrypts once per saved version", async () => {
|
||||
const { deps, secrets } = store();
|
||||
await saveMediaSettings(deps, {
|
||||
comfyui: { baseUrl: "http://127.0.0.1:8188" },
|
||||
transcription: null,
|
||||
});
|
||||
const load = vi.spyOn(secrets, "load");
|
||||
await loadMediaSettings(deps);
|
||||
await loadMediaSettings(deps);
|
||||
expect(load).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a saved key only for the same server, removes it when cleared", async () => {
|
||||
const { deps } = store();
|
||||
await saveMediaSettings(deps, {
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "http://127.0.0.1:8080", apiKey: "t-key" },
|
||||
});
|
||||
await saveMediaSettings(deps, {
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "http://127.0.0.1:8080/v1", model: "whisper-1" },
|
||||
});
|
||||
expect((await loadMediaSettings(deps)).transcription?.apiKey).toBe("t-key");
|
||||
await saveMediaSettings(deps, {
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "http://127.0.0.1:9090" },
|
||||
});
|
||||
expect((await loadMediaSettings(deps)).transcription?.apiKey).toBeUndefined();
|
||||
await saveMediaSettings(deps, {
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "http://127.0.0.1:9090", apiKey: "new" },
|
||||
});
|
||||
await saveMediaSettings(deps, {
|
||||
comfyui: null,
|
||||
transcription: { baseUrl: "http://127.0.0.1:9090", apiKey: "" },
|
||||
});
|
||||
expect((await loadMediaSettings(deps)).transcription?.apiKey).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses invalid settings with a message for the owner and removes empty ones", async () => {
|
||||
const { deps, rows } = store();
|
||||
await expect(
|
||||
saveMediaSettings(deps, {
|
||||
comfyui: { baseUrl: "http://169.254.169.254" },
|
||||
transcription: null,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(MediaSettingsInvalidError);
|
||||
expect(rows.size).toBe(0);
|
||||
await saveMediaSettings(deps, {
|
||||
comfyui: { baseUrl: "http://127.0.0.1:8188" },
|
||||
transcription: null,
|
||||
});
|
||||
expect(await saveMediaSettings(deps, { comfyui: null, transcription: null })).toEqual({
|
||||
comfyui: null,
|
||||
transcription: null,
|
||||
});
|
||||
expect(rows.size).toBe(0);
|
||||
expect(await loadMediaSettings(deps)).toEqual({ comfyui: null, transcription: null });
|
||||
});
|
||||
});
|
||||
@ -9,6 +9,10 @@ import {
|
||||
} from "./openai-compatible-url.js";
|
||||
import type { EncryptedSecretStore } from "./secrets.js";
|
||||
|
||||
/** Recipient ids for AI data-sharing disclosures of the media servers. */
|
||||
export const MEDIA_COMFYUI_PROVIDER_ID = "comfyui";
|
||||
export const MEDIA_TRANSCRIPTION_PROVIDER_ID = "media-transcription";
|
||||
|
||||
/** The IntegrationProviderConfig row that stores the media backends, encrypted as one record. */
|
||||
export const MEDIA_SETTINGS_ID = "media";
|
||||
const RECORD_ID = `integration-provider:${MEDIA_SETTINGS_ID}`;
|
||||
|
||||
@ -70,6 +70,36 @@ describe("media tools", () => {
|
||||
expect(selectBuiltinToolsForRun(base).some((tool) => isMediaToolName(tool.name))).toBe(false);
|
||||
});
|
||||
|
||||
it("exposes a media tool only for operations the space can run, within the bot allowlist", () => {
|
||||
const base = {
|
||||
graphicalToolsAllowed: true,
|
||||
groupId: null,
|
||||
trigger: "user",
|
||||
semanticMemoryEnabled: false,
|
||||
messagingChannelRun: false,
|
||||
bot: { kind: "specialist", restrictBuiltinTools: false, allowedBuiltinTools: [] as string[] },
|
||||
};
|
||||
const mediaNames = (options: Partial<Parameters<typeof selectBuiltinToolsForRun>[0]>) =>
|
||||
selectBuiltinToolsForRun({ ...base, ...options })
|
||||
.map((tool) => tool.name)
|
||||
.filter((name) => isMediaToolName(name));
|
||||
expect(mediaNames({})).toEqual([]);
|
||||
expect(mediaNames({ mediaCapabilities: none })).toEqual([]);
|
||||
expect(mediaNames({ mediaCapabilities: { ...none, generate_image: true } })).toEqual([
|
||||
"generate_image",
|
||||
]);
|
||||
expect(
|
||||
mediaNames({
|
||||
mediaCapabilities: { ...none, generate_image: true, edit_image: true },
|
||||
bot: {
|
||||
kind: "specialist",
|
||||
restrictBuiltinTools: true,
|
||||
allowedBuiltinTools: ["edit_image"],
|
||||
},
|
||||
}),
|
||||
).toEqual(["edit_image"]);
|
||||
});
|
||||
|
||||
it("adds continue_after to every media tool schema", () => {
|
||||
expect(
|
||||
mediaToolInputSchema({
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import type { ConnectorTool, MediaCapabilities, MediaRuntime } from "@rakazo/adapter-kit";
|
||||
import { type MediaOperation, MediaOperationSchema } from "@rakazo/contracts";
|
||||
import type { MediaOperation } from "@rakazo/contracts";
|
||||
import { MediaOperationSchema } from "@rakazo/contracts";
|
||||
import { MediaJobRejectedError } from "./media-runtime.js";
|
||||
|
||||
/** Media tools are named exactly like their operations and dispatched by one module. */
|
||||
|
||||
@ -8,7 +8,10 @@ const comfyui = (workflows: { imageWorkflow?: string; videoWorkflow?: string })
|
||||
|
||||
describe("ComfyUI settings", () => {
|
||||
it("refuses custom workflows the provider could not fill", async () => {
|
||||
const deps = {} as Parameters<typeof saveMediaSettings>[0];
|
||||
// Nothing saved yet; validation fails before anything is stored.
|
||||
const deps = {
|
||||
prisma: { integrationProviderConfig: { findUnique: async () => null } },
|
||||
} as unknown as Parameters<typeof saveMediaSettings>[0];
|
||||
await expect(
|
||||
saveMediaSettings(
|
||||
deps,
|
||||
|
||||
255
packages/adapters/src/media/ffmpeg.test.ts
Normal file
255
packages/adapters/src/media/ffmpeg.test.ts
Normal file
@ -0,0 +1,255 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { FfmpegProgress } from "./ffmpeg.js";
|
||||
import {
|
||||
createFfmpegProgressParser,
|
||||
detectFfmpeg,
|
||||
extractAudio,
|
||||
FfmpegError,
|
||||
ffmpegInputArgs,
|
||||
ffmpegSeconds,
|
||||
packetDurationMs,
|
||||
parseProbeOutput,
|
||||
probeMedia,
|
||||
renderPoster,
|
||||
runFfmpeg,
|
||||
} from "./ffmpeg.js";
|
||||
import type { TestClips } from "./media-test-support.js";
|
||||
import { createTestClips } from "./media-test-support.js";
|
||||
|
||||
const ffmpeg = await detectFfmpeg();
|
||||
|
||||
describe("ffmpeg progress", () => {
|
||||
it("reports each block, tolerates split chunks and N/A times, and marks the end", () => {
|
||||
const updates: FfmpegProgress[] = [];
|
||||
const feed = createFfmpegProgressParser((update) => updates.push(update));
|
||||
feed("frame=0\nout_time_us=N/A\ntotal_size=48\nprogress=cont");
|
||||
feed("inue\nframe=75\nout_time_us=2995");
|
||||
feed("374\nout_time_ms=2995374\ntotal_size=42300\nprogress=end\n");
|
||||
expect(updates).toEqual([
|
||||
{ totalSize: 48, done: false },
|
||||
{ outTimeUs: 2_995_374, totalSize: 42_300, done: true },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("probe parsing", () => {
|
||||
const stream = (fields: Record<string, unknown>) => ({
|
||||
codec_type: "video",
|
||||
codec_name: "h264",
|
||||
...fields,
|
||||
});
|
||||
|
||||
it("swaps the size of rotated video and reads frame rate and audio", () => {
|
||||
const probe = parseProbeOutput(
|
||||
JSON.stringify({
|
||||
format: { duration: "12.345", format_name: "mov,mp4,m4a,3gp,3g2,mj2" },
|
||||
streams: [
|
||||
stream({
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
avg_frame_rate: "30000/1001",
|
||||
sample_aspect_ratio: "1:1",
|
||||
side_data_list: [{ side_data_type: "Display Matrix", rotation: -90 }],
|
||||
}),
|
||||
{ codec_type: "audio", codec_name: "aac", sample_rate: "48000", channels: 2 },
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(probe).toEqual({
|
||||
durationMs: 12_345,
|
||||
video: {
|
||||
codec: "h264",
|
||||
width: 1080,
|
||||
height: 1920,
|
||||
rotation: 270,
|
||||
fps: 30000 / 1001,
|
||||
sampleAspectRatio: 1,
|
||||
},
|
||||
audio: { codec: "aac", sampleRate: 48_000, channels: 2 },
|
||||
hasAudio: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("reads legacy rotate tags, skips cover art and falls back to stream durations", () => {
|
||||
const probe = parseProbeOutput(
|
||||
JSON.stringify({
|
||||
format: { format_name: "mov" },
|
||||
streams: [
|
||||
stream({ width: 600, height: 600, disposition: { attached_pic: 1 } }),
|
||||
stream({
|
||||
width: 640,
|
||||
height: 480,
|
||||
avg_frame_rate: "0/0",
|
||||
r_frame_rate: "90000/1",
|
||||
sample_aspect_ratio: "4:3",
|
||||
tags: { rotate: "90" },
|
||||
duration: "4.5",
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(probe.durationMs).toBe(4500);
|
||||
expect(probe.hasAudio).toBe(false);
|
||||
expect(probe.video).toMatchObject({
|
||||
width: 480,
|
||||
height: 640,
|
||||
rotation: 90,
|
||||
fps: 0,
|
||||
sampleAspectRatio: 3 / 4,
|
||||
});
|
||||
});
|
||||
|
||||
it("reads the end of the last packet when the container has no duration", () => {
|
||||
expect(packetDurationMs("0.000000,0.040000\n2.960000,0.040000\nN/A,N/A\n")).toBe(3000);
|
||||
expect(packetDurationMs("")).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ffmpeg arguments", () => {
|
||||
it("opens inputs through the file protocol with the demuxer of their MIME type", () => {
|
||||
expect(
|
||||
ffmpegInputArgs("/a/b.webm", {
|
||||
mimeType: "video/webm",
|
||||
startSeconds: 2,
|
||||
durationSeconds: 1.5,
|
||||
}),
|
||||
).toEqual([
|
||||
...["-ss", "2", "-t", "1.5", "-protocol_whitelist", "file", "-f", "matroska"],
|
||||
...["-i", "file:/a/b.webm"],
|
||||
]);
|
||||
expect(ffmpegInputArgs("/a/b", { mimeType: "application/octet-stream" })).toEqual([
|
||||
...["-protocol_whitelist", "file", "-i", "file:/a/b"],
|
||||
]);
|
||||
expect(ffmpegSeconds(1e-7)).toBe("0");
|
||||
expect(ffmpegSeconds(12.34567)).toBe("12.346");
|
||||
});
|
||||
});
|
||||
|
||||
// CI runners may not have ffmpeg; these run wherever it is installed.
|
||||
describe.skipIf(!ffmpeg)("ffmpeg integration", () => {
|
||||
let clips: TestClips;
|
||||
|
||||
beforeAll(async () => {
|
||||
clips = await createTestClips();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (clips) await rm(clips.dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("probes duration, size, frame rate and audio", async () => {
|
||||
const probe = await probeMedia(clips.landscape, { mimeType: "video/mp4" });
|
||||
expect(probe.durationMs).toBeGreaterThanOrEqual(2900);
|
||||
expect(probe.durationMs).toBeLessThanOrEqual(3100);
|
||||
expect(probe.video).toMatchObject({ codec: "h264", width: 320, height: 180, fps: 25 });
|
||||
expect(probe.hasAudio).toBe(true);
|
||||
const silent = await probeMedia(clips.silentPortrait, { mimeType: "video/mp4" });
|
||||
expect(silent.hasAudio).toBe(false);
|
||||
expect(silent.video).toMatchObject({ width: 180, height: 320 });
|
||||
});
|
||||
|
||||
it("refuses a playlist disguised as an MP4 instead of following its URLs", async () => {
|
||||
const playlist = join(clips.dir, "playlist.mp4");
|
||||
await writeFile(
|
||||
playlist,
|
||||
`#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:1.0,\nfile:${clips.srt}\n#EXT-X-ENDLIST\n`,
|
||||
);
|
||||
await expect(probeMedia(playlist, { mimeType: "video/mp4" })).rejects.toThrow(FfmpegError);
|
||||
});
|
||||
|
||||
it("reports progress up to 1 for a run", async () => {
|
||||
const output = join(clips.dir, "progress.mp4");
|
||||
const fractions: number[] = [];
|
||||
await runFfmpeg(
|
||||
[
|
||||
...ffmpegInputArgs(clips.landscape, { mimeType: "video/mp4" }),
|
||||
...[
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-f",
|
||||
"mp4",
|
||||
`file:${output}`,
|
||||
],
|
||||
],
|
||||
{ durationMs: 3000, onProgress: (fraction) => fractions.push(fraction) },
|
||||
);
|
||||
expect(fractions.at(-1)).toBe(1);
|
||||
expect(fractions).toEqual([...fractions].sort((a, b) => a - b));
|
||||
expect(existsSync(output)).toBe(true);
|
||||
});
|
||||
|
||||
it("fails with the last stderr line, without directory paths, and removes outputs", async () => {
|
||||
const output = join(clips.dir, "broken.mp4");
|
||||
await writeFile(output, "partial");
|
||||
const failure = runFfmpeg(
|
||||
[
|
||||
...ffmpegInputArgs(join(clips.dir, "missing.mp4"), { mimeType: "video/mp4" }),
|
||||
`file:${output}`,
|
||||
],
|
||||
{ outputs: [output] },
|
||||
);
|
||||
const error = await failure.catch((caught: unknown) => caught);
|
||||
expect(error).toBeInstanceOf(FfmpegError);
|
||||
expect((error as Error).message).toMatch(/^ffmpeg failed: .*missing\.mp4.*No such file/);
|
||||
expect((error as Error).message).not.toContain(clips.dir);
|
||||
expect(existsSync(output)).toBe(false);
|
||||
});
|
||||
|
||||
it("kills ffmpeg on abort and removes the partial output", async () => {
|
||||
const output = join(clips.dir, "aborted.mp4");
|
||||
const controller = new AbortController();
|
||||
const run = runFfmpeg(
|
||||
[
|
||||
...["-re", "-f", "lavfi", "-i", "testsrc=size=320x180:rate=25"],
|
||||
...["-t", "60", "-c:v", "libx264", "-preset", "ultrafast", "-f", "mp4", `file:${output}`],
|
||||
],
|
||||
{
|
||||
signal: controller.signal,
|
||||
durationMs: 60_000,
|
||||
onProgress: () => controller.abort(new Error("cancelled")),
|
||||
outputs: [output],
|
||||
},
|
||||
);
|
||||
await expect(run).rejects.toThrow("cancelled");
|
||||
expect(existsSync(output)).toBe(false);
|
||||
});
|
||||
|
||||
it("stops when the output outgrows its cap", async () => {
|
||||
const output = join(clips.dir, "too-large.mp4");
|
||||
const run = runFfmpeg(
|
||||
[
|
||||
...["-f", "lavfi", "-i", "testsrc=size=640x360:rate=25:duration=20"],
|
||||
...["-c:v", "libx264", "-preset", "ultrafast", "-crf", "0", "-f", "mp4", `file:${output}`],
|
||||
],
|
||||
{ maxOutputBytes: 64 * 1024, outputs: [output] },
|
||||
);
|
||||
await expect(run).rejects.toThrow("The result would exceed 0 MB.");
|
||||
expect(existsSync(output)).toBe(false);
|
||||
});
|
||||
|
||||
it("extracts 16 kHz mono WAV for transcription, optionally a range", async () => {
|
||||
const output = join(clips.dir, "speech.wav");
|
||||
await extractAudio(clips.landscape, output, { mimeType: "video/mp4", startMs: 1000 });
|
||||
const probe = await probeMedia(output, { mimeType: "audio/wav" });
|
||||
expect(probe.audio).toEqual({ codec: "pcm_s16le", sampleRate: 16_000, channels: 1 });
|
||||
expect(probe.video).toBeUndefined();
|
||||
expect(Math.abs(probe.durationMs - 2000)).toBeLessThan(100);
|
||||
await expect(
|
||||
extractAudio(clips.silentPortrait, join(clips.dir, "none.wav"), { mimeType: "video/mp4" }),
|
||||
).rejects.toThrow("The file has no audio track.");
|
||||
});
|
||||
|
||||
it("renders a poster frame, falling back to the first frame past the end", async () => {
|
||||
const poster = join(clips.dir, "poster.jpg");
|
||||
await renderPoster(clips.landscape, poster, { mimeType: "video/mp4", atMs: 60_000 });
|
||||
const probe = await probeMedia(poster);
|
||||
expect(probe.video).toMatchObject({ codec: "mjpeg", width: 320, height: 180 });
|
||||
});
|
||||
});
|
||||
588
packages/adapters/src/media/ffmpeg.ts
Normal file
588
packages/adapters/src/media/ffmpeg.ts
Normal file
@ -0,0 +1,588 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { constants } from "node:fs";
|
||||
import { access, rm, stat } from "node:fs/promises";
|
||||
import { setPriority } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
|
||||
/**
|
||||
* ffmpeg and ffprobe from PATH. Every call passes an argument array (never a shell), opens
|
||||
* inputs through the `file` protocol only, and forces the demuxer from the artifact's MIME type
|
||||
* so a crafted upload cannot make ffmpeg probe it as a playlist that reads other files or URLs.
|
||||
*/
|
||||
|
||||
export interface FfmpegInfo {
|
||||
ffmpegPath: string;
|
||||
ffprobePath: string;
|
||||
}
|
||||
|
||||
export interface MediaProbe {
|
||||
durationMs: number;
|
||||
/** Width and height as displayed: swapped when the stream is rotated by 90 or 270 degrees. */
|
||||
video?: {
|
||||
codec: string;
|
||||
width: number;
|
||||
height: number;
|
||||
rotation: 0 | 90 | 180 | 270;
|
||||
/** Frames per second; 0 when unknown or implausible (variable frame rate files). */
|
||||
fps: number;
|
||||
/** Pixel aspect ratio of the displayed frame; 1 for square pixels. */
|
||||
sampleAspectRatio: number;
|
||||
};
|
||||
audio?: { codec: string; sampleRate: number; channels: number };
|
||||
hasAudio: boolean;
|
||||
}
|
||||
|
||||
export interface FfmpegProgress {
|
||||
outTimeUs?: number;
|
||||
totalSize?: number;
|
||||
done: boolean;
|
||||
}
|
||||
|
||||
export interface RunFfmpegOptions {
|
||||
signal?: AbortSignal;
|
||||
/** Expected output duration; progress is reported as out_time / durationMs. */
|
||||
durationMs?: number;
|
||||
onProgress?: (fraction: number) => void;
|
||||
cwd?: string;
|
||||
/** Kills ffmpeg once the output grows past this size. */
|
||||
maxOutputBytes?: number;
|
||||
/** Files removed when the run fails or is aborted. */
|
||||
outputs?: readonly string[];
|
||||
}
|
||||
|
||||
export class FfmpegError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly exitCode: number | null,
|
||||
readonly stderrTail: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "FfmpegError";
|
||||
}
|
||||
}
|
||||
|
||||
/** ffmpeg reports progress twice a second while it works; silence this long means a hang. */
|
||||
const STALL_TIMEOUT_MS = 2 * 60_000;
|
||||
const PROBE_TIMEOUT_MS = 60_000;
|
||||
const STDERR_TAIL_BYTES = 8 * 1024;
|
||||
const KILL_GRACE_MS = 5_000;
|
||||
const MAX_PLAUSIBLE_FPS = 240;
|
||||
|
||||
const DEMUXERS: Record<string, string> = {
|
||||
"video/mp4": "mov",
|
||||
"video/quicktime": "mov",
|
||||
"audio/mp4": "mov",
|
||||
"video/webm": "matroska",
|
||||
"audio/webm": "matroska",
|
||||
"audio/mpeg": "mp3",
|
||||
"audio/wav": "wav",
|
||||
"audio/ogg": "ogg",
|
||||
};
|
||||
|
||||
let detection: Promise<FfmpegInfo | null> | undefined;
|
||||
|
||||
/** Finds ffmpeg and ffprobe on PATH once per process; null when either is missing or broken. */
|
||||
export function detectFfmpeg(): Promise<FfmpegInfo | null> {
|
||||
detection ??= locateFfmpeg();
|
||||
return detection;
|
||||
}
|
||||
|
||||
async function locateFfmpeg(): Promise<FfmpegInfo | null> {
|
||||
const [ffmpegPath, ffprobePath] = await Promise.all([
|
||||
findExecutable("ffmpeg"),
|
||||
findExecutable("ffprobe"),
|
||||
]);
|
||||
if (!ffmpegPath || !ffprobePath) return null;
|
||||
try {
|
||||
const [ffmpeg, ffprobe] = await Promise.all([
|
||||
capture(ffmpegPath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }),
|
||||
capture(ffprobePath, ["-hide_banner", "-version"], { timeoutMs: 10_000 }),
|
||||
]);
|
||||
return ffmpeg.code === 0 && ffprobe.code === 0 ? { ffmpegPath, ffprobePath } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function findExecutable(name: string): Promise<string | null> {
|
||||
const extensions =
|
||||
process.platform === "win32"
|
||||
? (process.env.PATHEXT ?? ".EXE").split(";").filter(Boolean)
|
||||
: [""];
|
||||
for (const directory of (process.env.PATH ?? "").split(delimiter)) {
|
||||
if (!directory) continue;
|
||||
for (const extension of extensions) {
|
||||
const candidate = join(directory, `${name}${extension}`);
|
||||
try {
|
||||
await access(candidate, constants.X_OK);
|
||||
if ((await stat(candidate)).isFile()) return candidate;
|
||||
} catch {
|
||||
// Not in this directory.
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function requireFfmpeg(): Promise<FfmpegInfo> {
|
||||
const info = await detectFfmpeg();
|
||||
if (!info) throw new Error("ffmpeg and ffprobe are not installed on this server.");
|
||||
return info;
|
||||
}
|
||||
|
||||
/** ffmpeg time argument with millisecond precision and no exponent notation. */
|
||||
export function ffmpegSeconds(seconds: number): string {
|
||||
return String(Number(Math.max(0, seconds).toFixed(3)));
|
||||
}
|
||||
|
||||
/** Input arguments for a local file: optional seek, file protocol only, forced demuxer. */
|
||||
export function ffmpegInputArgs(
|
||||
path: string,
|
||||
options: { mimeType?: string; startSeconds?: number; durationSeconds?: number } = {},
|
||||
): string[] {
|
||||
const demuxer = options.mimeType ? DEMUXERS[options.mimeType] : undefined;
|
||||
return [
|
||||
...(options.startSeconds ? ["-ss", ffmpegSeconds(options.startSeconds)] : []),
|
||||
...(options.durationSeconds !== undefined
|
||||
? ["-t", ffmpegSeconds(options.durationSeconds)]
|
||||
: []),
|
||||
"-protocol_whitelist",
|
||||
"file",
|
||||
...(demuxer ? ["-f", demuxer] : []),
|
||||
"-i",
|
||||
`file:${path}`,
|
||||
];
|
||||
}
|
||||
|
||||
export async function probeMedia(
|
||||
path: string,
|
||||
options: { signal?: AbortSignal; mimeType?: string } = {},
|
||||
): Promise<MediaProbe> {
|
||||
const { ffprobePath } = await requireFfmpeg();
|
||||
const input = ffmpegInputArgs(path, { mimeType: options.mimeType });
|
||||
const result = await capture(
|
||||
ffprobePath,
|
||||
["-v", "error", ...input, "-print_format", "json", "-show_format", "-show_streams"],
|
||||
{ signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS },
|
||||
);
|
||||
if (result.code !== 0) throw ffmpegFailure("ffprobe", result.code, result.stderr);
|
||||
const probe = parseProbeOutput(result.stdout);
|
||||
if (probe.durationMs > 0 || (!probe.video && !probe.audio)) return probe;
|
||||
// Recordings streamed by browsers often carry no duration header; read it from the packets.
|
||||
const packets = await capture(
|
||||
ffprobePath,
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
...input,
|
||||
"-select_streams",
|
||||
probe.video ? "v:0" : "a:0",
|
||||
"-show_entries",
|
||||
"packet=pts_time,duration_time",
|
||||
"-of",
|
||||
"csv=p=0",
|
||||
],
|
||||
{ signal: options.signal, timeoutMs: PROBE_TIMEOUT_MS * 5 },
|
||||
);
|
||||
if (packets.code !== 0) throw ffmpegFailure("ffprobe", packets.code, packets.stderr);
|
||||
return { ...probe, durationMs: packetDurationMs(packets.stdout) };
|
||||
}
|
||||
|
||||
/** End of the last packet, from ffprobe `packet=pts_time,duration_time` CSV lines. */
|
||||
export function packetDurationMs(csv: string): number {
|
||||
let end = 0;
|
||||
for (const line of csv.split("\n")) {
|
||||
const [pts, duration] = line.split(",").map(Number);
|
||||
if (pts === undefined || !Number.isFinite(pts)) continue;
|
||||
end = Math.max(end, pts + (duration !== undefined && Number.isFinite(duration) ? duration : 0));
|
||||
}
|
||||
return Math.round(end * 1000);
|
||||
}
|
||||
|
||||
interface ProbeStream {
|
||||
codec_type?: string;
|
||||
codec_name?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
avg_frame_rate?: string;
|
||||
r_frame_rate?: string;
|
||||
sample_aspect_ratio?: string;
|
||||
sample_rate?: string;
|
||||
channels?: number;
|
||||
duration?: string;
|
||||
disposition?: { attached_pic?: number };
|
||||
tags?: { rotate?: string };
|
||||
side_data_list?: { rotation?: number }[];
|
||||
}
|
||||
|
||||
/** Parses `ffprobe -print_format json -show_format -show_streams` output. */
|
||||
export function parseProbeOutput(json: string): MediaProbe {
|
||||
const data = JSON.parse(json) as {
|
||||
format?: { duration?: string };
|
||||
streams?: ProbeStream[];
|
||||
};
|
||||
const streams = data.streams ?? [];
|
||||
// Cover art in audio files is a single attached picture, not a video track.
|
||||
const video = streams.find(
|
||||
(stream) => stream.codec_type === "video" && stream.disposition?.attached_pic !== 1,
|
||||
);
|
||||
const audio = streams.find((stream) => stream.codec_type === "audio");
|
||||
const seconds =
|
||||
positive(Number(data.format?.duration)) ??
|
||||
Math.max(0, ...streams.map((stream) => positive(Number(stream.duration)) ?? 0));
|
||||
return {
|
||||
durationMs: Math.round(seconds * 1000),
|
||||
...(video?.width && video.height
|
||||
? { video: probeVideo(video, video.width, video.height) }
|
||||
: {}),
|
||||
...(audio
|
||||
? {
|
||||
audio: {
|
||||
codec: audio.codec_name ?? "",
|
||||
sampleRate: Number(audio.sample_rate) || 0,
|
||||
channels: audio.channels ?? 0,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
hasAudio: Boolean(audio),
|
||||
};
|
||||
}
|
||||
|
||||
function probeVideo(stream: ProbeStream, width: number, height: number) {
|
||||
const rotation = normalizeRotation(
|
||||
stream.side_data_list?.find((entry) => typeof entry.rotation === "number")?.rotation ??
|
||||
Number(stream.tags?.rotate ?? 0),
|
||||
);
|
||||
const sideways = rotation === 90 || rotation === 270;
|
||||
const sar = ratio(stream.sample_aspect_ratio, ":") || 1;
|
||||
const fps = ratio(stream.avg_frame_rate, "/") || ratio(stream.r_frame_rate, "/");
|
||||
return {
|
||||
codec: stream.codec_name ?? "",
|
||||
width: sideways ? height : width,
|
||||
height: sideways ? width : height,
|
||||
rotation,
|
||||
fps: fps > 0 && fps <= MAX_PLAUSIBLE_FPS ? fps : 0,
|
||||
sampleAspectRatio: sideways ? 1 / sar : sar,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeRotation(degrees: number): 0 | 90 | 180 | 270 {
|
||||
if (!Number.isFinite(degrees)) return 0;
|
||||
const quarter = (((Math.round(degrees / 90) % 4) + 4) % 4) as 0 | 1 | 2 | 3;
|
||||
return ([0, 90, 180, 270] as const)[quarter];
|
||||
}
|
||||
|
||||
function ratio(value: string | undefined, separator: string): number {
|
||||
if (!value) return 0;
|
||||
const [numerator, denominator] = value.split(separator).map(Number);
|
||||
if (!numerator || !denominator || !Number.isFinite(numerator / denominator)) return 0;
|
||||
return numerator / denominator;
|
||||
}
|
||||
|
||||
function positive(value: number): number | undefined {
|
||||
return Number.isFinite(value) && value > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
/** Feeds ffmpeg `-progress` output (key=value lines) and reports each completed block. */
|
||||
export function createFfmpegProgressParser(
|
||||
onUpdate: (progress: FfmpegProgress) => void,
|
||||
): (chunk: string) => void {
|
||||
let buffer = "";
|
||||
let current: Omit<FfmpegProgress, "done"> = {};
|
||||
return (chunk) => {
|
||||
buffer += chunk;
|
||||
for (let newline = buffer.indexOf("\n"); newline >= 0; newline = buffer.indexOf("\n")) {
|
||||
const line = buffer.slice(0, newline).trim();
|
||||
buffer = buffer.slice(newline + 1);
|
||||
const separator = line.indexOf("=");
|
||||
if (separator < 0) continue;
|
||||
const key = line.slice(0, separator);
|
||||
const value = line.slice(separator + 1).trim();
|
||||
const number = Number(value);
|
||||
// out_time_ms is microseconds too (a long-standing ffmpeg naming bug).
|
||||
if ((key === "out_time_us" || key === "out_time_ms") && value && number >= 0) {
|
||||
current.outTimeUs = number;
|
||||
} else if (key === "total_size" && value && number >= 0) {
|
||||
current.totalSize = number;
|
||||
} else if (key === "progress") {
|
||||
onUpdate({ ...current, done: value === "end" });
|
||||
current = {};
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Runs ffmpeg with progress on stdout; rejects with the stderr tail when it fails. */
|
||||
export async function runFfmpeg(
|
||||
args: readonly string[],
|
||||
options: RunFfmpegOptions = {},
|
||||
): Promise<void> {
|
||||
options.signal?.throwIfAborted();
|
||||
const { ffmpegPath } = await requireFfmpeg();
|
||||
try {
|
||||
await spawnFfmpeg(ffmpegPath, args, options);
|
||||
} catch (error) {
|
||||
await Promise.all((options.outputs ?? []).map((path) => rm(path, { force: true })));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function spawnFfmpeg(
|
||||
ffmpegPath: string,
|
||||
args: readonly string[],
|
||||
options: RunFfmpegOptions,
|
||||
): Promise<void> {
|
||||
const { signal, durationMs, onProgress, maxOutputBytes } = options;
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(
|
||||
ffmpegPath,
|
||||
[
|
||||
"-hide_banner",
|
||||
"-nostdin",
|
||||
"-nostats",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-y",
|
||||
...args,
|
||||
],
|
||||
{ cwd: options.cwd, stdio: ["ignore", "pipe", "pipe"], windowsHide: true },
|
||||
);
|
||||
if (child.pid) {
|
||||
try {
|
||||
// Long encodes must not starve the API process that shares this machine.
|
||||
setPriority(child.pid, 10);
|
||||
} catch {
|
||||
// Lowering priority is best effort.
|
||||
}
|
||||
}
|
||||
let stderr = "";
|
||||
let settled = false;
|
||||
let reported = 0;
|
||||
let failure: unknown;
|
||||
let stallTimer: NodeJS.Timeout | undefined;
|
||||
const finish = (error?: unknown) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(stallTimer);
|
||||
signal?.removeEventListener("abort", abort);
|
||||
if (error === undefined) resolve();
|
||||
else reject(error);
|
||||
};
|
||||
// Settles once the process is gone, so callers can delete its files right away.
|
||||
const kill = (error: unknown) => {
|
||||
if (failure !== undefined || settled) return;
|
||||
failure = error;
|
||||
clearTimeout(stallTimer);
|
||||
child.kill("SIGKILL");
|
||||
setTimeout(() => finish(error), KILL_GRACE_MS).unref?.();
|
||||
};
|
||||
const abort = () => kill(signal?.reason ?? new Error("aborted"));
|
||||
const armStallTimer = () => {
|
||||
clearTimeout(stallTimer);
|
||||
stallTimer = setTimeout(
|
||||
() => kill(new FfmpegError("ffmpeg stopped making progress.", null, tail(stderr))),
|
||||
STALL_TIMEOUT_MS,
|
||||
);
|
||||
stallTimer.unref?.();
|
||||
};
|
||||
const parse = createFfmpegProgressParser((progress) => {
|
||||
if (failure !== undefined) return;
|
||||
if (maxOutputBytes !== undefined && (progress.totalSize ?? 0) > maxOutputBytes) {
|
||||
kill(new Error(outputTooLargeMessage(maxOutputBytes)));
|
||||
return;
|
||||
}
|
||||
if (!onProgress) return;
|
||||
const fraction = progress.done
|
||||
? 1
|
||||
: durationMs && progress.outTimeUs !== undefined
|
||||
? Math.min(1, progress.outTimeUs / (durationMs * 1000))
|
||||
: 0;
|
||||
if (fraction > reported) {
|
||||
reported = fraction;
|
||||
onProgress(fraction);
|
||||
}
|
||||
});
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
armStallTimer();
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
if (failure === undefined) armStallTimer();
|
||||
parse(chunk);
|
||||
});
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
stderr = tail(stderr + chunk);
|
||||
});
|
||||
child.on("error", (error) => finish(failure ?? error));
|
||||
child.on("close", (code) => {
|
||||
if (failure !== undefined) finish(failure);
|
||||
else if (code === 0) finish();
|
||||
else finish(ffmpegFailure("ffmpeg", code, stderr));
|
||||
});
|
||||
if (signal?.aborted) abort();
|
||||
});
|
||||
}
|
||||
|
||||
export function outputTooLargeMessage(maxBytes: number): string {
|
||||
return `The result would exceed ${Math.round(maxBytes / (1024 * 1024))} MB.`;
|
||||
}
|
||||
|
||||
function tail(text: string): string {
|
||||
return text.length > STDERR_TAIL_BYTES ? text.slice(-STDERR_TAIL_BYTES) : text;
|
||||
}
|
||||
|
||||
function ffmpegFailure(tool: string, code: number | null, stderr: string): FfmpegError {
|
||||
// The cause is usually the line before ffmpeg's generic summary ("Conversion failed!").
|
||||
const lines = stderr
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.slice(-2);
|
||||
// Name files by their base name only; the job's temp paths mean nothing to the reader.
|
||||
const detail = lines.join(" ").replaceAll(/(?:file:)?(?:[A-Za-z]:)?[\\/][^\s'"]*[\\/]/g, "");
|
||||
return new FfmpegError(
|
||||
detail ? `${tool} failed: ${detail}` : `${tool} failed with exit code ${code}.`,
|
||||
code,
|
||||
tail(stderr),
|
||||
);
|
||||
}
|
||||
|
||||
/** Extracts the first audio track as 16 kHz mono 16-bit WAV, the format speech models expect. */
|
||||
export async function extractAudio(
|
||||
input: string,
|
||||
output: string,
|
||||
options: {
|
||||
signal?: AbortSignal;
|
||||
mimeType?: string;
|
||||
startMs?: number;
|
||||
endMs?: number;
|
||||
onProgress?: (fraction: number) => void;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
const probe = await probeMedia(input, { signal: options.signal, mimeType: options.mimeType });
|
||||
if (!probe.hasAudio) throw new Error("The file has no audio track.");
|
||||
const startMs = Math.max(0, options.startMs ?? 0);
|
||||
const endMs = Math.min(probe.durationMs, options.endMs ?? probe.durationMs);
|
||||
if (endMs <= startMs) throw new Error("The audio range is empty.");
|
||||
await runFfmpeg(
|
||||
[
|
||||
...ffmpegInputArgs(input, {
|
||||
mimeType: options.mimeType,
|
||||
startSeconds: startMs / 1000,
|
||||
durationSeconds: (endMs - startMs) / 1000,
|
||||
}),
|
||||
"-map",
|
||||
"0:a:0",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
"-map_metadata",
|
||||
"-1",
|
||||
"-f",
|
||||
"wav",
|
||||
`file:${output}`,
|
||||
],
|
||||
{
|
||||
signal: options.signal,
|
||||
durationMs: endMs - startMs,
|
||||
onProgress: options.onProgress,
|
||||
outputs: [output],
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Writes one JPEG frame at atMs (default 0), falling back to the first frame past the end. */
|
||||
export async function renderPoster(
|
||||
input: string,
|
||||
output: string,
|
||||
options: { signal?: AbortSignal; mimeType?: string; atMs?: number } = {},
|
||||
): Promise<void> {
|
||||
const render = (atMs: number) =>
|
||||
runFfmpeg(
|
||||
[
|
||||
...ffmpegInputArgs(input, { mimeType: options.mimeType, startSeconds: atMs / 1000 }),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-c:v",
|
||||
"mjpeg",
|
||||
"-q:v",
|
||||
"3",
|
||||
"-map_metadata",
|
||||
"-1",
|
||||
"-f",
|
||||
"image2",
|
||||
"-update",
|
||||
"1",
|
||||
`file:${output}`,
|
||||
],
|
||||
{ signal: options.signal, outputs: [output] },
|
||||
);
|
||||
const atMs = Math.max(0, options.atMs ?? 0);
|
||||
await render(atMs);
|
||||
// Seeking into the last frame's gap writes nothing; the first frame is better than no poster.
|
||||
if (atMs > 0 && !(await nonEmptyFile(output))) await render(0);
|
||||
if (!(await nonEmptyFile(output))) throw new Error("Could not render a poster frame.");
|
||||
}
|
||||
|
||||
async function nonEmptyFile(path: string): Promise<boolean> {
|
||||
try {
|
||||
return (await stat(path)).size > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
interface CaptureResult {
|
||||
code: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
function capture(
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
options: { signal?: AbortSignal; timeoutMs: number },
|
||||
): Promise<CaptureResult> {
|
||||
options.signal?.throwIfAborted();
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], windowsHide: true });
|
||||
const stdout: Buffer[] = [];
|
||||
let stderr = "";
|
||||
let settled = false;
|
||||
const finish = (error: unknown, result?: CaptureResult) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
options.signal?.removeEventListener("abort", abort);
|
||||
if (result) resolve(result);
|
||||
else reject(error);
|
||||
};
|
||||
const kill = (error: unknown) => {
|
||||
child.kill("SIGKILL");
|
||||
finish(error);
|
||||
};
|
||||
const abort = () => kill(options.signal?.reason ?? new Error("aborted"));
|
||||
const timer = setTimeout(
|
||||
() => kill(new Error(`${command} timed out after ${options.timeoutMs} ms`)),
|
||||
options.timeoutMs,
|
||||
);
|
||||
timer.unref?.();
|
||||
options.signal?.addEventListener("abort", abort, { once: true });
|
||||
child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk));
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
stderr = tail(stderr + chunk);
|
||||
});
|
||||
child.on("error", (error) => finish(error));
|
||||
child.on("close", (code) =>
|
||||
finish(undefined, { code, stdout: Buffer.concat(stdout).toString("utf8"), stderr }),
|
||||
);
|
||||
});
|
||||
}
|
||||
106
packages/adapters/src/media/media-test-support.ts
Normal file
106
packages/adapters/src/media/media-test-support.ts
Normal file
@ -0,0 +1,106 @@
|
||||
import { copyFile, mkdtemp, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { MediaJobContext, MediaJobInputFile, MediaJobOutputFile } from "@rakazo/adapter-kit";
|
||||
import type { MediaOutput, MediaOutputKind } from "@rakazo/contracts";
|
||||
import { attachmentKindOf } from "@rakazo/contracts";
|
||||
import { runFfmpeg } from "./ffmpeg.js";
|
||||
|
||||
export interface TestClips {
|
||||
dir: string;
|
||||
/** 320×180, 25 fps, 3 s, 440 Hz tone. */
|
||||
landscape: string;
|
||||
/** 180×320, 30 fps, 2 s, no audio. */
|
||||
silentPortrait: string;
|
||||
srt: string;
|
||||
vtt: string;
|
||||
}
|
||||
|
||||
/** Tiny generated clips; nothing binary is committed. */
|
||||
export async function createTestClips(): Promise<TestClips> {
|
||||
const dir = await mkdtemp(join(tmpdir(), "evoo-media-test-"));
|
||||
const landscape = join(dir, "landscape.mp4");
|
||||
const silentPortrait = join(dir, "silent-portrait.mp4");
|
||||
const encode = ["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p"];
|
||||
await runFfmpeg([
|
||||
...["-f", "lavfi", "-i", "testsrc=size=320x180:rate=25:duration=3"],
|
||||
...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=3"],
|
||||
...encode,
|
||||
...["-c:a", "aac", "-shortest", `file:${landscape}`],
|
||||
]);
|
||||
await runFfmpeg([
|
||||
...["-f", "lavfi", "-i", "testsrc2=size=180x320:rate=30:duration=2"],
|
||||
...encode,
|
||||
`file:${silentPortrait}`,
|
||||
]);
|
||||
const srt = join(dir, "captions.srt");
|
||||
await writeFile(
|
||||
srt,
|
||||
"1\n00:00:00,100 --> 00:00:01,500\nHello from the subtitles\n\n2\n00:00:01,500 --> 00:00:02,900\nZweite Zeile äöü\n",
|
||||
);
|
||||
const vtt = join(dir, "captions.vtt");
|
||||
await writeFile(vtt, "WEBVTT\n\n00:00.100 --> 00:01.500\nHello from WebVTT\n");
|
||||
return { dir, landscape, silentPortrait, srt, vtt };
|
||||
}
|
||||
|
||||
export interface TestArtifact {
|
||||
path: string;
|
||||
name: string;
|
||||
mimeType: string;
|
||||
kind?: MediaOutputKind;
|
||||
}
|
||||
|
||||
/** A MediaJobContext over local files that records progress and outputs. */
|
||||
export async function fakeMediaJobContext(input: unknown, artifacts: Record<string, TestArtifact>) {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), "evoo-media-job-"));
|
||||
const controller = new AbortController();
|
||||
const progress: number[] = [];
|
||||
const outputs: (MediaJobOutputFile & { size: number; posterSize?: number })[] = [];
|
||||
const context: MediaJobContext = {
|
||||
jobId: "job-1",
|
||||
spaceId: "space-1",
|
||||
userId: "user-1",
|
||||
botId: null,
|
||||
input,
|
||||
tempDir,
|
||||
signal: controller.signal,
|
||||
progress: async (fraction) => {
|
||||
progress.push(fraction);
|
||||
},
|
||||
fetchArtifact: async (artifactId): Promise<MediaJobInputFile> => {
|
||||
const artifact = artifacts[artifactId];
|
||||
if (!artifact) throw new Error(`Artifact ${artifactId} not found.`);
|
||||
const path = join(tempDir, `input-${artifactId}`);
|
||||
await copyFile(artifact.path, path);
|
||||
return {
|
||||
artifactId,
|
||||
path,
|
||||
name: artifact.name,
|
||||
mimeType: artifact.mimeType,
|
||||
kind: artifact.kind ?? attachmentKindOf(artifact.mimeType),
|
||||
size: (await stat(path)).size,
|
||||
};
|
||||
},
|
||||
addOutput: async (file): Promise<MediaOutput> => {
|
||||
// Files must exist when handed over; the runner stores them before the job ends.
|
||||
const size = (await stat(file.path)).size;
|
||||
const posterSize = file.posterPath ? (await stat(file.posterPath)).size : undefined;
|
||||
outputs.push({ ...file, size, ...(posterSize !== undefined ? { posterSize } : {}) });
|
||||
const index = outputs.length;
|
||||
return {
|
||||
artifactId: `output-${index}`,
|
||||
kind: attachmentKindOf(file.mimeType),
|
||||
mimeType: file.mimeType,
|
||||
name: file.name,
|
||||
size,
|
||||
...(file.width ? { width: file.width } : {}),
|
||||
...(file.height ? { height: file.height } : {}),
|
||||
...(file.durationMs !== undefined ? { durationMs: file.durationMs } : {}),
|
||||
...(file.posterPath ? { posterArtifactId: `poster-${index}` } : {}),
|
||||
...(file.parentArtifactId ? { parentArtifactId: file.parentArtifactId } : {}),
|
||||
...(file.preset ? { preset: file.preset } : {}),
|
||||
};
|
||||
},
|
||||
};
|
||||
return { context, controller, progress, outputs, tempDir };
|
||||
}
|
||||
@ -11,6 +11,7 @@ import {
|
||||
createOpenAiCompatibleTranscriptionProvider,
|
||||
TranscriptionServerUnavailableError,
|
||||
} from "../openai-compatible-transcription.js";
|
||||
import { detectFfmpeg, extractAudio, probeMedia } from "./ffmpeg.js";
|
||||
|
||||
/** Bytes per second of the 16 kHz mono 16-bit WAV that audio is converted to before upload. */
|
||||
export const TRANSCRIPTION_WAV_BYTES_PER_SECOND = 16_000 * 2;
|
||||
@ -105,6 +106,13 @@ export interface TranscriptionAudioTools {
|
||||
): Promise<void>;
|
||||
}
|
||||
|
||||
/** Audio conversion through the server's ffmpeg, when it has one. */
|
||||
export const ffmpegTranscriptionAudio: TranscriptionAudioTools = {
|
||||
available: async () => (await detectFfmpeg()) !== null,
|
||||
durationMs: async (input, options) => (await probeMedia(input, options)).durationMs,
|
||||
extract: (input, output, options) => extractAudio(input, output, options),
|
||||
};
|
||||
|
||||
/** Time ranges of converted audio that each fit one upload. */
|
||||
export function planAudioChunks(
|
||||
durationMs: number,
|
||||
|
||||
300
packages/adapters/src/media/video-edit-plan.test.ts
Normal file
300
packages/adapters/src/media/video-edit-plan.test.ts
Normal file
@ -0,0 +1,300 @@
|
||||
import type { EditVideoStep } from "@rakazo/contracts";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { MediaProbe } from "./ffmpeg.js";
|
||||
import type { PlanInput } from "./video-edit-plan.js";
|
||||
import { atempoFilters, planVideoEdit, subtitleStyle } from "./video-edit-plan.js";
|
||||
|
||||
function probe(
|
||||
overrides: Partial<NonNullable<MediaProbe["video"]>> & {
|
||||
durationMs?: number;
|
||||
hasAudio?: boolean;
|
||||
} = {},
|
||||
): MediaProbe {
|
||||
const { durationMs = 10_000, hasAudio = true, ...video } = overrides;
|
||||
return {
|
||||
durationMs,
|
||||
video: {
|
||||
codec: "h264",
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
rotation: 0,
|
||||
fps: 25,
|
||||
sampleAspectRatio: 1,
|
||||
...video,
|
||||
},
|
||||
...(hasAudio ? { audio: { codec: "aac", sampleRate: 44_100, channels: 2 } } : {}),
|
||||
hasAudio,
|
||||
};
|
||||
}
|
||||
|
||||
const source: PlanInput = {
|
||||
path: "/jobs/1/input-source",
|
||||
mimeType: "video/mp4",
|
||||
name: "holiday.mp4",
|
||||
probe: probe(),
|
||||
};
|
||||
|
||||
function plan(steps: EditVideoStep[], extra: Partial<Parameters<typeof planVideoEdit>[0]> = {}) {
|
||||
return planVideoEdit({ source, steps, outputPath: "/jobs/1/output.mp4", ...extra });
|
||||
}
|
||||
|
||||
function graphOf(args: string[]): string {
|
||||
return args[args.indexOf("-filter_complex") + 1]!;
|
||||
}
|
||||
|
||||
describe("planVideoEdit", () => {
|
||||
it("seeks the input for a leading trim and encodes H.264/AAC MP4", () => {
|
||||
const result = plan([{ op: "trim", start: 1.5, end: 4 }]);
|
||||
expect(result.args).toEqual([
|
||||
...["-ss", "1.5", "-t", "2.5", "-protocol_whitelist", "file", "-f", "mov"],
|
||||
...["-i", "file:/jobs/1/input-source"],
|
||||
"-filter_complex",
|
||||
"[0:v:0]format=yuv420p[v1];[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a2]",
|
||||
...["-map", "[v1]", "-map", "[a2]", "-c:a", "aac", "-b:a", "160k"],
|
||||
...["-c:v", "libx264", "-preset", "veryfast", "-crf", "21", "-pix_fmt", "yuv420p"],
|
||||
...["-movflags", "+faststart", "-map_metadata", "-1", "-map_chapters", "-1"],
|
||||
...["-f", "mp4", "file:/jobs/1/output.mp4"],
|
||||
]);
|
||||
expect(result).toMatchObject({ kind: "video", durationMs: 2500, width: 1920, height: 1080 });
|
||||
});
|
||||
|
||||
it("keeps sorted, merged segments from one seek with split, trims and concat", () => {
|
||||
const result = plan([
|
||||
{
|
||||
op: "keep",
|
||||
segments: [
|
||||
{ start: 6, end: 8 },
|
||||
{ start: 2, end: 3 },
|
||||
{ start: 2.5, end: 4 },
|
||||
],
|
||||
},
|
||||
]);
|
||||
expect(result.args.slice(0, 4)).toEqual(["-ss", "2", "-t", "6"]);
|
||||
const graph = graphOf(result.args);
|
||||
expect(graph).toContain("[0:v:0]split=2[v1][v2]");
|
||||
expect(graph).toContain("[v1]trim=start=0:end=2,setpts=PTS-STARTPTS[v3]");
|
||||
expect(graph).toContain("[v2]trim=start=4:end=6,setpts=PTS-STARTPTS[v4]");
|
||||
expect(graph).toContain("[0:a:0]asplit=2[a5][a6]");
|
||||
expect(graph).toContain("[v3][a7][v4][a8]concat=n=2:v=1:a=1[v9][a10]");
|
||||
expect(result.durationMs).toBe(4000);
|
||||
});
|
||||
|
||||
it("applies later cuts on the timeline left by earlier steps", () => {
|
||||
const result = plan([
|
||||
{ op: "speed", factor: 2 },
|
||||
{ op: "trim", start: 1, end: 99 },
|
||||
]);
|
||||
expect(graphOf(result.args)).toContain(
|
||||
"[0:v:0]setpts=PTS/2,fps=25,trim=start=1:end=5,setpts=PTS-STARTPTS,format=yuv420p[v1]",
|
||||
);
|
||||
expect(result.durationMs).toBe(4000);
|
||||
});
|
||||
|
||||
it("rejects ranges outside the video and too short ones", () => {
|
||||
expect(() => plan([{ op: "trim", start: 12, end: 18 }])).toThrow(
|
||||
"trim 12–18 s is outside the 10 s video.",
|
||||
);
|
||||
expect(() =>
|
||||
plan([
|
||||
{ op: "speed", factor: 2 },
|
||||
{
|
||||
op: "keep",
|
||||
segments: [
|
||||
{ start: 1, end: 2 },
|
||||
{ start: 6, end: 7 },
|
||||
],
|
||||
},
|
||||
]),
|
||||
).toThrow("keep 6–7 s is outside the 5 s video.");
|
||||
expect(() => plan([{ op: "trim", start: 9.95, end: 12 }])).toThrow("shorter than 0.1 s");
|
||||
});
|
||||
|
||||
it("appends clips fitted to the frame, with silence for clips without audio", () => {
|
||||
const clip: PlanInput = {
|
||||
path: "/jobs/1/input-clip",
|
||||
mimeType: "video/webm",
|
||||
name: "clip.webm",
|
||||
probe: probe({ width: 1080, height: 1920, fps: 30, durationMs: 2000, hasAudio: false }),
|
||||
};
|
||||
const result = plan([{ op: "concat", artifact_ids: ["clip"] }], {
|
||||
clips: new Map([["clip", clip]]),
|
||||
});
|
||||
expect(result.args).toContain("matroska");
|
||||
const graph = graphOf(result.args);
|
||||
expect(graph).toContain("[0:v:0]setsar=1,format=yuv420p[v1]");
|
||||
expect(graph).toContain(
|
||||
"[1:v:0]scale=1920:1080:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1920:1080:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=25,format=yuv420p[v3]",
|
||||
);
|
||||
expect(graph).toContain(
|
||||
"anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=2,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a4]",
|
||||
);
|
||||
expect(graph).toContain("[v1][a2][v3][a4]concat=n=2:v=1:a=1");
|
||||
expect(result.durationMs).toBe(12_000);
|
||||
});
|
||||
|
||||
it("adds silence to a silent source when an appended clip has audio", () => {
|
||||
const silent = { ...source, probe: probe({ hasAudio: false }) };
|
||||
const result = planVideoEdit({
|
||||
source: silent,
|
||||
steps: [{ op: "concat", artifact_ids: ["clip"] }],
|
||||
clips: new Map([["clip", source]]),
|
||||
outputPath: "/out.mp4",
|
||||
});
|
||||
expect(graphOf(result.args)).toContain(
|
||||
"anullsrc=channel_layout=stereo:sample_rate=48000,atrim=end=10",
|
||||
);
|
||||
expect(result.args.filter((arg) => arg === "-map")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("appends clips with the last reframe's fit", () => {
|
||||
const result = plan(
|
||||
[
|
||||
{ op: "reframe", preset: "vertical", fit: "crop" },
|
||||
{ op: "concat", artifact_ids: ["clip"] },
|
||||
],
|
||||
{ clips: new Map([["clip", source]]) },
|
||||
);
|
||||
expect(graphOf(result.args)).toContain(
|
||||
"[1:v:0]scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920,setsar=1,fps=25",
|
||||
);
|
||||
});
|
||||
|
||||
it("reframes by crop, pad or blur, and only scales when the aspect already matches", () => {
|
||||
expect(graphOf(plan([{ op: "reframe", preset: "square", fit: "crop" }]).args)).toContain(
|
||||
"scale=1080:1080:force_original_aspect_ratio=increase,crop=1080:1080,setsar=1",
|
||||
);
|
||||
expect(graphOf(plan([{ op: "reframe", preset: "portrait", fit: "pad" }]).args)).toContain(
|
||||
"pad=1080:1350:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1",
|
||||
);
|
||||
const blur = plan([{ op: "reframe", preset: "vertical", fit: "blur" }]);
|
||||
expect(graphOf(blur.args)).toBe(
|
||||
[
|
||||
"[0:v:0]split=2[v1][v2]",
|
||||
"[v1]scale=270:480:force_original_aspect_ratio=increase,crop=270:480,boxblur=27:2,scale=1080:1920,setsar=1[v3]",
|
||||
"[v2]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,setsar=1[v4]",
|
||||
"[v3][v4]overlay=(W-w)/2:(H-h)/2[v5]",
|
||||
"[v5]format=yuv420p[v6]",
|
||||
"[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a7]",
|
||||
].join(";"),
|
||||
);
|
||||
expect(blur).toMatchObject({ width: 1080, height: 1920 });
|
||||
expect(graphOf(plan([{ op: "reframe", preset: "landscape", fit: "blur" }]).args)).toContain(
|
||||
"[0:v:0]scale=1920:1080,setsar=1,format=yuv420p",
|
||||
);
|
||||
});
|
||||
|
||||
it("changes speed with atempo chains and keeps the source frame rate", () => {
|
||||
const fast = plan([{ op: "speed", factor: 4 }]);
|
||||
expect(graphOf(fast.args)).toContain("setpts=PTS/4,fps=25");
|
||||
expect(graphOf(fast.args)).toContain("atempo=2,atempo=2");
|
||||
expect(fast.durationMs).toBe(2500);
|
||||
expect(atempoFilters(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]);
|
||||
expect(atempoFilters(3)).toEqual(["atempo=2", "atempo=1.5"]);
|
||||
expect(atempoFilters(0.8)).toEqual(["atempo=0.8"]);
|
||||
});
|
||||
|
||||
it("changes volume, and mute drops the audio track", () => {
|
||||
expect(graphOf(plan([{ op: "volume", gain: 0.5 }]).args)).toContain(
|
||||
"[0:a:0]volume=0.5,aformat",
|
||||
);
|
||||
const muted = plan([{ op: "volume", gain: 2 }, { op: "mute" }]);
|
||||
expect(muted.args.filter((arg) => arg === "-map")).toHaveLength(1);
|
||||
expect(muted.args).not.toContain("aac");
|
||||
});
|
||||
|
||||
it("burns in subtitles styled for the current frame from a generated file name", () => {
|
||||
const result = plan(
|
||||
[
|
||||
{ op: "reframe", preset: "vertical", fit: "crop" },
|
||||
{ op: "subtitles", artifact_id: "subs" },
|
||||
],
|
||||
{ subtitleFiles: new Map([["subs", "subtitles-0.srt"]]) },
|
||||
);
|
||||
expect(graphOf(result.args)).toContain(
|
||||
`subtitles=filename=subtitles-0.srt:force_style='${subtitleStyle(1080, 1920)}'`,
|
||||
);
|
||||
expect(() =>
|
||||
plan([{ op: "subtitles", artifact_id: "subs" }], {
|
||||
subtitleFiles: new Map([["subs", "x.srt:force_style='a'"]]),
|
||||
}),
|
||||
).toThrow("Unsafe subtitle file name.");
|
||||
});
|
||||
|
||||
it("styles subtitles above the controls of tall frames", () => {
|
||||
expect(subtitleStyle(1080, 1920)).toContain("MarginV=384");
|
||||
expect(subtitleStyle(1080, 1920)).toContain("FontSize=65");
|
||||
expect(subtitleStyle(1920, 1080)).toContain("MarginV=65");
|
||||
expect(subtitleStyle(1920, 1080)).toContain("PlayResX=1920,PlayResY=1080");
|
||||
});
|
||||
|
||||
it("extracts audio without decoding video work", () => {
|
||||
const result = planVideoEdit({
|
||||
source,
|
||||
steps: [
|
||||
{ op: "trim", start: 0, end: 5 },
|
||||
{ op: "reframe", preset: "vertical", fit: "blur" },
|
||||
{ op: "subtitles", artifact_id: "subs" },
|
||||
{ op: "extract_audio" },
|
||||
],
|
||||
subtitleFiles: new Map([["subs", "subtitles-0.srt"]]),
|
||||
outputPath: "/out.m4a",
|
||||
});
|
||||
expect(graphOf(result.args)).toBe(
|
||||
"[0:a:0]aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[a1]",
|
||||
);
|
||||
expect(result.args).toEqual(expect.arrayContaining(["-map", "[a1]", "-b:a", "192k"]));
|
||||
expect(result.args).not.toContain("libx264");
|
||||
expect(result).toMatchObject({ kind: "audio", durationMs: 5000 });
|
||||
expect(() =>
|
||||
planVideoEdit({
|
||||
source: { ...source, probe: probe({ hasAudio: false }) },
|
||||
steps: [{ op: "extract_audio" }],
|
||||
outputPath: "/out.m4a",
|
||||
}),
|
||||
).toThrow("holiday.mp4 has no audio.");
|
||||
});
|
||||
|
||||
it("validates the poster time against the result", () => {
|
||||
expect(
|
||||
plan([
|
||||
{ op: "trim", start: 0, end: 4 },
|
||||
{ op: "poster", at: 3 },
|
||||
]).posterAtMs,
|
||||
).toBe(3000);
|
||||
expect(() =>
|
||||
plan([
|
||||
{ op: "trim", start: 0, end: 4 },
|
||||
{ op: "poster", at: 5 },
|
||||
]),
|
||||
).toThrow("poster at 5 s is past the end of the 4 s result.");
|
||||
});
|
||||
|
||||
it("keeps file paths out of the filter graph", () => {
|
||||
const hostile = { ...source, path: "/jobs/1/a'b[c]:d;e,f.mp4", name: "x'];[0:v]drawtext=" };
|
||||
const result = planVideoEdit({
|
||||
source: hostile,
|
||||
steps: [{ op: "reframe", preset: "square", fit: "blur" }],
|
||||
outputPath: "/jobs/1/out.mp4",
|
||||
});
|
||||
expect(graphOf(result.args)).not.toContain("a'b");
|
||||
expect(graphOf(result.args)).not.toContain("drawtext");
|
||||
expect(result.args).toContain("file:/jobs/1/a'b[c]:d;e,f.mp4");
|
||||
});
|
||||
|
||||
it("squares pixels and evens odd frame sizes", () => {
|
||||
const anamorphic = {
|
||||
...source,
|
||||
probe: probe({ width: 1440, height: 1080, sampleAspectRatio: 4 / 3 }),
|
||||
};
|
||||
const result = planVideoEdit({
|
||||
source: anamorphic,
|
||||
steps: [{ op: "mute" }],
|
||||
outputPath: "/o.mp4",
|
||||
});
|
||||
expect(graphOf(result.args)).toBe("[0:v:0]scale=1920:1080,setsar=1,format=yuv420p[v1]");
|
||||
expect(result).toMatchObject({ width: 1920, height: 1080 });
|
||||
const odd = { ...source, probe: probe({ width: 721, height: 405 }) };
|
||||
const evened = planVideoEdit({ source: odd, steps: [{ op: "mute" }], outputPath: "/o.mp4" });
|
||||
expect(graphOf(evened.args)).toBe("[0:v:0]crop=720:404:0:0,format=yuv420p[v1]");
|
||||
});
|
||||
});
|
||||
571
packages/adapters/src/media/video-edit-plan.ts
Normal file
571
packages/adapters/src/media/video-edit-plan.ts
Normal file
@ -0,0 +1,571 @@
|
||||
import type { EditVideoStep, MediaFit } from "@rakazo/contracts";
|
||||
import { MEDIA_PRESETS } from "@rakazo/contracts";
|
||||
import type { MediaProbe } from "./ffmpeg.js";
|
||||
import { ffmpegInputArgs, ffmpegSeconds } from "./ffmpeg.js";
|
||||
|
||||
/**
|
||||
* Turns an ordered list of edit steps into one ffmpeg invocation with a single filter_complex
|
||||
* pass. Pure: the caller probes the inputs and runs the arguments.
|
||||
*/
|
||||
|
||||
export interface PlanInput {
|
||||
path: string;
|
||||
mimeType: string;
|
||||
name: string;
|
||||
probe: MediaProbe;
|
||||
}
|
||||
|
||||
export interface VideoEditPlanRequest {
|
||||
source: PlanInput;
|
||||
steps: readonly EditVideoStep[];
|
||||
/** Videos named by concat steps, by artifact id. */
|
||||
clips?: ReadonlyMap<string, PlanInput>;
|
||||
/** Subtitle files named by subtitles steps, by artifact id: safe names relative to ffmpeg's cwd. */
|
||||
subtitleFiles?: ReadonlyMap<string, string>;
|
||||
outputPath: string;
|
||||
}
|
||||
|
||||
export interface VideoEditPlan {
|
||||
args: string[];
|
||||
kind: "video" | "audio";
|
||||
durationMs: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
posterAtMs?: number;
|
||||
}
|
||||
|
||||
export const VIDEO_ENCODE_ARGS = [
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"21",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
];
|
||||
const AUDIO_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "160k"];
|
||||
const AUDIO_ONLY_ENCODE_ARGS = ["-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart"];
|
||||
/** Location and other camera metadata stay private; chapters do not survive cuts anyway. */
|
||||
const STRIP_METADATA_ARGS = ["-map_metadata", "-1", "-map_chapters", "-1"];
|
||||
const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo";
|
||||
const SAFE_FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
||||
const MIN_SEGMENT_SECONDS = 0.1;
|
||||
const FALLBACK_FPS = 30;
|
||||
|
||||
interface Track {
|
||||
label: string;
|
||||
filters: string[];
|
||||
}
|
||||
|
||||
interface Timeline {
|
||||
/** Null when the result is audio only: video work is skipped. */
|
||||
video: Track | null;
|
||||
audio: Track | null;
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
seconds: number;
|
||||
/** How appended clips fill the frame: the last reframe's fit, otherwise bars. */
|
||||
fit: MediaFit;
|
||||
}
|
||||
|
||||
class FilterGraph {
|
||||
private readonly chains: string[] = [];
|
||||
private count = 0;
|
||||
|
||||
label(kind: "v" | "a"): string {
|
||||
this.count += 1;
|
||||
return `${kind}${this.count}`;
|
||||
}
|
||||
|
||||
add(inputs: readonly string[], filters: readonly string[], outputs: readonly string[]) {
|
||||
const pads = (labels: readonly string[]) => labels.map((label) => `[${label}]`).join("");
|
||||
this.chains.push(`${pads(inputs)}${filters.join(",")}${pads(outputs)}`);
|
||||
}
|
||||
|
||||
/** Ends a track's pending filters in a labelled pad other chains can consume. */
|
||||
flush(track: Track, kind: "v" | "a"): string {
|
||||
if (track.filters.length === 0) return track.label;
|
||||
const output = this.label(kind);
|
||||
this.add([track.label], track.filters, [output]);
|
||||
track.label = output;
|
||||
track.filters = [];
|
||||
return output;
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
return this.chains.join(";");
|
||||
}
|
||||
}
|
||||
|
||||
export function planVideoEdit(request: VideoEditPlanRequest): VideoEditPlan {
|
||||
const { source } = request;
|
||||
const video = source.probe.video;
|
||||
if (!video) throw new Error(`${source.name} has no video.`);
|
||||
const sourceSeconds = source.probe.durationMs / 1000;
|
||||
const audioOnly = request.steps.at(-1)?.op === "extract_audio";
|
||||
const graph = new FilterGraph();
|
||||
const inputs: string[][] = [];
|
||||
const addInput = (input: PlanInput, seek?: { start: number; end: number }) => {
|
||||
inputs.push(
|
||||
ffmpegInputArgs(input.path, {
|
||||
mimeType: input.mimeType,
|
||||
...(seek ? { startSeconds: seek.start, durationSeconds: seek.end - seek.start } : {}),
|
||||
}),
|
||||
);
|
||||
return inputs.length - 1;
|
||||
};
|
||||
|
||||
let steps = request.steps;
|
||||
// A leading cut seeks the input instead of decoding everything before it.
|
||||
const first = steps[0];
|
||||
let segments: Segment[] | undefined;
|
||||
if (first?.op === "trim" || first?.op === "keep") {
|
||||
segments = normalizeSegments(
|
||||
first.op === "trim" ? [first] : first.segments,
|
||||
sourceSeconds,
|
||||
first.op,
|
||||
);
|
||||
steps = steps.slice(1);
|
||||
}
|
||||
const seek = segments && { start: segments[0]!.start, end: segments.at(-1)!.end };
|
||||
const index = addInput(source, seek);
|
||||
const size = displaySize(video);
|
||||
const timeline: Timeline = {
|
||||
video: audioOnly ? null : { label: `${index}:v:0`, filters: sourceFilters(video) },
|
||||
audio: source.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null,
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
fps: video.fps,
|
||||
seconds: seek ? seek.end - seek.start : sourceSeconds,
|
||||
fit: "pad",
|
||||
};
|
||||
if (segments && seek && segments.length > 1) {
|
||||
keepSegments(
|
||||
graph,
|
||||
timeline,
|
||||
segments.map((segment) => ({
|
||||
start: segment.start - seek.start,
|
||||
end: segment.end - seek.start,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
let posterAt: number | undefined;
|
||||
for (const step of steps) {
|
||||
switch (step.op) {
|
||||
case "trim":
|
||||
case "keep":
|
||||
keepSegments(
|
||||
graph,
|
||||
timeline,
|
||||
normalizeSegments(step.op === "trim" ? [step] : step.segments, timeline.seconds, step.op),
|
||||
);
|
||||
break;
|
||||
case "concat":
|
||||
appendClips(
|
||||
graph,
|
||||
timeline,
|
||||
step.artifact_ids.map((id) => {
|
||||
const clip = request.clips?.get(id);
|
||||
if (!clip) throw new Error(`Video ${id} is not available.`);
|
||||
return { clip, index: addInput(clip) };
|
||||
}),
|
||||
);
|
||||
break;
|
||||
case "reframe": {
|
||||
const preset = MEDIA_PRESETS[step.preset];
|
||||
if (timeline.video) {
|
||||
timeline.video = fitFrame(
|
||||
graph,
|
||||
timeline.video,
|
||||
timeline,
|
||||
{ width: preset.width, height: preset.height },
|
||||
step.fit,
|
||||
);
|
||||
}
|
||||
timeline.width = preset.width;
|
||||
timeline.height = preset.height;
|
||||
timeline.fit = step.fit;
|
||||
break;
|
||||
}
|
||||
case "speed":
|
||||
timeline.video?.filters.push(
|
||||
`setpts=PTS/${number(step.factor)}`,
|
||||
...(timeline.fps ? [`fps=${number(timeline.fps)}`] : []),
|
||||
);
|
||||
timeline.audio?.filters.push(...atempoFilters(step.factor));
|
||||
timeline.seconds /= step.factor;
|
||||
break;
|
||||
case "volume":
|
||||
timeline.audio?.filters.push(`volume=${number(step.gain)}`);
|
||||
break;
|
||||
case "mute":
|
||||
timeline.audio = null;
|
||||
break;
|
||||
case "subtitles": {
|
||||
const file = request.subtitleFiles?.get(step.artifact_id);
|
||||
if (!file) throw new Error(`Subtitles ${step.artifact_id} are not available.`);
|
||||
// Only generated names reach the filter graph, so nothing here needs escaping.
|
||||
if (!SAFE_FILE_NAME.test(file)) throw new Error("Unsafe subtitle file name.");
|
||||
timeline.video?.filters.push(
|
||||
`subtitles=filename=${file}:force_style='${subtitleStyle(timeline.width, timeline.height)}'`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
case "extract_audio":
|
||||
break;
|
||||
case "poster":
|
||||
posterAt = step.at;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (posterAt !== undefined && posterAt >= timeline.seconds) {
|
||||
throw new Error(
|
||||
`poster at ${seconds(posterAt)} s is past the end of the ${seconds(timeline.seconds)} s result.`,
|
||||
);
|
||||
}
|
||||
const durationMs = Math.round(timeline.seconds * 1000);
|
||||
const output = `file:${request.outputPath}`;
|
||||
if (audioOnly || !timeline.video) {
|
||||
if (!timeline.audio) throw new Error(`${source.name} has no audio.`);
|
||||
timeline.audio.filters.push(AUDIO_FORMAT);
|
||||
const audio = graph.flush(timeline.audio, "a");
|
||||
return {
|
||||
args: [
|
||||
...inputs.flat(),
|
||||
"-filter_complex",
|
||||
graph.toString(),
|
||||
"-map",
|
||||
`[${audio}]`,
|
||||
...AUDIO_ONLY_ENCODE_ARGS,
|
||||
...STRIP_METADATA_ARGS,
|
||||
"-f",
|
||||
"mp4",
|
||||
output,
|
||||
],
|
||||
kind: "audio",
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
timeline.video.filters.push("format=yuv420p");
|
||||
const videoOut = graph.flush(timeline.video, "v");
|
||||
let audioOut: string | undefined;
|
||||
if (timeline.audio) {
|
||||
timeline.audio.filters.push(AUDIO_FORMAT);
|
||||
audioOut = graph.flush(timeline.audio, "a");
|
||||
}
|
||||
return {
|
||||
args: [
|
||||
...inputs.flat(),
|
||||
"-filter_complex",
|
||||
graph.toString(),
|
||||
"-map",
|
||||
`[${videoOut}]`,
|
||||
...(audioOut ? ["-map", `[${audioOut}]`, ...AUDIO_ENCODE_ARGS] : []),
|
||||
...VIDEO_ENCODE_ARGS,
|
||||
...STRIP_METADATA_ARGS,
|
||||
"-f",
|
||||
"mp4",
|
||||
output,
|
||||
],
|
||||
kind: "video",
|
||||
durationMs,
|
||||
width: timeline.width,
|
||||
height: timeline.height,
|
||||
...(posterAt !== undefined ? { posterAtMs: Math.round(posterAt * 1000) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
interface Segment {
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
|
||||
/** Validates ranges against the timeline, clamps ends to it, sorts and merges overlaps. */
|
||||
function normalizeSegments(
|
||||
ranges: readonly Segment[],
|
||||
duration: number,
|
||||
op: "trim" | "keep",
|
||||
): Segment[] {
|
||||
const clamped = ranges.map((range) => {
|
||||
if (range.start >= duration) {
|
||||
throw new Error(
|
||||
`${op} ${seconds(range.start)}–${seconds(range.end)} s is outside the ${seconds(duration)} s video.`,
|
||||
);
|
||||
}
|
||||
const end = Math.min(range.end, duration);
|
||||
if (end - range.start < MIN_SEGMENT_SECONDS) {
|
||||
throw new Error(
|
||||
`${op} ${seconds(range.start)}–${seconds(end)} s is shorter than ${MIN_SEGMENT_SECONDS} s.`,
|
||||
);
|
||||
}
|
||||
return { start: range.start, end };
|
||||
});
|
||||
clamped.sort((a, b) => a.start - b.start);
|
||||
const merged: Segment[] = [];
|
||||
for (const segment of clamped) {
|
||||
const last = merged.at(-1);
|
||||
if (last && segment.start <= last.end) last.end = Math.max(last.end, segment.end);
|
||||
else merged.push({ ...segment });
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/** Keeps sorted, disjoint segments of the timeline: one trim, or split + trims + concat. */
|
||||
function keepSegments(graph: FilterGraph, timeline: Timeline, segments: readonly Segment[]) {
|
||||
const [only] = segments;
|
||||
if (segments.length === 1 && only) {
|
||||
const range = `start=${ffmpegSeconds(only.start)}:end=${ffmpegSeconds(only.end)}`;
|
||||
timeline.video?.filters.push(`trim=${range}`, "setpts=PTS-STARTPTS");
|
||||
timeline.audio?.filters.push(`atrim=${range}`, "asetpts=PTS-STARTPTS");
|
||||
timeline.seconds = only.end - only.start;
|
||||
return;
|
||||
}
|
||||
const count = segments.length;
|
||||
const branch = (track: Track | null, kind: "v" | "a"): string[] => {
|
||||
if (!track) return [];
|
||||
const input = graph.flush(track, kind);
|
||||
const splits = segments.map(() => graph.label(kind));
|
||||
graph.add([input], [`${kind === "v" ? "split" : "asplit"}=${count}`], splits);
|
||||
return segments.map((segment, index) => {
|
||||
const output = graph.label(kind);
|
||||
const range = `start=${ffmpegSeconds(segment.start)}:end=${ffmpegSeconds(segment.end)}`;
|
||||
graph.add(
|
||||
[splits[index]!],
|
||||
kind === "v"
|
||||
? [`trim=${range}`, "setpts=PTS-STARTPTS"]
|
||||
: [`atrim=${range}`, "asetpts=PTS-STARTPTS"],
|
||||
[output],
|
||||
);
|
||||
return output;
|
||||
});
|
||||
};
|
||||
const videoParts = branch(timeline.video, "v");
|
||||
const audioParts = branch(timeline.audio, "a");
|
||||
concatParts(graph, timeline, videoParts, audioParts, count);
|
||||
timeline.seconds = segments.reduce((total, segment) => total + segment.end - segment.start, 0);
|
||||
}
|
||||
|
||||
/** Appends clips normalized to the timeline: frame size, fit, fps and 48 kHz stereo audio. */
|
||||
function appendClips(
|
||||
graph: FilterGraph,
|
||||
timeline: Timeline,
|
||||
clips: readonly { clip: PlanInput; index: number }[],
|
||||
) {
|
||||
const fps = timeline.fps || FALLBACK_FPS;
|
||||
const needsAudio = timeline.audio !== null || clips.some(({ clip }) => clip.probe.hasAudio);
|
||||
const videoParts: string[] = [];
|
||||
const audioParts: string[] = [];
|
||||
if (timeline.video) {
|
||||
timeline.video.filters.push("setsar=1", "format=yuv420p");
|
||||
videoParts.push(graph.flush(timeline.video, "v"));
|
||||
}
|
||||
if (needsAudio) audioParts.push(audioPart(graph, timeline.audio, timeline.seconds));
|
||||
let seconds = timeline.seconds;
|
||||
for (const { clip, index } of clips) {
|
||||
const video = clip.probe.video;
|
||||
if (!video) throw new Error(`${clip.name} has no video.`);
|
||||
const clipSeconds = clip.probe.durationMs / 1000;
|
||||
if (clipSeconds <= 0) throw new Error(`Could not read the length of ${clip.name}.`);
|
||||
if (timeline.video) {
|
||||
const track = fitFrame(
|
||||
graph,
|
||||
{ label: `${index}:v:0`, filters: sourceFilters(video) },
|
||||
displaySize(video),
|
||||
timeline,
|
||||
timeline.fit,
|
||||
);
|
||||
track.filters.push(`fps=${number(fps)}`, "format=yuv420p");
|
||||
videoParts.push(graph.flush(track, "v"));
|
||||
}
|
||||
if (needsAudio) {
|
||||
const track = clip.probe.hasAudio ? { label: `${index}:a:0`, filters: [] } : null;
|
||||
audioParts.push(audioPart(graph, track, clipSeconds));
|
||||
}
|
||||
seconds += clipSeconds;
|
||||
}
|
||||
concatParts(graph, timeline, videoParts, audioParts, clips.length + 1);
|
||||
timeline.seconds = seconds;
|
||||
}
|
||||
|
||||
/** A clip's audio in the common format, or silence as long as the clip when it has none. */
|
||||
function audioPart(graph: FilterGraph, track: Track | null, seconds: number): string {
|
||||
if (track) {
|
||||
track.filters.push(AUDIO_FORMAT);
|
||||
return graph.flush(track, "a");
|
||||
}
|
||||
const output = graph.label("a");
|
||||
graph.add(
|
||||
[],
|
||||
[
|
||||
"anullsrc=channel_layout=stereo:sample_rate=48000",
|
||||
`atrim=end=${ffmpegSeconds(seconds)}`,
|
||||
AUDIO_FORMAT,
|
||||
],
|
||||
[output],
|
||||
);
|
||||
return output;
|
||||
}
|
||||
|
||||
function concatParts(
|
||||
graph: FilterGraph,
|
||||
timeline: Timeline,
|
||||
videoParts: readonly string[],
|
||||
audioParts: readonly string[],
|
||||
count: number,
|
||||
) {
|
||||
const inputs = Array.from({ length: count }, (_, index) => [
|
||||
...(videoParts[index] ? [videoParts[index]] : []),
|
||||
...(audioParts[index] ? [audioParts[index]] : []),
|
||||
]).flat();
|
||||
const video = videoParts.length > 0 ? graph.label("v") : undefined;
|
||||
const audio = audioParts.length > 0 ? graph.label("a") : undefined;
|
||||
graph.add(
|
||||
inputs,
|
||||
[`concat=n=${count}:v=${video ? 1 : 0}:a=${audio ? 1 : 0}`],
|
||||
[...(video ? [video] : []), ...(audio ? [audio] : [])],
|
||||
);
|
||||
timeline.video = video ? { label: video, filters: [] } : null;
|
||||
timeline.audio = audio ? { label: audio, filters: [] } : null;
|
||||
}
|
||||
|
||||
interface Size {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/** Scales a frame of size `from` into `to`: center crop, bars, or a blurred copy behind it. */
|
||||
function fitFrame(graph: FilterGraph, track: Track, from: Size, to: Size, fit: MediaFit): Track {
|
||||
const { width, height } = to;
|
||||
if (Math.abs(from.width / from.height - width / height) < 0.01) {
|
||||
track.filters.push(`scale=${width}:${height}`, "setsar=1");
|
||||
return track;
|
||||
}
|
||||
const contain = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`;
|
||||
if (fit === "crop") {
|
||||
track.filters.push(
|
||||
`scale=${width}:${height}:force_original_aspect_ratio=increase`,
|
||||
`crop=${width}:${height}`,
|
||||
"setsar=1",
|
||||
);
|
||||
return track;
|
||||
}
|
||||
if (fit === "pad") {
|
||||
track.filters.push(
|
||||
contain,
|
||||
`pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`,
|
||||
"setsar=1",
|
||||
);
|
||||
return track;
|
||||
}
|
||||
// Blur a small copy and scale it up: as smooth as a large blur at a fraction of the cost.
|
||||
const input = graph.flush(track, "v");
|
||||
const [background, foreground, blurred, contained, output] = [
|
||||
graph.label("v"),
|
||||
graph.label("v"),
|
||||
graph.label("v"),
|
||||
graph.label("v"),
|
||||
graph.label("v"),
|
||||
];
|
||||
const smallWidth = even(width / 4);
|
||||
const smallHeight = even(height / 4);
|
||||
const radius = Math.max(2, Math.floor(Math.min(smallWidth, smallHeight) / 10));
|
||||
graph.add([input], ["split=2"], [background, foreground]);
|
||||
graph.add(
|
||||
[background],
|
||||
[
|
||||
`scale=${smallWidth}:${smallHeight}:force_original_aspect_ratio=increase`,
|
||||
`crop=${smallWidth}:${smallHeight}`,
|
||||
`boxblur=${radius}:2`,
|
||||
`scale=${width}:${height}`,
|
||||
"setsar=1",
|
||||
],
|
||||
[blurred],
|
||||
);
|
||||
graph.add([foreground], [contain, "setsar=1"], [contained]);
|
||||
graph.add([blurred, contained], ["overlay=(W-w)/2:(H-h)/2"], [output]);
|
||||
return { label: output, filters: [] };
|
||||
}
|
||||
|
||||
/** Square pixels and even dimensions, which yuv420p H.264 requires. */
|
||||
function sourceFilters(video: NonNullable<MediaProbe["video"]>): string[] {
|
||||
const size = displaySize(video);
|
||||
if (Math.abs(video.sampleAspectRatio - 1) > 0.01) {
|
||||
return [`scale=${size.width}:${size.height}`, "setsar=1"];
|
||||
}
|
||||
if (size.width !== video.width || size.height !== video.height) {
|
||||
return [`crop=${size.width}:${size.height}:0:0`];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function displaySize(video: NonNullable<MediaProbe["video"]>): Size {
|
||||
if (Math.abs(video.sampleAspectRatio - 1) > 0.01) {
|
||||
return {
|
||||
width: even(Math.round(video.width * video.sampleAspectRatio)),
|
||||
height: even(video.height),
|
||||
};
|
||||
}
|
||||
return { width: even(video.width), height: even(video.height) };
|
||||
}
|
||||
|
||||
/** atempo takes 0.5–2 per instance on older ffmpeg builds, so larger changes chain it. */
|
||||
export function atempoFilters(factor: number): string[] {
|
||||
const filters: string[] = [];
|
||||
let rest = factor;
|
||||
while (rest > 2) {
|
||||
filters.push("atempo=2");
|
||||
rest /= 2;
|
||||
}
|
||||
while (rest < 0.5) {
|
||||
filters.push("atempo=0.5");
|
||||
rest /= 0.5;
|
||||
}
|
||||
filters.push(`atempo=${number(rest)}`);
|
||||
return filters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Readable burned-in subtitles: bold white text with a dark outline, sized to the frame, and
|
||||
* lifted above the bottom fifth of tall frames where 9:16 apps place their own controls.
|
||||
*/
|
||||
export function subtitleStyle(width: number, height: number): string {
|
||||
const tall = height >= width * 1.5;
|
||||
const fontSize = Math.round(Math.min(width, height) * 0.06);
|
||||
const marginX = Math.round(width * (tall ? 0.1 : 0.05));
|
||||
const marginY = Math.round(height * (tall ? 0.2 : 0.06));
|
||||
return [
|
||||
`PlayResX=${width}`,
|
||||
`PlayResY=${height}`,
|
||||
"FontName=DejaVu Sans",
|
||||
`FontSize=${fontSize}`,
|
||||
"Bold=1",
|
||||
"PrimaryColour=&H00FFFFFF",
|
||||
"OutlineColour=&H00000000",
|
||||
"BackColour=&H80000000",
|
||||
"BorderStyle=1",
|
||||
`Outline=${Math.max(2, Math.round(fontSize / 16))}`,
|
||||
"Shadow=1",
|
||||
"Alignment=2",
|
||||
`MarginL=${marginX}`,
|
||||
`MarginR=${marginX}`,
|
||||
`MarginV=${marginY}`,
|
||||
].join(",");
|
||||
}
|
||||
|
||||
function even(value: number): number {
|
||||
return Math.max(2, 2 * Math.floor(value / 2));
|
||||
}
|
||||
|
||||
function number(value: number): string {
|
||||
return String(Number(value.toFixed(6)));
|
||||
}
|
||||
|
||||
function seconds(value: number): string {
|
||||
return String(Number(value.toFixed(2)));
|
||||
}
|
||||
214
packages/adapters/src/media/video-operations.postgres.test.ts
Normal file
214
packages/adapters/src/media/video-operations.postgres.test.ts
Normal file
@ -0,0 +1,214 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { JobPublisher } from "@rakazo/adapter-kit";
|
||||
import type { MediaJobBlock, MessageBlock } from "@rakazo/contracts";
|
||||
import type { PrismaClient } from "@rakazo/db";
|
||||
import { createDb } from "@rakazo/db";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { storeArtifactFromFile } from "../artifact-files.js";
|
||||
import { LocalArtifactStore } from "../artifacts.js";
|
||||
import { createMediaRuntime, MediaJobRejectedError } from "../media-runtime.js";
|
||||
import { detectFfmpeg } from "./ffmpeg.js";
|
||||
import type { TestClips } from "./media-test-support.js";
|
||||
import { createTestClips } from "./media-test-support.js";
|
||||
import { createVideoOperations } from "./video-operations.js";
|
||||
|
||||
// CI runners may not have ffmpeg; the suite runs wherever both it and Postgres are available.
|
||||
const describePostgres =
|
||||
process.env.VERIFY_DATABASE && process.env.DATABASE_URL && (await detectFfmpeg())
|
||||
? describe.sequential
|
||||
: describe.skip;
|
||||
|
||||
describePostgres("video operations in the media runtime (PostgreSQL + ffmpeg)", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let prisma: PrismaClient;
|
||||
let root: string;
|
||||
let clips: TestClips;
|
||||
const users: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
db = createDb(process.env.DATABASE_URL!);
|
||||
prisma = db.prisma;
|
||||
root = await mkdtemp(join(tmpdir(), "video-operations-pg-"));
|
||||
clips = await createTestClips();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (!db) return;
|
||||
await prisma.mediaJob.deleteMany({ where: { userId: { in: users } } });
|
||||
await prisma.artifact.deleteMany({ where: { userId: { in: users } } });
|
||||
await prisma.organization.deleteMany({ where: { id: { in: users } } });
|
||||
await prisma.user.deleteMany({ where: { id: { in: users } } });
|
||||
await prisma.$disconnect();
|
||||
await db.pool.end();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
await rm(clips.dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function setup() {
|
||||
const id = randomUUID();
|
||||
users.push(id);
|
||||
await prisma.user.create({
|
||||
data: { id, name: "Video test", email: `${id}@example.test`, emailVerified: true },
|
||||
});
|
||||
await prisma.organization.create({
|
||||
data: { id, name: "Video test", slug: id, createdAt: new Date() },
|
||||
});
|
||||
await prisma.space.create({ data: { id, organizationId: id, name: "Media", isDefault: true } });
|
||||
await prisma.member.create({
|
||||
data: { id, organizationId: id, userId: id, role: "owner", createdAt: new Date() },
|
||||
});
|
||||
const bot = await prisma.bot.create({
|
||||
data: { spaceId: id, userId: id, name: "Studio", color: "test-color" },
|
||||
});
|
||||
const thread = await prisma.thread.create({ data: { spaceId: id, userId: id, botId: bot.id } });
|
||||
const jobs: JobPublisher = {
|
||||
enqueue: async () => undefined,
|
||||
cancel: async () => undefined,
|
||||
close: async () => undefined,
|
||||
};
|
||||
const artifacts = new LocalArtifactStore(join(root, id, "artifacts"));
|
||||
const runtime = createMediaRuntime(
|
||||
{
|
||||
prisma,
|
||||
events: { notify: async () => undefined },
|
||||
jobs,
|
||||
artifacts,
|
||||
registrations: createVideoOperations(),
|
||||
},
|
||||
{ tempRoot: join(root, id, "jobs") },
|
||||
);
|
||||
const scope = {
|
||||
spaceId: id,
|
||||
userId: id,
|
||||
botId: bot.id,
|
||||
groupId: null,
|
||||
threadId: thread.id,
|
||||
runId: null,
|
||||
};
|
||||
const store = async (path: string, name: string, mimeType: string) =>
|
||||
(
|
||||
await storeArtifactFromFile(
|
||||
{ prisma, artifacts },
|
||||
{
|
||||
path,
|
||||
name,
|
||||
mimeType,
|
||||
spaceId: id,
|
||||
userId: id,
|
||||
botId: bot.id,
|
||||
groupId: null,
|
||||
runId: null,
|
||||
},
|
||||
)
|
||||
).id;
|
||||
return { runtime, scope, store };
|
||||
}
|
||||
|
||||
async function card(messageId: string, jobId: string): Promise<MediaJobBlock | undefined> {
|
||||
const message = await prisma.message.findUniqueOrThrow({ where: { id: messageId } });
|
||||
return (message.blocks as MessageBlock[]).find(
|
||||
(block): block is MediaJobBlock => block.kind === "media" && block.jobId === jobId,
|
||||
);
|
||||
}
|
||||
|
||||
it("edits a stored video into a linked 9:16 version with a poster on the card", async () => {
|
||||
const { runtime, scope, store } = await setup();
|
||||
const source = await store(clips.landscape, "holiday.mp4", "video/mp4");
|
||||
const subtitles = await store(clips.srt, "holiday.srt", "application/x-subrip");
|
||||
const started = await runtime.start({
|
||||
...scope,
|
||||
operation: "edit_video",
|
||||
input: {
|
||||
artifact_id: source,
|
||||
title: "Urlaub 9:16",
|
||||
steps: [
|
||||
{ op: "trim", start: 0.5, end: 2.5 },
|
||||
{ op: "reframe", preset: "vertical", fit: "blur" },
|
||||
{ op: "subtitles", artifact_id: subtitles },
|
||||
],
|
||||
},
|
||||
});
|
||||
await runtime.runJob(started.jobId);
|
||||
const job = await runtime.get(started.jobId);
|
||||
expect(job).toMatchObject({ status: "done", title: "Urlaub 9:16", progress: 1 });
|
||||
const [output] = job!.outputs;
|
||||
expect(output).toMatchObject({
|
||||
kind: "video",
|
||||
mimeType: "video/mp4",
|
||||
name: "Urlaub 9 16.mp4",
|
||||
width: 1080,
|
||||
height: 1920,
|
||||
parentArtifactId: source,
|
||||
});
|
||||
expect(output?.posterArtifactId).toBeDefined();
|
||||
const poster = await prisma.artifact.findUniqueOrThrow({
|
||||
where: { id: output!.posterArtifactId! },
|
||||
});
|
||||
expect(poster.mimeType).toBe("image/jpeg");
|
||||
const stored = await prisma.artifact.findUniqueOrThrow({ where: { id: output!.artifactId } });
|
||||
expect(stored).toMatchObject({ parentArtifactId: source, width: 1080, height: 1920 });
|
||||
expect(Math.abs((stored.durationMs ?? 0) - 2000)).toBeLessThan(150);
|
||||
expect(await card(started.messageId, started.jobId)).toMatchObject({
|
||||
status: "done",
|
||||
outputs: job?.outputs,
|
||||
});
|
||||
});
|
||||
|
||||
it("exports one stored version per preset", async () => {
|
||||
const { runtime, scope, store } = await setup();
|
||||
const source = await store(clips.landscape, "holiday.mp4", "video/mp4");
|
||||
const started = await runtime.start({
|
||||
...scope,
|
||||
operation: "export_video",
|
||||
input: { artifact_id: source, presets: ["square", "portrait"] },
|
||||
});
|
||||
await runtime.runJob(started.jobId);
|
||||
const job = await runtime.get(started.jobId);
|
||||
expect(job).toMatchObject({ status: "done", title: "1:1 · 4:5" });
|
||||
expect(
|
||||
job?.outputs.map((output) => [output.name, output.preset, output.width, output.height]),
|
||||
).toEqual([
|
||||
["holiday-1x1.mp4", "square", 1080, 1080],
|
||||
["holiday-4x5.mp4", "portrait", 1080, 1350],
|
||||
]);
|
||||
});
|
||||
|
||||
it("refuses appended clips and subtitles the user cannot read", async () => {
|
||||
const { runtime, scope, store } = await setup();
|
||||
const other = await setup();
|
||||
const source = await store(clips.landscape, "mine.mp4", "video/mp4");
|
||||
const foreignClip = await other.store(clips.silentPortrait, "theirs.mp4", "video/mp4");
|
||||
const foreignSubtitles = await other.store(clips.srt, "theirs.srt", "application/x-subrip");
|
||||
for (const step of [
|
||||
{ op: "concat", artifact_ids: [foreignClip] },
|
||||
{ op: "subtitles", artifact_id: foreignSubtitles },
|
||||
]) {
|
||||
await expect(
|
||||
runtime.start({
|
||||
...scope,
|
||||
operation: "edit_video",
|
||||
input: { artifact_id: source, title: "x", steps: [step] },
|
||||
}),
|
||||
).rejects.toThrow(MediaJobRejectedError);
|
||||
}
|
||||
});
|
||||
|
||||
it("fails the job with a readable error when a range is outside the video", async () => {
|
||||
const { runtime, scope, store } = await setup();
|
||||
const source = await store(clips.landscape, "short.mp4", "video/mp4");
|
||||
const started = await runtime.start({
|
||||
...scope,
|
||||
operation: "edit_video",
|
||||
input: { artifact_id: source, title: "x", steps: [{ op: "trim", start: 10, end: 20 }] },
|
||||
});
|
||||
await runtime.runJob(started.jobId);
|
||||
expect(await runtime.get(started.jobId)).toMatchObject({
|
||||
status: "failed",
|
||||
error: "trim 10–20 s is outside the 3 s video.",
|
||||
outputs: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
289
packages/adapters/src/media/video-operations.test.ts
Normal file
289
packages/adapters/src/media/video-operations.test.ts
Normal file
@ -0,0 +1,289 @@
|
||||
import { rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { builtinAgentTools } from "../builtin-tools.js";
|
||||
import { createMediaOperations } from "../media-operations.js";
|
||||
import { MEDIA_CONTINUE_AFTER_PROPERTY } from "../media-tools.js";
|
||||
import { parametersFor } from "../pi-runtime.js";
|
||||
import { detectFfmpeg, probeMedia, runFfmpeg } from "./ffmpeg.js";
|
||||
import type { TestArtifact, TestClips } from "./media-test-support.js";
|
||||
import { createTestClips, fakeMediaJobContext } from "./media-test-support.js";
|
||||
import { cleanFileName, createVideoOperations, subtitleFormat } from "./video-operations.js";
|
||||
|
||||
const ffmpeg = await detectFfmpeg();
|
||||
|
||||
function operation(name: "edit_video" | "export_video", deps = {}) {
|
||||
const registration = createVideoOperations(deps).find((entry) => entry.operation === name);
|
||||
if (!registration) throw new Error(`missing ${name}`);
|
||||
return registration;
|
||||
}
|
||||
|
||||
describe("video operation registrations", () => {
|
||||
it("are registered for the runtime and unavailable without ffmpeg", async () => {
|
||||
const names = createMediaOperations({ loadSettings: async () => ({}) as never }).map(
|
||||
(entry) => entry.operation,
|
||||
);
|
||||
expect(names).toEqual(expect.arrayContaining(["edit_video", "export_video"]));
|
||||
const edit = operation("edit_video", { detect: async () => null });
|
||||
expect(await edit.available({ spaceId: "space-1" })).toBe(false);
|
||||
const { context } = await fakeMediaJobContext({}, {});
|
||||
await expect(edit.handler.run(context)).rejects.toThrow(
|
||||
"ffmpeg and ffprobe are not installed on this server.",
|
||||
);
|
||||
});
|
||||
|
||||
it("names every artifact the input reads so start() can authorize them", () => {
|
||||
const edit = operation("edit_video");
|
||||
const input = edit.inputSchema.parse({
|
||||
artifact_id: "src",
|
||||
title: "Reel",
|
||||
steps: [
|
||||
{ op: "concat", artifact_ids: ["clip-1", "clip-2"] },
|
||||
{ op: "subtitles", artifact_id: "subs" },
|
||||
],
|
||||
});
|
||||
expect(edit.artifactIds?.(input)).toEqual(["src", "clip-1", "clip-2", "subs"]);
|
||||
const exported = operation("export_video");
|
||||
const exportInput = exported.inputSchema.parse({ artifact_id: "src", presets: ["vertical"] });
|
||||
expect(exported.artifactIds?.(exportInput)).toEqual(["src"]);
|
||||
});
|
||||
|
||||
it("titles exports by aspect ratio unless the bot gave a title", () => {
|
||||
const exported = operation("export_video");
|
||||
expect(
|
||||
exported.title(
|
||||
exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical", "square"] }),
|
||||
),
|
||||
).toBe("9:16 · 1:1");
|
||||
expect(
|
||||
exported.title(
|
||||
exported.inputSchema.parse({ artifact_id: "a", presets: ["vertical"], title: "Reel" }),
|
||||
),
|
||||
).toBe("Reel");
|
||||
});
|
||||
|
||||
it("defines tools named like the operations, with continue_after, that the runtime can encode", () => {
|
||||
for (const name of ["edit_video", "export_video"]) {
|
||||
const tool = builtinAgentTools.find((entry) => entry.name === name);
|
||||
expect(tool).toBeDefined();
|
||||
expect(tool?.inputSchema.properties).toMatchObject(MEDIA_CONTINUE_AFTER_PROPERTY);
|
||||
expect(() => parametersFor(tool!)).not.toThrow();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("subtitle and file name helpers", () => {
|
||||
it("recognizes SRT and VTT by content only", () => {
|
||||
expect(subtitleFormat("WEBVTT\n\n00:00.000 --> 00:01.000\nHi")).toBe("vtt");
|
||||
expect(subtitleFormat("1\r\n00:00:01,000 --> 00:00:02,000\r\nHi")).toBe("srt");
|
||||
expect(subtitleFormat("00:00:01.000 --> 00:00:02.000\nHi")).toBe("srt");
|
||||
expect(subtitleFormat("#EXTM3U\n#EXTINF:1,\nfile:/etc/passwd")).toBeNull();
|
||||
expect(subtitleFormat("WEBVTTX")).toBeNull();
|
||||
});
|
||||
|
||||
it("cleans file names without losing the user's language", () => {
|
||||
expect(cleanFileName("Reel: Sommer/Urlaub 2026", "video")).toBe("Reel Sommer Urlaub 2026");
|
||||
expect(cleanFileName("../..\u0000", "video")).toBe("video");
|
||||
expect(cleanFileName("x".repeat(200), "video")).toHaveLength(80);
|
||||
});
|
||||
});
|
||||
|
||||
// CI runners may not have ffmpeg; these run wherever it is installed.
|
||||
describe.skipIf(!ffmpeg)("video operations with ffmpeg", () => {
|
||||
let clips: TestClips;
|
||||
let artifacts: Record<string, TestArtifact>;
|
||||
|
||||
beforeAll(async () => {
|
||||
clips = await createTestClips();
|
||||
artifacts = {
|
||||
land: { path: clips.landscape, name: "Landscape clip.mp4", mimeType: "video/mp4" },
|
||||
silent: { path: clips.silentPortrait, name: "silent.mp4", mimeType: "video/mp4" },
|
||||
srt: { path: clips.srt, name: "captions.srt", mimeType: "application/x-subrip" },
|
||||
vtt: { path: clips.vtt, name: "captions.vtt", mimeType: "text/vtt" },
|
||||
notes: { path: clips.landscape, name: "notes.txt", mimeType: "text/plain" },
|
||||
};
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (clips) await rm(clips.dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function run(name: "edit_video" | "export_video", input: Record<string, unknown>) {
|
||||
const registration = operation(name);
|
||||
const job = await fakeMediaJobContext(registration.inputSchema.parse(input), artifacts);
|
||||
try {
|
||||
const result = await registration.handler.run(job.context);
|
||||
return { ...job, result };
|
||||
} finally {
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it("cuts, reframes to 9:16 with a blurred fill, burns in subtitles and speeds up", async () => {
|
||||
const { outputs, progress, result } = await run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "Sommer-Reel",
|
||||
steps: [
|
||||
{ op: "trim", start: 0.5, end: 2.5 },
|
||||
{ op: "reframe", preset: "vertical", fit: "blur" },
|
||||
{ op: "subtitles", artifact_id: "srt" },
|
||||
{ op: "speed", factor: 2 },
|
||||
{ op: "poster", at: 0.5 },
|
||||
],
|
||||
});
|
||||
expect(outputs).toHaveLength(1);
|
||||
const [output] = outputs;
|
||||
expect(output).toMatchObject({
|
||||
name: "Sommer-Reel.mp4",
|
||||
mimeType: "video/mp4",
|
||||
width: 1080,
|
||||
height: 1920,
|
||||
parentArtifactId: "land",
|
||||
});
|
||||
expect(output!.posterSize).toBeGreaterThan(0);
|
||||
expect(Math.abs(output!.durationMs! - 1000)).toBeLessThan(150);
|
||||
expect(progress.at(-1)).toBe(1);
|
||||
expect(progress).toEqual([...progress].sort((a, b) => a - b));
|
||||
expect(result.summary).toContain("Sommer-Reel.mp4 (1080×1920, ");
|
||||
});
|
||||
|
||||
it("keeps segments, appends a silent clip with generated silence, and changes volume", async () => {
|
||||
const registration = operation("edit_video");
|
||||
const job = await fakeMediaJobContext(
|
||||
registration.inputSchema.parse({
|
||||
artifact_id: "land",
|
||||
title: "Cut",
|
||||
steps: [
|
||||
{
|
||||
op: "keep",
|
||||
segments: [
|
||||
{ start: 0, end: 0.5 },
|
||||
{ start: 2, end: 2.8 },
|
||||
],
|
||||
},
|
||||
{ op: "concat", artifact_ids: ["silent"] },
|
||||
{ op: "volume", gain: 0.5 },
|
||||
{ op: "subtitles", artifact_id: "vtt" },
|
||||
],
|
||||
}),
|
||||
artifacts,
|
||||
);
|
||||
await registration.handler.run(job.context);
|
||||
const [output] = job.outputs;
|
||||
const probe = await probeMedia(output!.path, { mimeType: "video/mp4" });
|
||||
expect(probe.hasAudio).toBe(true);
|
||||
expect(probe.video).toMatchObject({ width: 320, height: 180, rotation: 0 });
|
||||
expect(Math.abs(probe.durationMs - 3300)).toBeLessThan(200);
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("plays rotated phone video upright without rotating it twice", async (context) => {
|
||||
const rotated = join(clips.dir, "rotated.mp4");
|
||||
try {
|
||||
await runFfmpeg([
|
||||
...["-display_rotation", "90", "-i", `file:${clips.landscape}`],
|
||||
...["-c", "copy", `file:${rotated}`],
|
||||
]);
|
||||
} catch {
|
||||
context.skip("this ffmpeg cannot write rotation metadata");
|
||||
}
|
||||
const registration = operation("edit_video");
|
||||
const job = await fakeMediaJobContext(
|
||||
registration.inputSchema.parse({
|
||||
artifact_id: "rotated",
|
||||
title: "Upright",
|
||||
steps: [{ op: "trim", start: 0, end: 1 }],
|
||||
}),
|
||||
{ rotated: { path: rotated, name: "rotated.mp4", mimeType: "video/mp4" } },
|
||||
);
|
||||
await registration.handler.run(job.context);
|
||||
const probe = await probeMedia(job.outputs[0]!.path, { mimeType: "video/mp4" });
|
||||
expect(probe.video).toMatchObject({ width: 180, height: 320, rotation: 0 });
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("extracts the audio as M4A", async () => {
|
||||
const { outputs } = await run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "Ton",
|
||||
steps: [{ op: "trim", start: 1, end: 2 }, { op: "extract_audio" }],
|
||||
});
|
||||
expect(outputs).toEqual([
|
||||
expect.objectContaining({ name: "Ton.m4a", mimeType: "audio/mp4", parentArtifactId: "land" }),
|
||||
]);
|
||||
expect(outputs[0]?.posterPath).toBeUndefined();
|
||||
expect(Math.abs(outputs[0]!.durationMs! - 1000)).toBeLessThan(100);
|
||||
});
|
||||
|
||||
it("saves a single frame when poster is the only step", async () => {
|
||||
const { outputs } = await run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "Cover",
|
||||
steps: [{ op: "poster", at: 1 }],
|
||||
});
|
||||
expect(outputs).toEqual([
|
||||
expect.objectContaining({
|
||||
name: "Cover.jpg",
|
||||
mimeType: "image/jpeg",
|
||||
width: 320,
|
||||
height: 180,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("exports one output per distinct preset with posters and batch progress", async () => {
|
||||
const { outputs, progress } = await run("export_video", {
|
||||
artifact_id: "land",
|
||||
presets: ["square", "portrait", "square"],
|
||||
fit: "pad",
|
||||
});
|
||||
expect(
|
||||
outputs.map((output) => [output.name, output.preset, output.width, output.height]),
|
||||
).toEqual([
|
||||
["Landscape clip-1x1.mp4", "square", 1080, 1080],
|
||||
["Landscape clip-4x5.mp4", "portrait", 1080, 1350],
|
||||
]);
|
||||
for (const output of outputs) {
|
||||
expect(output.parentArtifactId).toBe("land");
|
||||
expect(output.posterSize).toBeGreaterThan(0);
|
||||
}
|
||||
expect(progress).toContain(0.5);
|
||||
expect(progress.at(-1)).toBe(1);
|
||||
});
|
||||
|
||||
it("fails clearly on ranges past the end, non-subtitle files and non-videos", async () => {
|
||||
await expect(
|
||||
run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "x",
|
||||
steps: [{ op: "trim", start: 5, end: 6 }],
|
||||
}),
|
||||
).rejects.toThrow("trim 5–6 s is outside the 3 s video.");
|
||||
await expect(
|
||||
run("edit_video", {
|
||||
artifact_id: "land",
|
||||
title: "x",
|
||||
steps: [{ op: "subtitles", artifact_id: "land" }],
|
||||
}),
|
||||
).rejects.toThrow("Landscape clip.mp4 is not an SRT or VTT file.");
|
||||
await expect(
|
||||
run("export_video", { artifact_id: "notes", presets: ["square"] }),
|
||||
).rejects.toThrow("notes.txt is not a video.");
|
||||
});
|
||||
|
||||
it("stops ffmpeg and stores nothing when the job is cancelled", async () => {
|
||||
const registration = operation("export_video");
|
||||
const job = await fakeMediaJobContext(
|
||||
registration.inputSchema.parse({ artifact_id: "land", presets: ["cinema"], fit: "blur" }),
|
||||
artifacts,
|
||||
);
|
||||
const progress = job.context.progress;
|
||||
job.context.progress = async (fraction) => {
|
||||
await progress(fraction);
|
||||
job.controller.abort(new Error("cancelled"));
|
||||
};
|
||||
await expect(registration.handler.run(job.context)).rejects.toThrow("cancelled");
|
||||
expect(job.outputs).toEqual([]);
|
||||
await rm(job.tempDir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
324
packages/adapters/src/media/video-operations.ts
Normal file
324
packages/adapters/src/media/video-operations.ts
Normal file
@ -0,0 +1,324 @@
|
||||
import { copyFile, mkdtemp, open, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { MediaJobContext, MediaJobInputFile, MediaOperationResult } from "@rakazo/adapter-kit";
|
||||
import type { EditVideoInput, ExportVideoInput, MediaOutput } from "@rakazo/contracts";
|
||||
import {
|
||||
ARTIFACT_MAX_BYTES_BY_KIND,
|
||||
EditVideoInputSchema,
|
||||
ExportVideoInputSchema,
|
||||
MEDIA_PRESETS,
|
||||
} from "@rakazo/contracts";
|
||||
import type { MediaOperationRegistration } from "../media-runtime.js";
|
||||
import type { FfmpegInfo } from "./ffmpeg.js";
|
||||
import {
|
||||
detectFfmpeg,
|
||||
outputTooLargeMessage,
|
||||
probeMedia,
|
||||
renderPoster,
|
||||
runFfmpeg,
|
||||
} from "./ffmpeg.js";
|
||||
import type { PlanInput, VideoEditPlan } from "./video-edit-plan.js";
|
||||
import { planVideoEdit } from "./video-edit-plan.js";
|
||||
|
||||
export interface VideoOperationsDeps {
|
||||
/** Finds ffmpeg; defaults to PATH lookup. */
|
||||
detect?: () => Promise<FfmpegInfo | null>;
|
||||
}
|
||||
|
||||
/** Share of each output's progress spent encoding; the rest is probing and the poster. */
|
||||
const ENCODE_SHARE = 0.95;
|
||||
const SUBTITLE_MAX_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
/** `edit_video` and `export_video`, available wherever ffmpeg and ffprobe are installed. */
|
||||
export function createVideoOperations(
|
||||
deps: VideoOperationsDeps = {},
|
||||
): MediaOperationRegistration[] {
|
||||
const detect = deps.detect ?? detectFfmpeg;
|
||||
const available = async () => (await detect()) !== null;
|
||||
const requireTools = async () => {
|
||||
if (!(await available()))
|
||||
throw new Error("ffmpeg and ffprobe are not installed on this server.");
|
||||
};
|
||||
const editVideo: MediaOperationRegistration<EditVideoInput> = {
|
||||
operation: "edit_video",
|
||||
inputSchema: EditVideoInputSchema,
|
||||
title: (input) => input.title,
|
||||
available,
|
||||
artifactIds: (input) => [
|
||||
input.artifact_id,
|
||||
...input.steps.flatMap((step) =>
|
||||
step.op === "concat"
|
||||
? step.artifact_ids
|
||||
: step.op === "subtitles"
|
||||
? [step.artifact_id]
|
||||
: [],
|
||||
),
|
||||
],
|
||||
handler: {
|
||||
operation: "edit_video",
|
||||
run: async (context) => {
|
||||
await requireTools();
|
||||
return runEditVideo(context);
|
||||
},
|
||||
},
|
||||
};
|
||||
const exportVideo: MediaOperationRegistration<ExportVideoInput> = {
|
||||
operation: "export_video",
|
||||
inputSchema: ExportVideoInputSchema,
|
||||
title: (input) =>
|
||||
input.title ?? input.presets.map((preset) => MEDIA_PRESETS[preset].aspect).join(" · "),
|
||||
available,
|
||||
artifactIds: (input) => [input.artifact_id],
|
||||
handler: {
|
||||
operation: "export_video",
|
||||
run: async (context) => {
|
||||
await requireTools();
|
||||
return runExportVideo(context);
|
||||
},
|
||||
},
|
||||
};
|
||||
return [editVideo, exportVideo];
|
||||
}
|
||||
|
||||
async function runEditVideo(context: MediaJobContext): Promise<MediaOperationResult> {
|
||||
const input = EditVideoInputSchema.parse(context.input);
|
||||
const workDir = await mkdtemp(join(context.tempDir, "edit-"));
|
||||
const progress = createProgressReporter(context);
|
||||
const source = await loadVideo(context, input.artifact_id);
|
||||
const name = cleanFileName(input.title, cleanFileName(withoutExtension(source.name), "video"));
|
||||
|
||||
const [first] = input.steps;
|
||||
if (input.steps.length === 1 && first?.op === "poster") {
|
||||
const at = first.at;
|
||||
if (at * 1000 >= source.probe.durationMs) {
|
||||
throw new Error(
|
||||
`poster at ${at} s is past the end of the ${source.probe.durationMs / 1000} s video.`,
|
||||
);
|
||||
}
|
||||
const posterPath = join(workDir, "poster.jpg");
|
||||
await renderPoster(source.path, posterPath, {
|
||||
signal: context.signal,
|
||||
mimeType: source.mimeType,
|
||||
atMs: at * 1000,
|
||||
});
|
||||
const output = await context.addOutput({
|
||||
path: posterPath,
|
||||
name: `${name}.jpg`,
|
||||
mimeType: "image/jpeg",
|
||||
width: source.probe.video?.width,
|
||||
height: source.probe.video?.height,
|
||||
parentArtifactId: source.artifactId,
|
||||
});
|
||||
await progress.done();
|
||||
return { summary: `Saved the frame at ${at} s of ${source.name} as ${describe(output)}.` };
|
||||
}
|
||||
|
||||
const clips = new Map<string, PlanInput>();
|
||||
const subtitleFiles = new Map<string, string>();
|
||||
for (const step of input.steps) {
|
||||
if (step.op === "concat") {
|
||||
for (const id of step.artifact_ids) {
|
||||
if (!clips.has(id)) clips.set(id, await loadVideo(context, id));
|
||||
}
|
||||
} else if (step.op === "subtitles" && !subtitleFiles.has(step.artifact_id)) {
|
||||
subtitleFiles.set(
|
||||
step.artifact_id,
|
||||
await prepareSubtitles(context, step.artifact_id, workDir, subtitleFiles.size),
|
||||
);
|
||||
}
|
||||
}
|
||||
const audioOnly = input.steps.at(-1)?.op === "extract_audio";
|
||||
const outputPath = join(workDir, audioOnly ? "output.m4a" : "output.mp4");
|
||||
const plan = planVideoEdit({ source, steps: input.steps, clips, subtitleFiles, outputPath });
|
||||
const output = await encode(context, plan, outputPath, workDir, progress.range(0, 1), {
|
||||
name: `${name}${audioOnly ? ".m4a" : ".mp4"}`,
|
||||
parentArtifactId: source.artifactId,
|
||||
});
|
||||
await progress.done();
|
||||
return {
|
||||
summary: `Edited ${source.name} into ${describe(output)}.`,
|
||||
};
|
||||
}
|
||||
|
||||
async function runExportVideo(context: MediaJobContext): Promise<MediaOperationResult> {
|
||||
const input = ExportVideoInputSchema.parse(context.input);
|
||||
const presets = [...new Set(input.presets)];
|
||||
const workDir = await mkdtemp(join(context.tempDir, "export-"));
|
||||
const progress = createProgressReporter(context);
|
||||
const source = await loadVideo(context, input.artifact_id);
|
||||
const base = cleanFileName(withoutExtension(source.name), "video");
|
||||
const outputs: MediaOutput[] = [];
|
||||
for (const [index, preset] of presets.entries()) {
|
||||
const outputPath = join(workDir, `${preset}.mp4`);
|
||||
const plan = planVideoEdit({
|
||||
source,
|
||||
steps: [{ op: "reframe", preset, fit: input.fit }],
|
||||
outputPath,
|
||||
});
|
||||
const share = 1 / presets.length;
|
||||
outputs.push(
|
||||
await encode(
|
||||
context,
|
||||
plan,
|
||||
outputPath,
|
||||
workDir,
|
||||
progress.range(index * share, (index + 1) * share),
|
||||
{
|
||||
name: `${base}-${MEDIA_PRESETS[preset].aspect.replace(":", "x")}.mp4`,
|
||||
parentArtifactId: source.artifactId,
|
||||
preset,
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
await progress.done();
|
||||
return { summary: `Exported ${source.name} as ${outputs.map(describe).join(", ")}.` };
|
||||
}
|
||||
|
||||
/** Runs a plan, checks the result, renders its poster and stores it as an output. */
|
||||
async function encode(
|
||||
context: MediaJobContext,
|
||||
plan: VideoEditPlan,
|
||||
outputPath: string,
|
||||
workDir: string,
|
||||
report: (fraction: number) => void,
|
||||
file: Pick<Parameters<MediaJobContext["addOutput"]>[0], "name" | "parentArtifactId" | "preset">,
|
||||
): Promise<MediaOutput> {
|
||||
const maxBytes = ARTIFACT_MAX_BYTES_BY_KIND[plan.kind];
|
||||
await runFfmpeg(plan.args, {
|
||||
signal: context.signal,
|
||||
cwd: workDir,
|
||||
durationMs: plan.durationMs,
|
||||
onProgress: (fraction) => report(fraction * ENCODE_SHARE),
|
||||
maxOutputBytes: maxBytes,
|
||||
outputs: [outputPath],
|
||||
});
|
||||
if ((await stat(outputPath)).size > maxBytes) throw new Error(outputTooLargeMessage(maxBytes));
|
||||
const probe = await probeMedia(outputPath, {
|
||||
signal: context.signal,
|
||||
mimeType: plan.kind === "audio" ? "audio/mp4" : "video/mp4",
|
||||
});
|
||||
if (plan.kind === "audio") {
|
||||
const output = await context.addOutput({
|
||||
...file,
|
||||
path: outputPath,
|
||||
mimeType: "audio/mp4",
|
||||
durationMs: probe.durationMs,
|
||||
});
|
||||
report(1);
|
||||
return output;
|
||||
}
|
||||
const posterPath = outputPath.replace(/\.mp4$/, ".jpg");
|
||||
await renderPoster(outputPath, posterPath, {
|
||||
signal: context.signal,
|
||||
mimeType: "video/mp4",
|
||||
atMs: plan.posterAtMs ?? Math.min(1_000, probe.durationMs / 4),
|
||||
});
|
||||
const output = await context.addOutput({
|
||||
...file,
|
||||
path: outputPath,
|
||||
mimeType: "video/mp4",
|
||||
width: probe.video?.width ?? plan.width,
|
||||
height: probe.video?.height ?? plan.height,
|
||||
durationMs: probe.durationMs,
|
||||
posterPath,
|
||||
});
|
||||
report(1);
|
||||
return output;
|
||||
}
|
||||
|
||||
async function loadVideo(
|
||||
context: MediaJobContext,
|
||||
artifactId: string,
|
||||
): Promise<PlanInput & MediaJobInputFile> {
|
||||
const file = await context.fetchArtifact(artifactId);
|
||||
if (file.kind !== "video") throw new Error(`${file.name} is not a video.`);
|
||||
const probe = await probeMedia(file.path, {
|
||||
signal: context.signal,
|
||||
mimeType: file.mimeType,
|
||||
});
|
||||
if (!probe.video) throw new Error(`${file.name} has no video.`);
|
||||
if (probe.durationMs <= 0) throw new Error(`Could not read the length of ${file.name}.`);
|
||||
return { ...file, probe };
|
||||
}
|
||||
|
||||
/** Copies an SRT/VTT artifact to a generated name, so no user text reaches the filter graph. */
|
||||
async function prepareSubtitles(
|
||||
context: MediaJobContext,
|
||||
artifactId: string,
|
||||
workDir: string,
|
||||
index: number,
|
||||
): Promise<string> {
|
||||
const file = await context.fetchArtifact(artifactId);
|
||||
const format = file.size <= SUBTITLE_MAX_BYTES ? subtitleFormat(await readHead(file.path)) : null;
|
||||
if (!format) throw new Error(`${file.name} is not an SRT or VTT file.`);
|
||||
const name = `subtitles-${index}.${format}`;
|
||||
await copyFile(file.path, join(workDir, name));
|
||||
return name;
|
||||
}
|
||||
|
||||
async function readHead(path: string): Promise<string> {
|
||||
const handle = await open(path, "r");
|
||||
try {
|
||||
const buffer = Buffer.alloc(1024);
|
||||
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
|
||||
return buffer.subarray(0, bytesRead).toString("utf8");
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognizes WebVTT and SubRip by content. Anything else is refused, so ffmpeg never probes
|
||||
* an arbitrary file (a playlist, say) handed in as subtitles.
|
||||
*/
|
||||
export function subtitleFormat(head: string): "srt" | "vtt" | null {
|
||||
const text = head.replace(/^/, "").trimStart();
|
||||
if (/^WEBVTT(?:[ \t\r\n]|$)/.test(text)) return "vtt";
|
||||
if (/^(?:\d+[ \t]*\r?\n)?\d{1,2}:\d{2}:\d{2}[,.]\d{1,3}[ \t]*-->/.test(text)) return "srt";
|
||||
return null;
|
||||
}
|
||||
|
||||
/** A name for produced files: no path separators or control characters, at most 80 chars. */
|
||||
export function cleanFileName(text: string, fallback: string): string {
|
||||
const name = text
|
||||
.replaceAll(/[\p{Cc}\\/:*?"<>|]+/gu, " ")
|
||||
.replaceAll(/\s+/g, " ")
|
||||
.replace(/^[\s.]+/, "")
|
||||
.slice(0, 80)
|
||||
.trim();
|
||||
return name || fallback;
|
||||
}
|
||||
|
||||
function withoutExtension(name: string): string {
|
||||
return name.replace(/\.[A-Za-z0-9]{1,5}$/, "");
|
||||
}
|
||||
|
||||
/** "name (1080×1920, 12.5 s)" for the bot's summary; the runtime adds the artifact ids. */
|
||||
function describe(output: MediaOutput): string {
|
||||
const details = [
|
||||
output.width && output.height ? `${output.width}×${output.height}` : "",
|
||||
output.durationMs !== undefined ? `${Number((output.durationMs / 1000).toFixed(1))} s` : "",
|
||||
].filter(Boolean);
|
||||
return details.length ? `${output.name} (${details.join(", ")})` : output.name;
|
||||
}
|
||||
|
||||
/** Maps ffmpeg progress onto the card: monotonic, written in order, never failing the job. */
|
||||
function createProgressReporter(context: MediaJobContext) {
|
||||
let reported = 0;
|
||||
let writes = Promise.resolve();
|
||||
const write = (fraction: number) => {
|
||||
reported = fraction;
|
||||
writes = writes.then(() => context.progress(fraction)).catch(() => undefined);
|
||||
};
|
||||
return {
|
||||
range: (from: number, to: number) => (fraction: number) => {
|
||||
const value = Math.min(1, from + (to - from) * Math.min(1, Math.max(0, fraction)));
|
||||
if (value > reported) write(value);
|
||||
},
|
||||
done: async () => {
|
||||
if (reported < 1) write(1);
|
||||
await writes;
|
||||
},
|
||||
};
|
||||
}
|
||||
@ -14,6 +14,7 @@ export * from "./media-generation.js";
|
||||
export * from "./media-jobs.js";
|
||||
export * from "./media-settings.js";
|
||||
export * from "./media-transcription.js";
|
||||
export * from "./media-video.js";
|
||||
export * from "./model-tiers.js";
|
||||
export * from "./openai-compatible-ui.js";
|
||||
export * from "./project-goals.js";
|
||||
|
||||
56
packages/contracts/src/media-video.test.ts
Normal file
56
packages/contracts/src/media-video.test.ts
Normal file
@ -0,0 +1,56 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { EditVideoInputSchema, ExportVideoInputSchema } from "./media-video.js";
|
||||
|
||||
const edit = (steps: unknown[]) =>
|
||||
EditVideoInputSchema.safeParse({ artifact_id: "a", title: "Reel", steps });
|
||||
|
||||
describe("edit_video input", () => {
|
||||
it("accepts ordered steps and fills defaults", () => {
|
||||
const parsed = EditVideoInputSchema.parse({
|
||||
artifact_id: "a",
|
||||
title: " Reel ",
|
||||
steps: [
|
||||
{ op: "trim", start: 1, end: 5 },
|
||||
{ op: "reframe", preset: "vertical" },
|
||||
{ op: "extract_audio" },
|
||||
],
|
||||
});
|
||||
expect(parsed.title).toBe("Reel");
|
||||
expect(parsed.steps[1]).toEqual({ op: "reframe", preset: "vertical", fit: "crop" });
|
||||
// The job stores the parsed input and parses it again when it runs.
|
||||
expect(EditVideoInputSchema.parse(parsed)).toEqual(parsed);
|
||||
});
|
||||
|
||||
it("rejects malformed step lists", () => {
|
||||
expect(edit([]).success).toBe(false);
|
||||
expect(edit([{ op: "trim", start: 5, end: 5 }]).success).toBe(false);
|
||||
expect(edit([{ op: "keep", segments: [{ start: 3, end: 1 }] }]).success).toBe(false);
|
||||
expect(edit([{ op: "speed", factor: 8 }]).success).toBe(false);
|
||||
expect(edit([{ op: "extract_audio" }, { op: "mute" }]).success).toBe(false);
|
||||
expect(edit([{ op: "poster", at: 1 }, { op: "extract_audio" }]).success).toBe(false);
|
||||
expect(
|
||||
edit([
|
||||
{ op: "poster", at: 1 },
|
||||
{ op: "poster", at: 2 },
|
||||
]).success,
|
||||
).toBe(false);
|
||||
expect(edit([{ op: "blur" }]).success).toBe(false);
|
||||
expect(
|
||||
EditVideoInputSchema.safeParse({ artifact_id: "a", steps: [{ op: "mute" }] }).success,
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("export_video input", () => {
|
||||
it("needs at least one preset and defaults to cropping", () => {
|
||||
expect(ExportVideoInputSchema.parse({ artifact_id: "a", presets: ["square"] })).toEqual({
|
||||
artifact_id: "a",
|
||||
presets: ["square"],
|
||||
fit: "crop",
|
||||
});
|
||||
expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: [] }).success).toBe(false);
|
||||
expect(ExportVideoInputSchema.safeParse({ artifact_id: "a", presets: ["imax"] }).success).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
80
packages/contracts/src/media-video.ts
Normal file
80
packages/contracts/src/media-video.ts
Normal file
@ -0,0 +1,80 @@
|
||||
import * as z from "zod";
|
||||
import { Id } from "./ids.js";
|
||||
import { MediaFitSchema, MediaPresetIdSchema } from "./media.js";
|
||||
|
||||
/**
|
||||
* Inputs of the `edit_video` and `export_video` media operations, which are also the arguments
|
||||
* of the tools with those names. Times are seconds on the timeline left by the previous steps.
|
||||
*/
|
||||
|
||||
const Seconds = z.number().min(0).max(86_400);
|
||||
|
||||
const TimeRangeSchema = z
|
||||
.object({ start: Seconds, end: Seconds })
|
||||
.refine((range) => range.end > range.start, "end must be after start");
|
||||
|
||||
/** Written by the bot in the user's language; shown on the media card and used as file name. */
|
||||
const MediaTitleSchema = z.string().trim().min(1).max(120);
|
||||
|
||||
export const EditVideoStepSchema = z.discriminatedUnion("op", [
|
||||
z
|
||||
.object({ op: z.literal("trim"), start: Seconds, end: Seconds })
|
||||
.refine((step) => step.end > step.start, "end must be after start"),
|
||||
/** Keeps these parts and cuts everything between them. */
|
||||
z.object({ op: z.literal("keep"), segments: z.array(TimeRangeSchema).min(1).max(50) }),
|
||||
/** Appends these videos, fitted to the current frame. */
|
||||
z.object({ op: z.literal("concat"), artifact_ids: z.array(Id).min(1).max(20) }),
|
||||
z.object({
|
||||
op: z.literal("reframe"),
|
||||
preset: MediaPresetIdSchema,
|
||||
fit: MediaFitSchema.default("crop"),
|
||||
}),
|
||||
z.object({ op: z.literal("speed"), factor: z.number().min(0.25).max(4) }),
|
||||
/** Linear gain: 0.5 halves the loudness, 2 doubles it. */
|
||||
z.object({ op: z.literal("volume"), gain: z.number().min(0).max(8) }),
|
||||
z.object({ op: z.literal("mute") }),
|
||||
/** Burns in an SRT or VTT artifact. */
|
||||
z.object({ op: z.literal("subtitles"), artifact_id: Id }),
|
||||
/** Last step only: the result is the audio as M4A. */
|
||||
z.object({ op: z.literal("extract_audio") }),
|
||||
/** Cover frame at this second of the result; as the only kind of step, the result is that image. */
|
||||
z.object({ op: z.literal("poster"), at: Seconds }),
|
||||
]);
|
||||
export type EditVideoStep = z.infer<typeof EditVideoStepSchema>;
|
||||
|
||||
export const EditVideoInputSchema = z
|
||||
.object({
|
||||
artifact_id: Id,
|
||||
steps: z.array(EditVideoStepSchema).min(1).max(20),
|
||||
title: MediaTitleSchema,
|
||||
})
|
||||
.superRefine((input, context) => {
|
||||
const ops = input.steps.map((step) => step.op);
|
||||
const extract = ops.indexOf("extract_audio");
|
||||
if (extract >= 0 && extract !== ops.length - 1) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["steps", extract],
|
||||
message: "extract_audio must be the last step",
|
||||
});
|
||||
}
|
||||
if (extract >= 0 && ops.includes("poster")) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["steps"],
|
||||
message: "poster cannot be combined with extract_audio",
|
||||
});
|
||||
}
|
||||
if (ops.filter((op) => op === "poster").length > 1) {
|
||||
context.addIssue({ code: "custom", path: ["steps"], message: "use at most one poster step" });
|
||||
}
|
||||
});
|
||||
export type EditVideoInput = z.infer<typeof EditVideoInputSchema>;
|
||||
|
||||
export const ExportVideoInputSchema = z.object({
|
||||
artifact_id: Id,
|
||||
presets: z.array(MediaPresetIdSchema).min(1).max(6),
|
||||
fit: MediaFitSchema.default("crop"),
|
||||
title: MediaTitleSchema.optional(),
|
||||
});
|
||||
export type ExportVideoInput = z.infer<typeof ExportVideoInputSchema>;
|
||||
@ -86,6 +86,7 @@ describe("foreground AI consent", () => {
|
||||
expect(aiDataUsesForProcedure("threads/get")).toEqual([]);
|
||||
expect(aiDataUsesForProcedure("threads/send")).toEqual(["model", "memory"]);
|
||||
expect(aiDataUsesForProcedure("voice/prepare")).toEqual(["voice"]);
|
||||
expect(aiDataUsesForProcedure("media/jobs/start")).toEqual(["model"]);
|
||||
expect(
|
||||
aiDataUsesForProcedure("routines/update", { routineId: "routine", active: false }),
|
||||
).toEqual([]);
|
||||
|
||||
@ -32,6 +32,7 @@ export function aiDataUsesForProcedure(procedure: string, input?: unknown): AiDa
|
||||
].includes(path)
|
||||
)
|
||||
return ["model", "memory"];
|
||||
if (path === "media/jobs/start") return ["model"];
|
||||
if (["voice/prepare", "voice/speak", "voice/transcribe"].includes(path)) return ["voice"];
|
||||
return [];
|
||||
}
|
||||
|
||||
@ -120,6 +120,7 @@ async function main() {
|
||||
"packages/adapters/src/cloud-agent.postgres.test.ts",
|
||||
"packages/adapters/src/media/image-operations.postgres.test.ts",
|
||||
"packages/adapters/src/media/transcription.postgres.test.ts",
|
||||
"packages/adapters/src/media/video-operations.postgres.test.ts",
|
||||
"packages/adapters/src/media-runtime.postgres.test.ts",
|
||||
"packages/adapters/src/builtin-memory-provider.postgres.test.ts",
|
||||
"packages/adapters/src/profile-consolidation.postgres.test.ts",
|
||||
|
||||
@ -2,13 +2,10 @@ import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { ComposioEmulator, type MediaOperationRegistration } from "@rakazo/adapters";
|
||||
import {
|
||||
type MediaJobBlock,
|
||||
MediaOutputSchema,
|
||||
type MediaSettingsView,
|
||||
type MessageBlock,
|
||||
} from "@rakazo/contracts";
|
||||
import type { MediaOperationRegistration } from "@rakazo/adapters";
|
||||
import { ComposioEmulator } from "@rakazo/adapters";
|
||||
import type { MediaJobBlock, MediaSettingsView, MessageBlock } from "@rakazo/contracts";
|
||||
import { MediaOutputSchema } from "@rakazo/contracts";
|
||||
import { createThreadMessage } from "@rakazo/db";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { createApp } from "../../../apps/api/src/app.ts";
|
||||
@ -154,7 +151,7 @@ describeWithDatabase("media RPCs (PostgreSQL)", () => {
|
||||
baseUrl: "http://127.0.0.1:8188/",
|
||||
apiKey: "comfy-test-key",
|
||||
imageModel: "sd_xl_base_1.0.safetensors",
|
||||
imageWorkflow: '{"3":{"class_type":"KSampler","inputs":{}}}',
|
||||
imageWorkflow: '{"6":{"class_type":"CLIPTextEncode","inputs":{"text":"{{prompt}}"}}}',
|
||||
videoWorkflow: "",
|
||||
},
|
||||
transcription: { baseUrl: "http://127.0.0.1:8080", model: "whisper-1", apiKey: "" },
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user