cynarAI/evOo-local
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

merge: W1-domain media domain logic

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0138YqDj3ttfayRwuMKKrqVj
This commit is contained in:
Claude 2026-10-03 18:06:39 +00:00
commit d43e374534
No known key found for this signature in database
16 changed files with 6915 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

491
packages/media/src/__snapshots__/compile.test.ts.snap Normal file
View File

@ -0,0 +1,491 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`compileRenderPlan > compiles 16:9 to 9:16 with auto-reframe, captions, watermark and loudnorm 1`] = `
{
"outputs": [
{
"kind": "video",
"path": "/work/render-1/output.mp4",
},
{
"kind": "subtitles_srt",
"path": "/work/render-1/captions.srt",
},
{
"kind": "subtitles_vtt",
"path": "/work/render-1/captions.vtt",
},
{
"kind": "poster",
"path": "/work/render-1/poster.jpg",
},
],
"quality": "final",
"steps": [
{
"content": "[Script Info]
ScriptType: v4.00+
PlayResX: 1080
PlayResY: 1920
WrapStyle: 2
ScaledBorderAndShadow: yes
[V4+ Styles]
Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
Style: Default,Example Sans,96,&H00FFFFFF,&H00FFFFFF,&H00000000,&H80000000,0,0,0,0,100,100,0,0,1,2,0,2,92,168,422,1
[Events]
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
Dialogue: 0,0:00:00.20,0:00:01.80,Default,,0,0,0,,Wait for this!
Dialogue: 0,0:00:02.10,0:00:03.40,Default,,0,0,0,,Here it <b>is</b> --> now.
",
"id": "captions-ass",
"kind": "write_file",
"path": "/work/render-1/captions.ass",
},
{
"content": "1
00:00:00,200 --> 00:00:01,800
Wait for this!
2
00:00:02,100 --> 00:00:03,400
Here it <b>is</b> -> now.
",
"id": "captions-srt",
"kind": "write_file",
"path": "/work/render-1/captions.srt",
},
{
"content": "WEBVTT
00:00:00.200 --> 00:00:01.800
Wait for this!
00:00:02.100 --> 00:00:03.400
Here it &lt;b&gt;is&lt;/b&gt; --&gt; now.
",
"id": "captions-vtt",
"kind": "write_file",
"path": "/work/render-1/captions.vtt",
},
{
"argv": [
"-hide_banner",
"-nostdin",
"-y",
"-ss",
"0",
"-t",
"2",
"-i",
"/media/asset-a/proxy.mp4",
"-ss",
"5",
"-t",
"2",
"-i",
"/media/asset-a/proxy.mp4",
"-i",
"/brand/watermark.png",
"-filter_complex",
"[0:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,170.82\\,if(lt(t\\,1.5)\\,170.82+(167.81)*(t-0)/1.5\\,if(lt(t\\,2)\\,338.62+(19.97)*(t-1.5)/0.5\\,358.59))):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,1.5)\\,0+(0)*(t-0)/1.5\\,if(lt(t\\,2)\\,0+(0)*(t-1.5)/0.5\\,0))),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,1232\\,if(lt(t\\,2)\\,1232+(0)*(t-0)/2\\,1232)):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,2)\\,0+(0)*(t-0)/2\\,0)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c2];[v_c1][a_c1][v_c2][a_c2]concat=n=2:v=1:a=1[cv1][ca2];[2:v]format=rgba,scale=162:-2,colorchannelmixer=aa=0.6[img3];[cv1][img3]overlay=x=main_w-overlay_w-152:y=214:enable=between(t\\,0\\,4)[ov4];[ov4]ass=filename=/work/render-1/captions.ass:fontsdir=/brand/fonts[cap5];[cap5]format=yuv420p[vout];[ca2]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]",
"-map",
"[vout]",
"-map",
"[aout]",
"-c:v",
"libx264",
"-preset",
"medium",
"-crf",
"20",
"-profile:v",
"high",
"-maxrate",
"8000k",
"-bufsize",
"16000k",
"-pix_fmt",
"yuv420p",
"-colorspace",
"bt709",
"-color_primaries",
"bt709",
"-color_trc",
"bt709",
"-color_range",
"tv",
"-bsf:v",
"filter_units=remove_types=6",
"-r",
"30",
"-c:a",
"aac",
"-b:a",
"192k",
"-ar",
"48000",
"-ac",
"2",
"-map_metadata",
"-1",
"-map_chapters",
"-1",
"-fflags",
"+bitexact",
"-flags:v",
"+bitexact",
"-flags:a",
"+bitexact",
"-metadata",
"title=Example: teaser",
"-t",
"4",
"-movflags",
"+faststart",
"/work/render-1/output.mp4",
],
"durationMs": 4000,
"id": "render",
"kind": "ffmpeg",
"produces": [
"/work/render-1/output.mp4",
],
},
{
"argv": [
"-hide_banner",
"-nostdin",
"-y",
"-ss",
"1.333",
"-i",
"/work/render-1/output.mp4",
"-frames:v",
"1",
"-map_metadata",
"-1",
"-fflags",
"+bitexact",
"-flags:v",
"+bitexact",
"-q:v",
"2",
"/work/render-1/poster.jpg",
],
"durationMs": 0,
"id": "poster",
"kind": "ffmpeg",
"produces": [
"/work/render-1/poster.jpg",
],
},
],
"workDir": "/work/render-1",
}
`;
exports[`compileRenderPlan > compiles transitions, music ducking, voice-over and fades 1`] = `
{
"outputs": [
{
"kind": "video",
"path": "/work/render-1/output.mp4",
},
{
"kind": "poster",
"path": "/work/render-1/poster.jpg",
},
],
"quality": "draft",
"steps": [
{
"argv": [
"-hide_banner",
"-nostdin",
"-y",
"-ss",
"0",
"-t",
"2",
"-i",
"/media/asset-a/proxy.mp4",
"-ss",
"2",
"-t",
"2",
"-i",
"/media/asset-a/proxy.mp4",
"-ss",
"3",
"-t",
"1",
"-i",
"/media/asset-a/proxy.mp4",
"-ss",
"4",
"-t",
"1",
"-i",
"/media/asset-b/original.mov",
"-ss",
"0",
"-t",
"8",
"-i",
"/brand/music/bed.wav",
"-ss",
"0",
"-t",
"2",
"-i",
"/work/render-1/voiceover.wav",
"-filter_complex",
"[0:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/0.5,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=4,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,atempo=0.5,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=4,asetpts=PTS-STARTPTS[a_c2];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c3];[2:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c3];[3:v]crop=w=608:h=1080:x=656:y=0,scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c4];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c4];[v_c1][v_c2]xfade=transition=fade:duration=0.5:offset=1.5[xv1];[a_c1][a_c2]acrossfade=d=0.5:c1=tri:c2=tri[xa2];color=c=black:s=1080x1920:r=30,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=0.5,setpts=PTS-STARTPTS[gv3];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=0.5,asetpts=PTS-STARTPTS[ga4];[xv1][xa2][gv3][ga4][v_c3][a_c3][v_c4][a_c4]concat=n=4:v=1:a=1[cv5][ca6];[cv5]format=yuv420p[vout];[4:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,volume=-8dB,afade=t=in:st=0:d=0.5,afade=t=out:st=7:d=1[ac_bed];[5:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,adelay=1000|1000[ac_vo];[ca6][ac_vo]amix=inputs=2:duration=first:normalize=0[speech];[speech]asplit=2[speech_mix][speech_key];[ac_bed]anull[duck_bus];[duck_bus][speech_key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400:makeup=1[ducked];[speech_mix][ducked]amix=inputs=2:duration=first:normalize=0[mixed];[mixed]loudnorm=I=-16:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]",
"-map",
"[vout]",
"-map",
"[aout]",
"-c:v",
"libx264",
"-preset",
"veryfast",
"-crf",
"28",
"-profile:v",
"high",
"-maxrate",
"3000k",
"-bufsize",
"6000k",
"-pix_fmt",
"yuv420p",
"-colorspace",
"bt709",
"-color_primaries",
"bt709",
"-color_trc",
"bt709",
"-color_range",
"tv",
"-bsf:v",
"filter_units=remove_types=6",
"-r",
"30",
"-c:a",
"aac",
"-b:a",
"128k",
"-ar",
"48000",
"-ac",
"2",
"-map_metadata",
"-1",
"-map_chapters",
"-1",
"-fflags",
"+bitexact",
"-flags:v",
"+bitexact",
"-flags:a",
"+bitexact",
"-t",
"8",
"-movflags",
"+faststart",
"/work/render-1/output.mp4",
],
"durationMs": 8000,
"id": "render",
"kind": "ffmpeg",
"produces": [
"/work/render-1/output.mp4",
],
},
{
"argv": [
"-hide_banner",
"-nostdin",
"-y",
"-ss",
"2.667",
"-i",
"/work/render-1/output.mp4",
"-frames:v",
"1",
"-map_metadata",
"-1",
"-fflags",
"+bitexact",
"-flags:v",
"+bitexact",
"-q:v",
"2",
"/work/render-1/poster.jpg",
],
"durationMs": 0,
"id": "poster",
"kind": "ffmpeg",
"produces": [
"/work/render-1/poster.jpg",
],
},
],
"workDir": "/work/render-1",
}
`;
exports[`compileRenderPlan > compiles voice disguise, metadata stripping, intro/outro and text overlays 1`] = `
{
"outputs": [
{
"kind": "video",
"path": "/work/render-1/output.mp4",
},
{
"kind": "poster",
"path": "/work/render-1/poster.jpg",
},
],
"quality": "final",
"steps": [
{
"fps": 30,
"height": 1920,
"id": "motion-1",
"kind": "motion",
"outPath": "/work/render-1/motion-1.mov",
"template": {
"durationMs": 1000,
"params": {},
"templateId": "outro-simple",
},
"width": 1080,
},
{
"fps": 30,
"height": 1920,
"id": "motion-2",
"kind": "motion",
"outPath": "/work/render-1/motion-2.mov",
"template": {
"durationMs": 1000,
"params": {},
"templateId": "lower-third-a",
},
"width": 1080,
},
{
"argv": [
"-hide_banner",
"-nostdin",
"-y",
"-ss",
"0",
"-t",
"3",
"-i",
"/media/asset-a/proxy.mp4",
"-i",
"/work/render-1/intro.mov",
"-i",
"/work/render-1/motion-1.mov",
"-i",
"/work/render-1/motion-2.mov",
"-filter_complex",
"[0:v]deshake,crop=w=406:h=720:x=if(lt(t\\,0)\\,373\\,if(lt(t\\,3)\\,373+(768)*(t-0)/3\\,1141)):y=if(lt(t\\,0)\\,180\\,if(lt(t\\,3)\\,180+(0)*(t-0)/3\\,180)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,eq=brightness=0.05:contrast=1:saturation=1.1,colorbalance=rm=0.02:bm=-0.02,lut3d=file=/brand/look\\\\\\'s.cube,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=3,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[a_c1];[a_c1]rubberband=pitch=0.793701:formant=preserved,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[vd1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1.5,setpts=PTS-STARTPTS[intro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1.5,asetpts=PTS-STARTPTS[intro_a];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[outro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[outro_a];[intro_v][intro_a][v_c1][vd1][outro_v][outro_a]concat=n=3:v=1:a=1[prog_v][prog_a];[3:v]format=yuva420p,scale=1080:1920,setpts=PTS-STARTPTS+0/TB[mo2];[prog_v][mo2]overlay=x=0:y=0:eof_action=pass:enable=between(t\\,0\\,1)[ov3];color=c=0xFFAA00:s=1080x10:r=30:d=5.5[pb4];[ov3][pb4]overlay=x=-overlay_w+main_w*min(1\\,(t-1.5)/3):y=1504:enable=between(t\\,1.5\\,4.5)[ov5];[ov5]drawtext=fontfile=/brand/fonts/ExampleSans.ttf:text=Ana\\\\\\'s\\\\: \\[studio\\]\\; 100\\\\\\\\% real:fontsize=77:fontcolor=0xFFFFFF:x=(w-text_w)/2:y=h-text_h-560:box=1:boxcolor=0x112233@0.85:boxborderw=31:enable=between(t\\,2\\,3.5)[ov6];[ov6]format=yuv420p[vout];[prog_a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]",
"-map",
"[vout]",
"-map",
"[aout]",
"-c:v",
"libx264",
"-preset",
"medium",
"-crf",
"20",
"-profile:v",
"high",
"-maxrate",
"8000k",
"-bufsize",
"16000k",
"-pix_fmt",
"yuv420p",
"-colorspace",
"bt709",
"-color_primaries",
"bt709",
"-color_trc",
"bt709",
"-color_range",
"tv",
"-bsf:v",
"filter_units=remove_types=6",
"-r",
"30",
"-c:a",
"aac",
"-b:a",
"192k",
"-ar",
"48000",
"-ac",
"2",
"-map_metadata",
"-1",
"-map_chapters",
"-1",
"-fflags",
"+bitexact",
"-flags:v",
"+bitexact",
"-flags:a",
"+bitexact",
"-t",
"5.5",
"-movflags",
"+faststart",
"/work/render-1/output.mp4",
],
"durationMs": 5500,
"id": "render",
"kind": "ffmpeg",
"produces": [
"/work/render-1/output.mp4",
],
},
{
"argv": [
"-hide_banner",
"-nostdin",
"-y",
"-ss",
"2.5",
"-i",
"/work/render-1/output.mp4",
"-frames:v",
"1",
"-map_metadata",
"-1",
"-fflags",
"+bitexact",
"-flags:v",
"+bitexact",
"-q:v",
"2",
"/work/render-1/poster.jpg",
],
"durationMs": 0,
"id": "poster",
"kind": "ffmpeg",
"produces": [
"/work/render-1/poster.jpg",
],
},
],
"workDir": "/work/render-1",
}
`;

206
packages/media/src/clustering.test.ts Normal file
View File

@ -0,0 +1,206 @@
import { describe, expect, it } from "vitest";
import type { ClusterAsset } from "./clustering.js";
import {
buildSessions,
calendarDay,
clusterFaces,
clusterThemes,
ruleLabel,
} from "./clustering.js";
const at = (iso: string) => new Date(iso);
function asset(
id: string,
captured: string | null,
extra: Partial<ClusterAsset> = {},
): ClusterAsset {
return {
id,
kind: "photo",
capturedAt: captured ? at(captured) : null,
gpsLat: null,
gpsLon: null,
deviceModel: null,
albumNames: [],
embedding: null,
...extra,
};
}
const vec = (...values: number[]) => new Float32Array(values);
describe("buildSessions", () => {
it("splits on gaps longer than 45 minutes and sorts by capture time", () => {
const sessions = buildSessions([
asset("c", "2026-05-01T10:40:00Z"),
asset("a", "2026-05-01T10:00:00Z"),
asset("b", "2026-05-01T10:20:00Z"),
asset("d", "2026-05-01T11:26:00Z"),
]);
expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]);
expect(sessions[0]!.key).toBe("session:a");
expect(sessions[0]!.startAt).toEqual(at("2026-05-01T10:00:00Z"));
expect(sessions[0]!.endAt).toEqual(at("2026-05-01T10:40:00Z"));
});
it("keeps a gap of exactly 45 minutes in one session", () => {
const sessions = buildSessions([
asset("a", "2026-05-01T10:00:00Z"),
asset("b", "2026-05-01T10:45:00Z"),
]);
expect(sessions).toHaveLength(1);
});
it("puts undated assets into one trailing session", () => {
const sessions = buildSessions([
asset("z", null),
asset("a", "2026-05-01T10:00:00Z"),
asset("m", null),
asset("bad", null, { capturedAt: new Date(Number.NaN) }),
]);
expect(sessions.map((s) => s.key)).toEqual(["session:a", "session:undated"]);
expect(sessions[1]!.assetIds).toEqual(["bad", "m", "z"]);
expect(sessions[1]!.startAt).toBeNull();
});
it("ignores duplicate ids and mixes photos and videos", () => {
const sessions = buildSessions([
asset("a", "2026-05-01T10:00:00Z"),
asset("v", "2026-05-01T10:05:00Z", { kind: "video" }),
asset("a", "2026-05-01T23:00:00Z"),
asset("l", "2026-05-01T10:06:00Z", { kind: "live_photo" }),
]);
expect(sessions.map((s) => s.assetIds)).toEqual([["a", "v", "l"]]);
});
it("splits on a location jump over 2 km and reports a rounded centroid", () => {
const sessions = buildSessions([
asset("a", "2026-05-01T10:00:00Z", { gpsLat: 10.0, gpsLon: 20.0 }),
asset("b", "2026-05-01T10:05:00Z"),
asset("c", "2026-05-01T10:10:00Z", { gpsLat: 10.004, gpsLon: 20.004 }),
asset("d", "2026-05-01T10:15:00Z", { gpsLat: 10.1, gpsLon: 20.0 }),
]);
expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]);
expect(sessions[0]!.centroid).toEqual({ lat: 10, lon: 20 });
expect(sessions[1]!.centroid).toEqual({ lat: 10.1, lon: 20 });
});
it("breaks on a calendar-day change only with a gap over 3 hours, in the given time zone", () => {
const assets = [asset("a", "2026-05-01T22:30:00Z"), asset("b", "2026-05-02T02:00:00Z")];
// Long gap rule disabled so only the day-break rule decides.
const options = { gapMinutes: 24 * 60 };
expect(buildSessions(assets, options)).toHaveLength(2);
// In UTC-10 both are on 1 May: one session.
expect(buildSessions(assets, { ...options, timeZone: "Pacific/Honolulu" })).toHaveLength(1);
// Crossing midnight with a short gap stays one session.
expect(
buildSessions(
[asset("x", "2026-05-01T23:50:00Z"), asset("y", "2026-05-02T00:20:00Z")],
options,
),
).toHaveLength(1);
expect(calendarDay(at("2026-05-01T22:30:00Z"), "Europe/Berlin")).toBe("2026-05-02");
});
it("merges neighbours with similar embeddings and a gap under 2 hours", () => {
const red = vec(1, 0, 0);
const blue = vec(0, 1, 0);
const sessions = buildSessions([
asset("a", "2026-05-01T10:00:00Z", { embedding: red }),
asset("b", "2026-05-01T11:00:00Z", { embedding: vec(0.95, 0.05, 0) }),
asset("c", "2026-05-01T12:00:00Z", { embedding: blue }),
asset("d", "2026-05-01T15:00:00Z", { embedding: blue }),
]);
expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b"], ["c"], ["d"]]);
});
it("is deterministic for any input order", () => {
const assets = [
asset("a", "2026-05-01T10:00:00Z"),
asset("b", "2026-05-01T10:00:00Z"),
asset("c", "2026-05-02T10:00:00Z"),
asset("d", null),
];
const forward = buildSessions(assets);
const reversed = buildSessions([...assets].reverse());
expect(reversed).toEqual(forward);
expect(forward[0]!.assetIds).toEqual(["a", "b"]);
});
});
describe("clusterThemes and clusterFaces", () => {
const items = [
{ id: "p3", vector: vec(0.9, 0.1, 0) },
{ id: "p1", vector: vec(1, 0, 0) },
{ id: "q1", vector: vec(0, 1, 0) },
{ id: "p2", vector: vec(0.95, 0.05, 0) },
{ id: "q2", vector: vec(0.05, 0.95, 0) },
{ id: "r1", vector: vec(0, 0, 1) },
];
it("groups by average linkage with stable keys, independent of input order", () => {
const themes = clusterThemes(items);
expect(themes).toEqual([
{ key: "theme:p1", ids: ["p1", "p2", "p3"] },
{ key: "theme:q1", ids: ["q1", "q2"] },
{ key: "theme:r1", ids: ["r1"] },
]);
expect(clusterThemes([...items].reverse())).toEqual(themes);
expect(clusterThemes(items, { minSize: 2 }).map((t) => t.key)).toEqual([
"theme:p1",
"theme:q1",
]);
expect(clusterThemes(items, { similarity: -1 })).toHaveLength(1);
expect(clusterThemes([])).toEqual([]);
});
it("uses average rather than single linkage", () => {
// b is close to a and c, but a and c are far apart: average linkage keeps c out.
const chain = [
{ id: "a", vector: vec(1, 0) },
{ id: "b", vector: vec(Math.cos(0.4), Math.sin(0.4)) },
{ id: "c", vector: vec(Math.cos(1.0), Math.sin(1.0)) },
];
expect(clusterThemes(chain, { similarity: 0.8 }).map((g) => g.ids)).toEqual([
["a", "b"],
["c"],
]);
});
it("returns face groups with a mean centroid", () => {
const faces = clusterFaces([
{ id: "t1", vector: vec(1, 0) },
{ id: "t2", vector: vec(0.8, 0.2) },
{ id: "t3", vector: vec(0, 1) },
]);
expect(faces.map((f) => f.key)).toEqual(["person:t1", "person:t3"]);
expect(Array.from(faces[0]!.centroid)).toEqual([0.8999999761581421, 0.10000000149011612]);
expect(() =>
clusterFaces([
{ id: "x", vector: vec(1) },
{ id: "y", vector: vec(1, 2) },
]),
).toThrow();
});
});
describe("ruleLabel", () => {
it("combines the dominant album with a locale-aware date range", () => {
const label = ruleLabel(
{
startAt: at("2026-05-01T10:00:00Z"),
endAt: at("2026-05-03T10:00:00Z"),
albumNames: ["Trip", "Favorites", "Trip"],
},
"en-US",
);
expect(label.startsWith("Trip · ")).toBe(true);
expect(label).toMatch(/May 1\D+3, 2026/);
expect(
ruleLabel({ startAt: at("2026-05-01T10:00:00Z"), endAt: null, albumNames: [] }, "de-DE"),
).toBe("01.05.2026");
expect(ruleLabel({ startAt: null, endAt: null, albumNames: [" "] })).toBe("");
expect(ruleLabel({ startAt: null, endAt: null, albumNames: ["B", "A"] })).toBe("A");
});
});

292
packages/media/src/clustering.ts
View File

@ -1,5 +1,5 @@
import type { MediaAssetKind } from "@rakazo/contracts"; import type { MediaAssetKind } from "@rakazo/contracts";
import { notImplemented } from "./not-implemented.js"; import { cosineSimilarity, haversineKm, roundCoordinate } from "./geo.js";
export interface ClusterAsset { export interface ClusterAsset {
id: string; id: string;
@ -46,15 +46,157 @@ export interface SessionCluster {
centroid: { lat: number; lon: number } | null; centroid: { lat: number; lon: number } | null;
} }
const MINUTE_MS = 60_000;
const HOUR_MS = 60 * MINUTE_MS;
const dayFormatters = new Map<string, Intl.DateTimeFormat>();
/** Calendar day (YYYY-MM-DD) of an instant in an IANA time zone. */
export function calendarDay(date: Date, timeZone: string): string {
let formatter = dayFormatters.get(timeZone);
if (!formatter) {
formatter = new Intl.DateTimeFormat("en-CA", {
timeZone,
year: "numeric",
month: "2-digit",
day: "2-digit",
});
dayFormatters.set(timeZone, formatter);
}
return formatter.format(date);
}
type DatedAsset = ClusterAsset & { capturedAt: Date };
type PositionedAsset = ClusterAsset & { gpsLat: number; gpsLon: number };
function hasPosition(asset: ClusterAsset): asset is PositionedAsset {
return (
asset.gpsLat !== null &&
asset.gpsLon !== null &&
Number.isFinite(asset.gpsLat) &&
Number.isFinite(asset.gpsLon)
);
}
function isDated(asset: ClusterAsset): asset is DatedAsset {
return asset.capturedAt !== null && Number.isFinite(asset.capturedAt.getTime());
}
/** Mean of the given vectors, or null when none are present or dimensions disagree. */
export function meanVector(vectors: ArrayLike<number>[]): Float32Array | null {
const first = vectors[0];
if (!first) return null;
const sum = new Float64Array(first.length);
for (const vector of vectors) {
if (vector.length !== first.length) return null;
for (let i = 0; i < vector.length; i++) sum[i] = sum[i]! + (vector[i] as number);
}
const mean = new Float32Array(first.length);
for (let i = 0; i < sum.length; i++) mean[i] = sum[i]! / vectors.length;
return mean;
}
function sessionEmbedding(assets: ClusterAsset[]): Float32Array | null {
return meanVector(assets.flatMap((asset) => (asset.embedding ? [asset.embedding] : [])));
}
function finishSession(assets: ClusterAsset[], key: string): SessionCluster {
const dated = assets.filter(isDated);
const positioned = assets.filter(hasPosition);
const mean = (values: number[]) => values.reduce((sum, v) => sum + v, 0) / values.length;
return {
key,
assetIds: assets.map((asset) => asset.id),
startAt: dated[0]?.capturedAt ?? null,
endAt: dated.at(-1)?.capturedAt ?? null,
centroid:
positioned.length === 0
? null
: {
lat: roundCoordinate(mean(positioned.map((a) => a.gpsLat))),
lon: roundCoordinate(mean(positioned.map((a) => a.gpsLon))),
},
};
}
/** /**
* Deterministic session grouping (spec 4.4). Assets without capturedAt form * Deterministic session grouping (spec 4.4). Assets without capturedAt form
* one trailing "undated" session; exact duplicates (same id) are ignored. * one trailing "undated" session; exact duplicates (same id) are ignored.
*
* A new session starts on a gap longer than gapMinutes, a jump of more than
* maxDistanceKm from the session's last known position, or a calendar-day
* change (in timeZone) with a gap longer than dayBreakHours. Adjacent sessions
* are then merged when their mean embeddings reach mergeSimilarity and the gap
* between them stays below mergeMaxGapHours. Keys derive from the earliest
* asset id, so they survive re-clustering as long as that asset stays first.
*/ */
export function buildSessions( export function buildSessions(
_assets: ClusterAsset[], assets: ClusterAsset[],
_options?: Partial<SessionOptions>, options?: Partial<SessionOptions>,
): SessionCluster[] { ): SessionCluster[] {
return notImplemented("buildSessions"); const opts = { ...DEFAULT_SESSION_OPTIONS, ...options };
const unique = new Map<string, ClusterAsset>();
for (const asset of assets) if (!unique.has(asset.id)) unique.set(asset.id, asset);
const dated = [...unique.values()]
.filter(isDated)
.sort((a, b) => a.capturedAt.getTime() - b.capturedAt.getTime() || a.id.localeCompare(b.id));
const undated = [...unique.values()]
.filter((asset) => !isDated(asset))
.sort((a, b) => a.id.localeCompare(b.id));
const split: DatedAsset[][] = [];
let lastPosition: { lat: number; lon: number } | null = null;
for (const asset of dated) {
const current = split.at(-1);
const previous = current?.at(-1);
let startNew = true;
if (current && previous) {
const gap = asset.capturedAt.getTime() - previous.capturedAt.getTime();
const jumped =
hasPosition(asset) &&
lastPosition !== null &&
haversineKm(lastPosition, { lat: asset.gpsLat, lon: asset.gpsLon }) > opts.maxDistanceKm;
const dayChanged =
gap > opts.dayBreakHours * HOUR_MS &&
calendarDay(asset.capturedAt, opts.timeZone) !==
calendarDay(previous.capturedAt, opts.timeZone);
startNew = gap > opts.gapMinutes * MINUTE_MS || jumped || dayChanged;
}
if (startNew || !current) {
split.push([asset]);
lastPosition = null;
} else {
current.push(asset);
}
if (hasPosition(asset)) lastPosition = { lat: asset.gpsLat, lon: asset.gpsLon };
}
// Merge visually similar neighbours that are close in time.
const merged: DatedAsset[][] = [];
for (const session of split) {
const last = merged.at(-1);
if (last) {
const gap = session[0]!.capturedAt.getTime() - last.at(-1)!.capturedAt.getTime();
const a = sessionEmbedding(last);
const b = sessionEmbedding(session);
if (
a &&
b &&
a.length === b.length &&
gap < opts.mergeMaxGapHours * HOUR_MS &&
cosineSimilarity(a, b) >= opts.mergeSimilarity
) {
last.push(...session);
continue;
}
}
merged.push([...session]);
}
const sessions = merged.map((group) => finishSession(group, `session:${group[0]!.id}`));
if (undated.length > 0) sessions.push(finishSession(undated, "session:undated"));
return sessions;
} }
export interface VectorItem { export interface VectorItem {
@ -62,26 +204,144 @@ export interface VectorItem {
vector: Float32Array; vector: Float32Array;
} }
/** Agglomerative average-linkage clustering; returns groups with stable keys. */ /**
* Average-linkage agglomerative clustering cut at a similarity threshold,
* using the nearest-neighbour chain algorithm (O(n²) time and memory). Items
* are sorted by id first and ties resolve to the lower index, so the result
* does not depend on input order. Callers batch very large libraries.
*/
function agglomerate(items: VectorItem[], threshold: number): VectorItem[][] {
const unique = new Map<string, VectorItem>();
for (const item of items) if (!unique.has(item.id)) unique.set(item.id, item);
const sorted = [...unique.values()].sort((a, b) => a.id.localeCompare(b.id));
const n = sorted.length;
if (n === 0) return [];
const dim = sorted[0]!.vector.length;
if (sorted.some((item) => item.vector.length !== dim)) throw new Error("vector length mismatch");
const sim = new Float64Array(n * n);
for (let i = 0; i < n; i++) {
for (let j = i + 1; j < n; j++) {
const value = cosineSimilarity(sorted[i]!.vector, sorted[j]!.vector);
sim[i * n + j] = value;
sim[j * n + i] = value;
}
}
const size = new Array<number>(n).fill(1);
const members: number[][] = sorted.map((_, i) => [i]);
const active = new Set<number>(members.map((_, i) => i));
const finished: number[][] = [];
const chain: number[] = [];
while (active.size > 0) {
if (chain.length === 0) chain.push(Math.min(...active));
const top = chain.at(-1)!;
const below = chain.length >= 2 ? chain.at(-2)! : -1;
let best = -1;
let bestSim = Number.NEGATIVE_INFINITY;
for (const j of active) {
if (j === top) continue;
const s = sim[top * n + j]!;
if (s > bestSim || (s === bestSim && j < best)) {
best = j;
bestSim = s;
}
}
// Prefer the chain predecessor on ties so the chain cannot cycle.
if (below >= 0 && sim[top * n + below]! === bestSim) best = below;
if (best < 0 || bestSim < threshold) {
// Average linkage is reducible: merges never raise similarity to this cluster again.
active.delete(top);
finished.push(members[top]!);
chain.length = 0;
continue;
}
if (best !== below) {
chain.push(best);
continue;
}
chain.length -= 2;
const a = Math.min(top, below);
const b = Math.max(top, below);
for (const k of active) {
if (k === a || k === b) continue;
const value =
(size[a]! * sim[a * n + k]! + size[b]! * sim[b * n + k]!) / (size[a]! + size[b]!);
sim[a * n + k] = value;
sim[k * n + a] = value;
}
size[a] = size[a]! + size[b]!;
members[a] = [...members[a]!, ...members[b]!];
active.delete(b);
}
return finished.map((group) => group.sort((x, y) => x - y).map((i) => sorted[i]!));
}
function sortGroups<T extends { key: string; ids: string[] }>(groups: T[]): T[] {
return groups.sort((a, b) => b.ids.length - a.ids.length || a.key.localeCompare(b.key));
}
/**
* Agglomerative average-linkage clustering; returns groups with stable keys
* (from the smallest member id), largest first. Default similarity 0.8.
*/
export function clusterThemes( export function clusterThemes(
_items: VectorItem[], items: VectorItem[],
_options?: { similarity?: number; minSize?: number }, options?: { similarity?: number; minSize?: number },
): Array<{ key: string; ids: string[] }> { ): Array<{ key: string; ids: string[] }> {
return notImplemented("clusterThemes"); const minSize = options?.minSize ?? 1;
return sortGroups(
agglomerate(items, options?.similarity ?? 0.8)
.filter((group) => group.length >= minSize)
.map((group) => ({ key: `theme:${group[0]!.id}`, ids: group.map((item) => item.id) })),
);
} }
/** Group face tracks into person suggestions by identity embedding. Local only. */ /** Group face tracks into person suggestions by identity embedding. Local only. Default similarity 0.6. */
export function clusterFaces( export function clusterFaces(
_tracks: VectorItem[], tracks: VectorItem[],
_options?: { similarity?: number }, options?: { similarity?: number },
): Array<{ key: string; ids: string[]; centroid: Float32Array }> { ): Array<{ key: string; ids: string[]; centroid: Float32Array }> {
return notImplemented("clusterFaces"); return sortGroups(
agglomerate(tracks, options?.similarity ?? 0.6).map((group) => ({
key: `person:${group[0]!.id}`,
ids: group.map((item) => item.id),
centroid: meanVector(group.map((item) => item.vector))!,
})),
);
} }
/** Rule-based title from date range, place centroid, and dominant album. No model involved. */ /**
* Rule-based title from the date range and dominant album. Places are not
* named because only rounded coordinates are stored. Returns "" when there is
* nothing to say, so the caller shows its own localized placeholder.
*/
export function ruleLabel( export function ruleLabel(
_cluster: { startAt: Date | null; endAt: Date | null; albumNames: string[] }, cluster: { startAt: Date | null; endAt: Date | null; albumNames: string[] },
_locale?: string, locale?: string,
timeZone = "UTC",
): string { ): string {
return notImplemented("ruleLabel"); const counts = new Map<string, number>();
for (const raw of cluster.albumNames) {
const name = raw.trim();
if (name) counts.set(name, (counts.get(name) ?? 0) + 1);
}
const album = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0];
let dates = "";
const start = cluster.startAt ?? cluster.endAt;
const end = cluster.endAt ?? cluster.startAt;
if (start && end) {
let formatter: Intl.DateTimeFormat;
try {
formatter = new Intl.DateTimeFormat(locale, { dateStyle: "medium", timeZone });
} catch {
formatter = new Intl.DateTimeFormat("en", { dateStyle: "medium", timeZone });
}
dates =
calendarDay(start, timeZone) === calendarDay(end, timeZone)
? formatter.format(start)
: formatter.formatRange(start, end);
}
return [album?.[0], dates].filter(Boolean).join(" · ");
} }

322
packages/media/src/compile.ffmpeg.test.ts Normal file
View File

@ -0,0 +1,322 @@
import { execFile, execFileSync } from "node:child_process";
import { existsSync } from "node:fs";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { promisify } from "node:util";
import type { RenderPlan } from "@rakazo/contracts";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import type { CompileInput } from "./compile.js";
import { compileRenderPlan, escapeFilterText, escapeFilterValue } from "./compile.js";
import { edlDurationMs } from "./edl.js";
import { findMetadataLeaks } from "./gate.js";
import { testEdl, testKit, testProfile } from "./test-fixtures.js";
const run = promisify(execFile);
function onPath(binary: string): boolean {
try {
execFileSync(binary, ["-version"], { stdio: "ignore" });
return true;
} catch {
return false;
}
}
const hasFfmpeg = onPath("ffmpeg") && onPath("ffprobe");
const hasRubberband =
hasFfmpeg &&
execFileSync("ffmpeg", ["-hide_banner", "-filters"]).toString().includes(" rubberband ");
async function ffmpeg(args: string[]): Promise<void> {
await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...args]);
}
async function execute(plan: RenderPlan): Promise<void> {
for (const step of plan.steps) {
if (step.kind === "write_file") await writeFile(step.path, step.content);
else if (step.kind === "ffmpeg") await run("ffmpeg", ["-loglevel", "error", ...step.argv]);
else throw new Error(`unexpected ${step.kind} step`);
}
}
interface Probe {
durationMs: number;
width: number;
height: number;
tags: Record<string, string>;
}
async function probe(file: string): Promise<Probe> {
const { stdout } = await run("ffprobe", [
"-v",
"error",
"-show_format",
"-show_streams",
"-of",
"json",
file,
]);
const json = JSON.parse(stdout) as {
format: { duration: string; tags?: Record<string, string> };
streams: Array<{
codec_type: string;
width?: number;
height?: number;
tags?: Record<string, string>;
}>;
};
const tags: Record<string, string> = {};
for (const [key, value] of Object.entries(json.format.tags ?? {})) tags[`format.${key}`] = value;
json.streams.forEach((stream, index) => {
for (const [key, value] of Object.entries(stream.tags ?? {}))
tags[`stream.${index}.${key}`] = value;
});
const video = json.streams.find((s) => s.codec_type === "video");
return {
durationMs: Number(json.format.duration) * 1000,
width: video?.width ?? 0,
height: video?.height ?? 0,
tags,
};
}
describe.skipIf(!hasFfmpeg)("compiled plans run on ffmpeg", () => {
let dir: string;
let source: string;
let watermark: string;
let music: string;
let intro: string;
let lut: string;
beforeAll(async () => {
dir = await mkdtemp(path.join(tmpdir(), "media-compile-"));
source = path.join(dir, "source.mp4");
watermark = path.join(dir, "watermark.png");
music = path.join(dir, "music.wav");
intro = path.join(dir, "intro.mov");
lut = path.join(dir, "brand look.cube");
// Identity 2x2x2 LUT.
const lattice = [];
for (const b of [0, 1])
for (const g of [0, 1]) for (const r of [0, 1]) lattice.push(`${r} ${g} ${b}`);
await writeFile(lut, `LUT_3D_SIZE 2\n${lattice.join("\n")}\n`);
await ffmpeg([
"-f",
"lavfi",
"-i",
"testsrc2=size=480x270:rate=30:duration=8",
"-f",
"lavfi",
"-i",
"sine=frequency=440:duration=8",
"-c:v",
"libx264",
"-preset",
"ultrafast",
"-c:a",
"aac",
"-shortest",
// Tags a phone would write; the export must drop every one of them.
"-metadata",
"creation_time=2020-01-01T10:00:00Z",
"-metadata",
"location=+00.0000+000.0000/",
"-metadata",
"make=ExampleCam",
source,
]);
await ffmpeg(["-f", "lavfi", "-i", "color=c=white:s=64x32", "-frames:v", "1", watermark]);
await ffmpeg(["-f", "lavfi", "-i", "sine=frequency=220:duration=6", music]);
await ffmpeg([
"-f",
"lavfi",
"-i",
"testsrc=size=320x180:rate=30:duration=1",
"-c:v",
"libx264",
"-preset",
"ultrafast",
intro,
]);
});
afterAll(async () => {
if (dir) await rm(dir, { recursive: true, force: true });
});
const baseInput = (overrides: Partial<CompileInput>): CompileInput => ({
edl: testEdl(),
profile: testProfile(),
brandKit: testKit(),
quality: "draft",
workDir: dir,
sources: new Map([
[
"asset-a",
{
path: source,
width: 480,
height: 270,
durationMs: 8000,
fps: 30,
hasAudio: true,
rotation: 0,
},
],
]),
transcripts: new Map([
[
"asset-a",
[
{ startMs: 200, endMs: 600, text: "Wait" },
{ startMs: 650, endMs: 1200, text: "for" },
{ startMs: 1250, endMs: 1800, text: "this!" },
{ startMs: 5100, endMs: 5600, text: "Here" },
{ startMs: 5650, endMs: 6400, text: "it is." },
],
],
]),
subjects: new Map([
[
"asset-a",
[
{ tMs: 0, cx: 0.3, cy: 0.5, strength: 1 },
{ tMs: 6000, cx: 0.7, cy: 0.5, strength: 1 },
],
],
]),
motionClips: new Map(),
files: {
lutPath: null,
watermarkPath: watermark,
fonts: new Map(),
audio: new Map([["music-1", music]]),
},
videoEncoder: "libx264",
...overrides,
});
it("renders 16:9 to 9:16 with auto-reframe, captions, watermark, ducked music and clean metadata", async () => {
const edl = testEdl({
tracks: {
video: [
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" },
{ id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 1500, reframe: "auto" },
],
audio: [
{
id: "bed",
kind: "music",
artifactId: "music-1",
inMs: 0,
outMs: 3500,
atMs: 0,
gainDb: -6,
fadeOutMs: 500,
duckUnderSpeech: true,
},
],
},
transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }],
captions: { mode: "both" },
overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 3500 }],
metadata: { title: "Example teaser" },
});
const plan = compileRenderPlan(baseInput({ edl }));
await execute(plan);
const video = plan.outputs.find((o) => o.kind === "video")!.path;
const result = await probe(video);
expect(result.width).toBe(1080);
expect(result.height).toBe(1920);
expect(Math.abs(result.durationMs - edlDurationMs(edl))).toBeLessThanOrEqual(1000 / 30 + 1);
const keys = Object.keys(result.tags).map((k) => k.toLowerCase());
expect(keys.some((k) => k.includes("creation_time") || k.includes("location"))).toBe(false);
expect(findMetadataLeaks(result.tags)).toEqual([]);
expect(result.tags["format.title"]).toBe("Example teaser");
const srt = await readFile(plan.outputs.find((o) => o.kind === "subtitles_srt")!.path, "utf8");
expect(srt).toContain("Wait for this!");
const poster = plan.outputs.find((o) => o.kind === "poster")!.path;
expect(existsSync(poster)).toBe(true);
expect((await probe(poster)).tags).toEqual({});
});
it("renders an intro, voice disguise and an escaped text overlay", async () => {
const edl = testEdl({
tracks: {
video: [
{
id: "c1",
assetId: "asset-a",
inMs: 1000,
outMs: 3000,
atMs: 0,
reframe: "none",
stabilize: true,
color: { exposure: 0.3, temperature: -0.4, saturation: 1.2, useBrandLut: true },
},
],
},
intro: { templateId: "intro-minimal", durationMs: 1000 },
overlays: [
{
id: "t1",
kind: "text",
atMs: 1000,
durationMs: 1500,
text: "50% off: it's [0:v]; rm -rf / %{pts} \\ done",
box: { x: 0.1, y: 0.3, w: 0.8, h: 0.05 },
},
{ id: "lt", kind: "lower_third", atMs: 1200, durationMs: 1000, text: "Lower third" },
{ id: "pb", kind: "progress_bar", atMs: 1000, durationMs: 2000 },
],
redactions: { voice: { method: hasRubberband ? "pitch" : "formant", semitones: 3 } },
metadata: {},
});
const plan = compileRenderPlan(
baseInput({
edl,
motionClips: new Map([["intro", intro]]),
quality: "final",
files: { ...baseInput({}).files, lutPath: lut },
}),
);
await execute(plan);
const result = await probe(plan.outputs[0]!.path);
expect(result.width).toBe(1080);
expect(Math.abs(result.durationMs - 3000)).toBeLessThanOrEqual(1000 / 30 + 1);
expect(findMetadataLeaks(result.tags)).toEqual([]);
expect(Object.keys(result.tags).some((k) => k.endsWith("title"))).toBe(false);
});
it("draws escaped text exactly like the literal text", async () => {
const text = "it's 50% [0:v]; a,b=c \\ %{pts} 'q'";
const literal = path.join(dir, "literal.txt");
await writeFile(literal, text);
const frame = async (filter: string) => {
const { stdout } = await run("ffmpeg", [
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`color=c=black:s=320x80:d=0.1,${filter}`,
"-frames:v",
"1",
"-f",
"framemd5",
"-",
]);
return stdout.split("\n").filter((line) => line && !line.startsWith("#"));
};
const escaped = await frame(`drawtext=text=${escapeFilterText(text)}:fontcolor=white:x=4:y=20`);
const reference = await frame(
`drawtext=expansion=none:textfile=${escapeFilterValue(literal)}:fontcolor=white:x=4:y=20`,
);
const blank = await frame("null");
expect(escaped).toEqual(reference);
expect(escaped).not.toEqual(blank);
});
});

576
packages/media/src/compile.test.ts Normal file
View File

@ -0,0 +1,576 @@
import type { RenderPlan } from "@rakazo/contracts";
import { RenderPlanSchema } from "@rakazo/contracts";
import { describe, expect, it } from "vitest";
import type { CaptionWord, CompileInput } from "./compile.js";
import {
atempoChain,
autoReframe,
buildCaptionCues,
compileRenderPlan,
escapeFilterText,
escapeFilterValue,
outputCaptionWords,
reframeExcluding,
toAss,
toSrt,
toVtt,
} from "./compile.js";
import { testEdl, testKit, testProfile } from "./test-fixtures.js";
const words: CaptionWord[] = [
{ startMs: 200, endMs: 600, text: "Wait" },
{ startMs: 650, endMs: 1200, text: "for" },
{ startMs: 1250, endMs: 1800, text: "this!" },
{ startMs: 5100, endMs: 5600, text: "Here" },
{ startMs: 5650, endMs: 6400, text: "it <b>is</b> --> now." },
];
function input(overrides: Partial<CompileInput> = {}): CompileInput {
return {
edl: testEdl(),
profile: testProfile(),
brandKit: testKit(),
quality: "final",
workDir: "/work/render-1",
sources: new Map([
[
"asset-a",
{
path: "/media/asset-a/proxy.mp4",
width: 1920,
height: 1080,
durationMs: 10_000,
fps: 30,
hasAudio: true,
rotation: 0,
},
],
[
"asset-b",
{
path: "/media/asset-b/original.mov",
width: 1080,
height: 1920,
durationMs: 8000,
fps: 60,
hasAudio: false,
rotation: 90,
},
],
]),
transcripts: new Map([["asset-a", words]]),
subjects: new Map([
[
"asset-a",
[
{ tMs: 0, cx: 0.2, cy: 0.5, strength: 1 },
{ tMs: 1500, cx: 0.4, cy: 0.5, strength: 1 },
{ tMs: 5500, cx: 0.8, cy: 0.4, strength: 0.5 },
],
],
]),
motionClips: new Map(),
files: {
lutPath: "/brand/look's.cube",
watermarkPath: "/brand/watermark.png",
fonts: new Map([["font-1", "/brand/fonts/ExampleSans.ttf"]]),
audio: new Map([
["music-1", "/brand/music/bed.wav"],
["vo-1", "/work/render-1/voiceover.wav"],
]),
},
videoEncoder: "libx264",
...overrides,
};
}
function ffmpegStep(plan: RenderPlan, id = "render") {
const step = plan.steps.find((s) => s.id === id);
if (step?.kind !== "ffmpeg") throw new Error(`no ffmpeg step ${id}`);
return step;
}
function graphOf(plan: RenderPlan): string {
const argv = ffmpegStep(plan).argv;
return argv[argv.indexOf("-filter_complex") + 1]!;
}
describe("compileRenderPlan", () => {
it("compiles 16:9 to 9:16 with auto-reframe, captions, watermark and loudnorm", () => {
const plan = compileRenderPlan(
input({
edl: testEdl({
captions: { mode: "both", presetId: "clean" },
overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 }],
metadata: { title: "Example: teaser" },
}),
}),
);
expect(RenderPlanSchema.parse(plan)).toEqual(plan);
expect(plan).toMatchSnapshot();
const graph = graphOf(plan);
expect(graph).toContain("crop=w=608:h=1080:x=");
expect(graph).toContain("scale=1080:1920");
expect(graph).toContain("loudnorm=I=-14:TP=-1:LRA=11:linear=true");
expect(graph).toContain("colorchannelmixer=aa=0.6");
expect(graph).toContain("ass=filename=/work/render-1/captions.ass");
expect(plan.outputs.map((o) => o.kind)).toEqual([
"video",
"subtitles_srt",
"subtitles_vtt",
"poster",
]);
const argv = ffmpegStep(plan).argv;
for (const flag of [
"-map_metadata",
"-map_chapters",
"+bitexact",
"+faststart",
"yuv420p",
"bt709",
]) {
expect(argv).toContain(flag);
}
expect(argv).toContain("title=Example: teaser");
expect(argv.join(" ")).toContain("-c:a aac -b:a 192k -ar 48000");
});
it("compiles transitions, music ducking, voice-over and fades", () => {
const plan = compileRenderPlan(
input({
edl: testEdl({
tracks: {
video: [
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" },
{
id: "c2",
assetId: "asset-a",
inMs: 2000,
outMs: 4000,
atMs: 1500,
reframe: "none",
speed: 0.5,
},
{
id: "c3",
assetId: "asset-a",
inMs: 3000,
outMs: 4000,
atMs: 6000,
reframe: "none",
},
{
id: "c4",
assetId: "asset-b",
inMs: 4000,
outMs: 5000,
atMs: 7000,
reframe: "auto",
},
],
audio: [
{
id: "bed",
kind: "music",
artifactId: "music-1",
inMs: 0,
outMs: 8000,
atMs: 0,
gainDb: -8,
fadeInMs: 500,
fadeOutMs: 1000,
duckUnderSpeech: true,
},
{ id: "vo", kind: "voiceover", artifactId: "vo-1", inMs: 0, outMs: 2000, atMs: 1000 },
],
},
transitions: [
{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 },
{ fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 },
{ fromClipId: "c3", toClipId: "c4", kind: "whip", durationMs: 0 },
],
targetLufs: -16,
}),
quality: "draft",
}),
);
expect(plan).toMatchSnapshot();
const graph = graphOf(plan);
expect(graph).toContain("xfade=transition=fade:duration=0.5:offset=1.5");
expect(graph).toContain("acrossfade=d=0.5");
expect(graph).toContain("atempo=0.5");
expect(graph).toContain("color=c=black:s=1080x1920"); // gap from 5.5 s to 6 s
expect(graph).toContain("sidechaincompress");
expect(graph).toContain("adelay=1000|1000");
expect(graph).toContain("loudnorm=I=-16");
expect(ffmpegStep(plan).argv).toEqual(expect.arrayContaining(["veryfast", "28", "3000k"]));
});
it("compiles voice disguise, metadata stripping, intro/outro and text overlays", () => {
const plan = compileRenderPlan(
input({
edl: testEdl({
intro: { templateId: "intro-minimal", durationMs: 1500 },
outro: { templateId: "outro-simple", durationMs: 1000 },
tracks: {
video: [
{
id: "c1",
assetId: "asset-a",
inMs: 0,
outMs: 3000,
atMs: 0,
reframe: [
{ tMs: 0, cx: 0.3, cy: 0.5, scale: 1 },
{ tMs: 3000, cx: 0.7, cy: 0.5, scale: 1.5 },
],
stabilize: true,
color: { exposure: 0.5, temperature: 0.2, saturation: 1.1, useBrandLut: true },
},
],
},
overlays: [
{
id: "lt",
kind: "lower_third",
atMs: 2000,
durationMs: 1500,
text: "Ana's: [studio]; 100% real",
},
{
id: "logo-anim",
kind: "motion",
atMs: 0,
durationMs: 1000,
motion: { templateId: "lower-third-a", durationMs: 1000 },
},
{ id: "pb", kind: "progress_bar", atMs: 1500, durationMs: 3000 },
],
redactions: { voice: { method: "pitch", semitones: -4 } },
metadata: {},
}),
motionClips: new Map([["intro", "/work/render-1/intro.mov"]]),
}),
);
expect(plan).toMatchSnapshot();
const graph = graphOf(plan);
expect(graph).toContain("rubberband=pitch=0.793701:formant=preserved");
expect(graph).toContain("deshake");
expect(graph).toContain("lut3d=file=/brand/look\\\\\\'s.cube");
expect(graph).toContain("concat=n=3:v=1:a=1[prog_v][prog_a]");
expect(plan.steps.filter((s) => s.kind === "motion").map((s) => s.id)).toEqual([
"motion-1",
"motion-2",
]);
const argv = ffmpegStep(plan).argv;
expect(argv.some((a) => a.startsWith("title="))).toBe(false);
expect(argv).toEqual(expect.arrayContaining(["-map_metadata", "-1", "-map_chapters", "-1"]));
});
it("uses the asetrate chain for formant disguise and HEVC settings for HEVC profiles", () => {
const plan = compileRenderPlan(
input({
profile: testProfile({
video: {
aspectRatios: ["9:16"],
width: 1080,
height: 1920,
fps: 30,
codec: "hevc",
maxBitrateKbps: 6000,
maxFileSizeMb: 100,
},
export: { poster: false },
}),
edl: testEdl({ redactions: { voice: { method: "formant", semitones: 12 } }, metadata: {} }),
videoEncoder: "hevc_videotoolbox",
}),
);
expect(graphOf(plan)).toContain("asetrate=96000,aresample=48000,atempo=0.5");
const argv = ffmpegStep(plan).argv;
expect(argv).toEqual(expect.arrayContaining(["-tag:v", "hvc1", "-b:v", "4800k"]));
expect(argv).toContain("filter_units=remove_types=39|40");
expect(plan.outputs.map((o) => o.kind)).toEqual(["video"]);
});
it("only ever uses SafeId labels and keeps text out of argv", () => {
const plan = compileRenderPlan(
input({
edl: testEdl({
overlays: [
{ id: "t1", kind: "text", atMs: 0, durationMs: 1000, text: "'; rm -rf / [0:v] %{pts}" },
],
}),
}),
);
const graph = graphOf(plan);
for (const label of graph.matchAll(/\[([^\]\\]*)\]/g)) {
expect(label[1]).toMatch(/^[a-z0-9_:-]+$/);
}
expect(graph).toContain("text=\\\\\\'\\; rm -rf / \\[0\\\\:v\\] \\\\\\\\%{pts}");
expect(ffmpegStep(plan).argv.every((arg) => !arg.includes("\0"))).toBe(true);
});
it("refuses unsafe or impossible input instead of guessing", () => {
expect(() =>
compileRenderPlan(
input({
edl: {
...testEdl(),
overlays: [{ id: "BAD;ID", kind: "text", atMs: 0, durationMs: 1, text: "x" }],
} as never,
}),
),
).toThrow();
expect(() => compileRenderPlan(input({ videoEncoder: "libx265" }))).toThrow(
/does not produce h264/,
);
expect(() => compileRenderPlan(input({ sources: new Map() }))).toThrow(/no source/);
expect(() =>
compileRenderPlan(
input({
edl: testEdl({ overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 1000 }] }),
files: { ...input().files, watermarkPath: null },
}),
),
).toThrow(/watermark/);
});
it("moves very large filtergraphs into a script file", () => {
const video = Array.from({ length: 300 }, (_, i) => ({
id: `c${i}`,
assetId: "asset-a",
inMs: 0,
outMs: 100,
atMs: i * 100,
reframe: Array.from({ length: 8 }, (_, k) => ({
tMs: k * 12,
cx: 0.3 + k * 0.05,
cy: 0.5,
scale: 1,
})),
}));
const plan = compileRenderPlan(input({ edl: testEdl({ tracks: { video } }) }));
const script = plan.steps.find((s) => s.id === "filtergraph");
expect(script?.kind).toBe("write_file");
expect(ffmpegStep(plan).argv).toContain("-filter_complex_script");
});
});
describe("escaping", () => {
it("escapes every filtergraph and drawtext special character", () => {
expect(escapeFilterText("plain text")).toBe("plain text");
expect(escapeFilterText("a:b")).toBe("a\\\\:b");
expect(escapeFilterText("a,b;c")).toBe("a\\,b\\;c");
expect(escapeFilterText("[0:v]")).toBe("\\[0\\\\:v\\]");
expect(escapeFilterText("x=y")).toBe("x\\\\=y");
expect(escapeFilterText("it's")).toBe("it\\\\\\'s");
expect(escapeFilterText("100%")).toBe("100\\\\\\\\%");
expect(escapeFilterText("a\\b")).toBe("a\\\\\\\\\\\\\\\\b");
expect(escapeFilterText(" line one\r\nline two\u0007 ")).toBe("line one\nline two");
});
/** av_get_token: one parser level; returns the token and what it left unconsumed. */
function getToken(value: string, terms: string): { token: string; rest: string } {
let token = "";
let i = 0;
while (i < value.length && !terms.includes(value[i]!)) {
const char = value[i++]!;
if (char === "\\" && i < value.length) token += value[i++];
else if (char === "'") {
while (i < value.length && value[i] !== "'") token += value[i++];
i++;
} else token += char;
}
return { token, rest: value.slice(i) };
}
function drawtextExpand(value: string): string {
let out = "";
for (let i = 0; i < value.length; i++) {
if (value[i] === "\\" && i + 1 < value.length) out += value[++i];
else if (value[i] === "%") throw new Error("expansion sequence reached drawtext");
else out += value[i];
}
return out;
}
it("neutralizes injection attempts", () => {
const attempts = [
"'; rm -rf /",
"[0:v]",
"%{pts}",
"%{eif:1:d}",
"x:fontfile=/etc/passwd",
"a',drawtext=text='b",
"a\\",
"50% off; [in]null[out]",
];
for (const attempt of attempts) {
const graphLevel = getToken(escapeFilterText(attempt), "[],;");
expect(graphLevel.rest).toBe("");
const optionLevel = getToken(graphLevel.token, ":");
expect(optionLevel.rest).toBe("");
expect(drawtextExpand(optionLevel.token)).toBe(attempt);
}
expect(escapeFilterText("%{pts}")).toBe("\\\\\\\\%{pts}");
expect(escapeFilterValue("C:\\media\\a,b.cube")).toBe("C\\\\:\\\\\\\\media\\\\\\\\a\\,b.cube");
expect(() => escapeFilterValue("a\0b")).toThrow();
});
it("chains atempo within its clean range", () => {
expect(atempoChain(1)).toEqual([]);
expect(atempoChain(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]);
expect(atempoChain(3)).toEqual(["atempo=2", "atempo=1.5"]);
expect(atempoChain(4)).toEqual(["atempo=2", "atempo=2"]);
});
});
describe("captions", () => {
it("builds cues by line length, line count, pauses and sentence ends", () => {
const cues = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 1 });
expect(cues.map((c) => c.text)).toEqual(["Wait for", "this!", "Here", "it <b>is</b> --> now."]);
const two = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 2 });
expect(two[0]).toMatchObject({ startMs: 200, endMs: 1800, text: "Wait for\nthis!" });
expect(two[0]!.words).toHaveLength(3);
expect(
buildCaptionCues([{ startMs: 0, endMs: 0, text: " " }], { maxCharsPerLine: 10, maxLines: 2 }),
).toEqual([]);
const short = buildCaptionCues([{ startMs: 1000, endMs: 1100, text: "Hi" }], {
maxCharsPerLine: 10,
maxLines: 2,
});
expect(short[0]!.endMs).toBe(1500);
});
it("formats SRT and VTT", () => {
const cues = buildCaptionCues(words, { maxCharsPerLine: 32, maxLines: 2 });
expect(toSrt(cues)).toBe(
"1\n00:00:00,200 --> 00:00:01,800\nWait for this!\n\n2\n00:00:05,100 --> 00:00:06,400\nHere it <b>is</b> -> now.\n",
);
expect(toVtt(cues)).toBe(
"WEBVTT\n\n00:00:00.200 --> 00:00:01.800\nWait for this!\n\n00:00:05.100 --> 00:00:06.400\nHere it &lt;b&gt;is&lt;/b&gt; --&gt; now.\n",
);
expect(toSrt([{ startMs: 3_723_004, endMs: 3_723_500, text: "a\n\nb", words: [] }])).toBe(
"1\n01:02:03,004 --> 01:02:03,500\na\nb\n",
);
});
it("maps source words to output time across speed and intro", () => {
const edl = testEdl({
intro: { templateId: "intro-minimal", durationMs: 1000 },
tracks: {
video: [
{
id: "c1",
assetId: "asset-a",
inMs: 1000,
outMs: 2000,
atMs: 0,
reframe: "none",
speed: 2,
},
{
id: "c2",
assetId: "asset-a",
inMs: 5000,
outMs: 6000,
atMs: 500,
reframe: "none",
muted: true,
},
],
},
});
expect(outputCaptionWords(edl, new Map([["asset-a", words]]))).toEqual([
{ startMs: 1125, endMs: 1400, text: "this!" },
]);
});
it("writes ASS without override injection and with karaoke timing", () => {
const cues = buildCaptionCues(
[
{ startMs: 0, endMs: 400, text: "{\\pos(0,0)}hi" },
{ startMs: 600, endMs: 900, text: "you\\N" },
],
{ maxCharsPerLine: 32, maxLines: 2 },
);
const ass = toAss(cues, {
width: 1080,
height: 1920,
profile: testProfile(),
preset: {
...testKit().captionPresets[0]!,
template: "karaoke",
highlightColor: "#FFAA00",
uppercase: true,
},
fallbackTextColor: "#FFFFFF",
position: "lower",
maxCharsPerLine: 32,
wordHighlight: true,
});
const dialogue = ass.split("\n").find((line) => line.startsWith("Dialogue:"))!;
expect(dialogue).toBe(
"Dialogue: 0,0:00:00.00,0:00:00.90,Default,,0,0,0,,{\\k40}\uFF5B\\\u2060POS(0,0)\uFF5DHI {\\k20}{\\k30}YOU\\\u2060N",
);
expect(ass).toContain("Style: Default,Example Sans,96,&H0000AAFF,&H00FFFFFF");
expect(ass).toContain(",2,92,168,422,1");
});
});
describe("reframing", () => {
const geometry = {
sourceWidth: 1920,
sourceHeight: 1080,
outWidth: 1080,
outHeight: 1920,
durationMs: 4000,
};
it("follows the subject smoothly and keeps the crop inside the frame", () => {
const path = autoReframe(
[
{ tMs: 0, cx: 0, cy: 0.5, strength: 1 },
{ tMs: 2000, cx: 0.5, cy: 0.5, strength: 1 },
{ tMs: 4000, cx: 1, cy: 0.5, strength: 1 },
],
geometry,
);
const half = 608 / 1920 / 2;
for (const k of path) {
expect(k.cx).toBeGreaterThanOrEqual(half - 1e-4);
expect(k.cx).toBeLessThanOrEqual(1 - half + 1e-4);
expect(k.cy).toBe(0.5);
expect(k.scale).toBe(1);
}
expect(path[0]!.tMs).toBe(0);
expect(path.at(-1)!.tMs).toBe(4000);
for (let i = 1; i < path.length; i++)
expect(path[i]!.cx).toBeGreaterThanOrEqual(path[i - 1]!.cx);
expect(path.length).toBeLessThanOrEqual(64);
expect(autoReframe([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]);
});
it("keeps faces out of the frame or reports that it cannot", () => {
const left = reframeExcluding(
[{ tMs: 100, box: { x: 0.05, y: 0.3, w: 0.15, h: 0.3 } }],
geometry,
);
expect(left).toHaveLength(1);
const k = left![0]!;
expect(k.tMs).toBe(100);
expect(k.scale).toBe(1);
expect(k.cx - 608 / 1920 / 2).toBeGreaterThanOrEqual(0.22 - 1e-6);
// Faces all over the frame: no crop can exclude them.
const everywhere = [0, 0.25, 0.5, 0.75].map((x) => ({
tMs: 0,
box: { x, y: 0, w: 0.25, h: 1 },
}));
expect(reframeExcluding(everywhere, geometry)).toBeNull();
// A face across the middle forces a tighter crop.
const wide = reframeExcluding([{ tMs: 0, box: { x: 0.2, y: 0.4, w: 0.6, h: 0.2 } }], geometry);
expect(wide![0]!.scale).toBeGreaterThan(1);
expect(reframeExcluding([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]);
});
});

1332
packages/media/src/compile.ts
View File

File diff suppressed because it is too large Load Diff

337
packages/media/src/edl.test.ts Normal file
View File

@ -0,0 +1,337 @@
import type { EdlError } from "@rakazo/contracts";
import { EdlErrorSchema } from "@rakazo/contracts";
import { describe, expect, it } from "vitest";
import type { EdlContext } from "./edl.js";
import { edlDurationMs, locateOutputTime, locateOutputTimeAll, validateEdl } from "./edl.js";
import { testContext, testEdl, testProfile } from "./test-fixtures.js";
/** Raw plan (pre-defaults) so cases can break it in any way. */
function raw(patch: (plan: Record<string, any>) => void): unknown {
const plan = JSON.parse(JSON.stringify(testEdl())) as Record<string, any>;
patch(plan);
return plan;
}
function codes(input: unknown, context: EdlContext = testContext()): EdlError["code"][] {
const result = validateEdl(input, context);
return result.ok ? [] : result.errors.map((e) => e.code);
}
describe("validateEdl", () => {
it("accepts a valid plan and returns the parsed EDL", () => {
const result = validateEdl(testEdl(), testContext());
expect(result.ok).toBe(true);
if (result.ok) expect(result.edl.tracks.video).toHaveLength(2);
});
const cases: Array<{
code: EdlError["code"];
input: unknown;
context?: Partial<EdlContext>;
path?: string;
}> = [
{ code: "schema", input: raw((p) => (p.edlVersion = 2)), path: "edlVersion" },
{
code: "schema",
input: raw((p) => (p.tracks.video[1].id = "c1")),
path: "tracks.video[1].id",
},
{
code: "schema",
input: raw((p) => (p.tracks.video[0].id = "Bad Id;[x]")),
path: "tracks.video[0].id",
},
{
code: "unknown_asset",
input: raw((p) => (p.tracks.video[0].assetId = "missing")),
path: "tracks.video[0].assetId",
},
{
code: "unknown_clip",
input: raw((p) => {
p.transitions = [{ fromClipId: "c1", toClipId: "c9", kind: "cut", durationMs: 0 }];
}),
path: "transitions[0].toClipId",
},
{
code: "unknown_clip",
input: raw((p) => {
p.redactions.regions = [
{
id: "r1",
clipId: "zz",
keyframes: [{ tMs: 0, box: { x: 0, y: 0, w: 0.1, h: 0.1 } }],
treatment: "blur",
},
];
}),
},
{
code: "unknown_artifact",
input: raw((p) => {
p.tracks.audio = [
{ id: "m", kind: "sfx", artifactId: "nope", inMs: 0, outMs: 1000, atMs: 0 },
];
}),
},
{
code: "unknown_artifact",
input: raw((p) => (p.overlays = [{ id: "w", kind: "watermark", atMs: 0, durationMs: 1000 }])),
context: { brandKit: null },
},
{
code: "unknown_mark",
input: raw((p) => (p.redactions.marks = [{ markId: "mark-x", treatment: "blur" }])),
},
{
code: "unknown_person",
input: raw((p) => {
p.redactions.faces = [{ id: "f", target: { personId: "person-x" }, mode: "blur" }];
}),
},
{
code: "unknown_track",
input: raw((p) => {
p.redactions.faces = [{ id: "f", target: { trackId: "track-x" }, mode: "pixelate" }];
}),
},
{
code: "unknown_template",
input: raw((p) => (p.intro = { templateId: "intro-none", durationMs: 1000 })),
path: "intro.templateId",
},
{
code: "unknown_template",
input: raw((p) => (p.captions = { mode: "burned", presetId: "missing" })),
path: "captions.presetId",
},
{
code: "range_out_of_asset",
input: raw((p) => (p.tracks.video[1].outMs = 12_000)),
path: "tracks.video[1].outMs",
},
{
code: "range_out_of_asset",
input: raw((p) => (p.tracks.video[0].reframe = [{ tMs: 9000, cx: 0.5, cy: 0.5, scale: 1 }])),
},
{
code: "range_empty",
input: raw((p) => (p.tracks.video[0].outMs = 0)),
},
{
code: "range_empty",
input: raw(
(p) => (p.overlays = [{ id: "t", kind: "text", atMs: 0, durationMs: 0, text: "x" }]),
),
},
{
code: "overlap",
input: raw((p) => (p.tracks.video[1].atMs = 1500)),
path: "tracks.video[1].atMs",
},
{
code: "overlap",
// A crossfade needs exactly its duration of overlap.
input: raw((p) => {
p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }];
}),
},
{
code: "overlap",
input: raw((p) => {
p.tracks.audio = [
{ id: "m1", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 0 },
{ id: "m2", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 2000 },
];
}),
},
{
code: "duration_below_min",
input: raw((p) => (p.output.durationHardMinMs = 5000)),
},
{
code: "duration_below_min",
input: testEdl(),
context: { profile: testProfile({ duration: { hardMinMs: 6000, hardMaxMs: 60_000 } }) },
},
{
code: "duration_above_max",
input: raw((p) => (p.output.durationHardMaxMs = 3000)),
},
{
code: "duration_above_max",
input: testEdl(),
context: { profile: testProfile({ duration: { hardMinMs: 0, hardMaxMs: 3000 } }) },
},
{
code: "overlay_outside_safe_zone",
input: raw((p) => {
p.overlays = [
{
id: "t",
kind: "text",
atMs: 0,
durationMs: 1000,
text: "Hi",
box: { x: 0.1, y: 0.85, w: 0.5, h: 0.1 },
},
];
}),
path: "overlays[0].box",
},
{
code: "paid_only_segment",
input: testEdl(),
context: { paidOnly: [{ assetId: "asset-a", startMs: 6500, endMs: 9000 }] },
path: "tracks.video[1]",
},
{
code: "paid_only_segment",
input: testEdl({ output: { ...testEdl().output, profileId: "example-paid-video" } }),
context: {
profile: testProfile({ id: "example-paid-video", kind: "paid_video" }),
paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 100 }],
goal: "teaser",
},
},
{
code: "profile_mismatch",
input: raw((p) => (p.output.profileId = "other-profile")),
},
{
code: "profile_mismatch",
input: raw((p) => {
p.output.width = 1920;
p.output.height = 1080;
}),
},
{
code: "music_license",
input: raw((p) => {
p.tracks.audio = [
{ id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 },
];
}),
},
{
code: "music_license",
input: raw((p) => {
p.tracks.audio = [
{ id: "m", kind: "music", artifactId: "vo-1", inMs: 0, outMs: 1000, atMs: 0 },
];
}),
},
{
code: "music_license",
input: raw((p) => {
p.tracks.audio = [
{ id: "m", kind: "music", artifactId: "music-1", inMs: 0, outMs: 1000, atMs: 0 },
];
}),
context: { profile: testProfile({ audio: { music: "platform_library_only" } }) },
},
];
it.each(cases)("rejects $code ($path)", ({ code, input, context, path }) => {
const result = validateEdl(input, testContext(context));
expect(result.ok).toBe(false);
if (result.ok) return;
for (const error of result.errors) expect(EdlErrorSchema.safeParse(error).success).toBe(true);
const match = result.errors.find((e) => e.code === code);
expect(match, JSON.stringify(result.errors)).toBeDefined();
if (path) expect(match!.path).toBe(path);
});
it("covers every error code", () => {
const covered = new Set(cases.map((c) => c.code));
expect([...covered].sort()).toEqual([...EdlErrorSchema.shape.code.options].sort());
});
it("allows paid-only material in a full release for a paid profile and any music where allowed", () => {
const paid = testProfile({
id: "example-paid-video",
kind: "paid_video",
audio: { music: "any" },
});
const plan = raw((p) => {
p.output.profileId = "example-paid-video";
p.tracks.audio = [
{ id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 },
];
});
expect(
codes(
plan,
testContext({ profile: paid, paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 9000 }] }),
),
).toEqual([]);
});
it("accepts photos without a duration and a crossfade with the right overlap", () => {
const plan = raw((p) => {
p.tracks.video[0].assetId = "photo-1";
p.tracks.video[1].atMs = 1500;
p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }];
});
expect(codes(plan)).toEqual([]);
});
it("reports every failure at once", () => {
const plan = raw((p) => {
p.tracks.video[0].assetId = "missing";
p.tracks.video[1].outMs = 4000;
p.redactions.marks = [{ markId: "mark-x", treatment: "cover" }];
});
expect(codes(plan)).toEqual(["unknown_asset", "range_empty", "unknown_mark"]);
});
});
describe("timeline", () => {
const edl = testEdl({
intro: { templateId: "intro-minimal", durationMs: 1000 },
outro: { templateId: "outro-simple", durationMs: 500 },
tracks: {
video: [
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" },
{
id: "c2",
assetId: "asset-b",
inMs: 1000,
outMs: 5000,
atMs: 1500,
speed: 2,
reframe: "none",
},
],
},
transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "dip", durationMs: 500 }],
});
it("measures intro, speed, transition overlap and outro", () => {
expect(edlDurationMs(edl)).toBe(1000 + 1500 + 2000 + 500);
expect(edlDurationMs(testEdl())).toBe(4000);
});
it("maps output time back to clip and source time", () => {
expect(locateOutputTime(edl, 500)).toBeNull();
expect(locateOutputTime(edl, 1000)).toEqual({ clipId: "c1", assetId: "asset-a", sourceMs: 0 });
expect(locateOutputTime(edl, 2200)).toEqual({
clipId: "c1",
assetId: "asset-a",
sourceMs: 1200,
});
// Inside the transition both clips are visible; the incoming one wins.
expect(locateOutputTimeAll(edl, 2700).map((h) => h.clipId)).toEqual(["c1", "c2"]);
expect(locateOutputTime(edl, 2700)).toEqual({
clipId: "c2",
assetId: "asset-b",
sourceMs: 1400,
});
expect(locateOutputTime(edl, 4400)).toEqual({
clipId: "c2",
assetId: "asset-b",
sourceMs: 4800,
});
expect(locateOutputTime(edl, 4600)).toBeNull();
});
});

411
packages/media/src/edl.ts
View File

@ -1,5 +1,30 @@
import type { BrandKit, Edl, EdlError, MediaAssetKind, PlatformProfile } from "@rakazo/contracts"; import type {
import { notImplemented } from "./not-implemented.js"; AspectRatio,
BrandKit,
Edl,
EdlError,
EdlTransition,
EdlVideoClip,
MediaAssetKind,
MediaProjectGoal,
NormalizedBox,
PlatformProfile,
} from "@rakazo/contracts";
import { EdlSchema } from "@rakazo/contracts";
/*
* Timeline model shared by validation, compilation, gate and auto-fix:
*
* - The output is intro + main program + outro, played back to back.
* - `tracks.video[].atMs` is a position on the main program, which starts
* after the intro. A clip lasts (outMs - inMs) / speed.
* - Clips may only overlap where a crossfade, dip or whip transition joins
* them, and then by exactly the transition duration. Cuts and match cuts
* never overlap.
* - Overlays and audio clips are positioned on the output timeline (intro
* included), so music can run under the intro.
* - Reframe keyframes are in source time of the clip's asset.
*/
export interface EdlContext { export interface EdlContext {
profile: PlatformProfile; profile: PlatformProfile;
@ -18,28 +43,384 @@ export interface EdlContext {
templateIds: Set<string>; templateIds: Set<string>;
/** Segments the user reserved for the paid release. */ /** Segments the user reserved for the paid release. */
paidOnly: Array<{ assetId: string; startMs: number; endMs: number }>; paidOnly: Array<{ assetId: string; startMs: number; endMs: number }>;
/** Project goal; a teaser never uses paid-only segments, whatever the profile. */
goal?: MediaProjectGoal;
} }
export type EdlValidation = { ok: true; edl: Edl } | { ok: false; errors: EdlError[] }; export type EdlValidation = { ok: true; edl: Edl } | { ok: false; errors: EdlError[] };
/** Transitions rendered as a timed blend (xfade). Cuts and match cuts are plain concatenation. */
export const BLENDED_TRANSITIONS: ReadonlySet<EdlTransition["kind"]> = new Set([
"crossfade",
"dip",
"whip",
]);
export const ASPECT_RATIO_VALUES: Record<AspectRatio, number> = {
"9:16": 9 / 16,
"4:5": 4 / 5,
"1:1": 1,
"16:9": 16 / 9,
"4:3": 4 / 3,
"21:9": 21 / 9,
};
const ASPECT_TOLERANCE = 0.01;
/** Whether width × height matches one of the allowed aspect ratios within 1 %. */
export function matchesAspect(width: number, height: number, allowed: AspectRatio[]): boolean {
const actual = width / height;
return allowed.some(
(ratio) =>
Math.abs(actual - ASPECT_RATIO_VALUES[ratio]) / ASPECT_RATIO_VALUES[ratio] <=
ASPECT_TOLERANCE,
);
}
/** Output length of one video clip after speed change. */
export function clipLengthMs(clip: Pick<EdlVideoClip, "inMs" | "outMs" | "speed">): number {
return Math.max(0, Math.round((clip.outMs - clip.inMs) / clip.speed));
}
/** End of the main program (excluding intro and outro). */
export function mainProgramMs(edl: Edl): number {
let end = 0;
for (const clip of edl.tracks.video) end = Math.max(end, clip.atMs + clipLengthMs(clip));
return end;
}
export function introMs(edl: Edl): number {
return edl.intro?.durationMs ?? 0;
}
/** Output length in ms including intro, outro, speed, and transition overlap. */
export function edlDurationMs(edl: Edl): number {
return introMs(edl) + mainProgramMs(edl) + (edl.outro?.durationMs ?? 0);
}
/** Every clip visible at an output timestamp (two during a blended transition), earliest first. */
export function locateOutputTimeAll(
edl: Edl,
outputMs: number,
): Array<{ clipId: string; assetId: string; sourceMs: number }> {
const t = outputMs - introMs(edl);
if (t < 0) return [];
return [...edl.tracks.video]
.filter((clip) => t >= clip.atMs && t < clip.atMs + clipLengthMs(clip))
.sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id))
.map((clip) => ({
clipId: clip.id,
assetId: clip.assetId,
sourceMs: Math.min(clip.outMs, Math.round(clip.inMs + (t - clip.atMs) * clip.speed)),
}));
}
/**
* Map an output timestamp back to the clip and source time it shows. During a
* blended transition the incoming clip wins; intro and outro map to null.
*/
export function locateOutputTime(
edl: Edl,
outputMs: number,
): { clipId: string; assetId: string; sourceMs: number } | null {
return locateOutputTimeAll(edl, outputMs).at(-1) ?? null;
}
/** Whether a normalized box lies inside the profile's safe zones. */
export function boxInsideSafeZone(box: NormalizedBox, profile: PlatformProfile): boolean {
const epsilon = 1e-6;
const zones = profile.safeZones;
return (
box.x >= zones.leftPct / 100 - epsilon &&
box.y >= zones.topPct / 100 - epsilon &&
box.x + box.w <= 1 - zones.rightPct / 100 + epsilon &&
box.y + box.h <= 1 - zones.bottomPct / 100 + epsilon
);
}
function formatPath(path: ReadonlyArray<PropertyKey>): string {
let out = "";
for (const part of path) {
if (typeof part === "number") out += `[${part}]`;
else out += out ? `.${String(part)}` : String(part);
}
return out || "(root)";
}
/** Video clips in timeline order, ties broken by id for determinism. */
export function sortedVideoClips(edl: Edl): EdlVideoClip[] {
return [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id));
}
/** /**
* Parse with EdlSchema, then check references, ranges, overlaps, durations, * Parse with EdlSchema, then check references, ranges, overlaps, durations,
* safe zones, paid-only use (teaser-style profiles), and music licensing. * safe zones, paid-only use (teaser-style profiles), and music licensing.
* Every failure is reported; nothing is silently corrected. * Every failure is reported; nothing is silently corrected.
*/ */
export function validateEdl(_input: unknown, _context: EdlContext): EdlValidation { export function validateEdl(input: unknown, context: EdlContext): EdlValidation {
return notImplemented("validateEdl"); const parsed = EdlSchema.safeParse(input);
} if (!parsed.success) {
return {
ok: false,
errors: parsed.error.issues.map((issue) => ({
code: "schema",
path: formatPath(issue.path),
message: issue.message,
})),
};
}
const edl = parsed.data;
const errors: EdlError[] = [];
const fail = (code: EdlError["code"], path: string, message: string) =>
errors.push({ code, path, message });
const { profile } = context;
/** Output length in ms including intro, outro, speed, and transition overlap. */ if (edl.output.profileId !== profile.id) {
export function edlDurationMs(_edl: Edl): number { fail("profile_mismatch", "output.profileId", `plan targets ${edl.output.profileId}`);
return notImplemented("edlDurationMs"); }
} if (!matchesAspect(edl.output.width, edl.output.height, profile.video.aspectRatios)) {
fail(
"profile_mismatch",
"output",
`${edl.output.width}x${edl.output.height} is not one of ${profile.video.aspectRatios.join(", ")}`,
);
}
if (edl.output.fps > profile.video.fps * 1.001) {
fail("profile_mismatch", "output.fps", `fps above profile maximum ${profile.video.fps}`);
}
/** Map an output timestamp back to the clip and source time it shows. */ const seenIds = new Map<string, string>();
export function locateOutputTime( const claimId = (id: string, path: string) => {
_edl: Edl, const previous = seenIds.get(id);
_outputMs: number, if (previous) fail("schema", path, `id ${id} already used at ${previous}`);
): { clipId: string; assetId: string; sourceMs: number } | null { else seenIds.set(id, path);
return notImplemented("locateOutputTime"); };
const clipsById = new Map<string, EdlVideoClip>();
edl.tracks.video.forEach((clip, index) => {
const path = `tracks.video[${index}]`;
claimId(clip.id, `${path}.id`);
clipsById.set(clip.id, clip);
if (clip.outMs <= clip.inMs) fail("range_empty", path, "outMs must be after inMs");
const asset = context.assets.get(clip.assetId);
if (!asset) {
fail("unknown_asset", `${path}.assetId`, `asset ${clip.assetId} is not available`);
} else if (asset.durationMs !== null && asset.kind !== "photo") {
if (clip.outMs > asset.durationMs) {
fail(
"range_out_of_asset",
`${path}.outMs`,
`outMs ${clip.outMs} exceeds asset duration ${asset.durationMs}`,
);
}
if (clip.inMs >= asset.durationMs) {
fail("range_out_of_asset", `${path}.inMs`, `inMs ${clip.inMs} is past the asset end`);
}
}
if (Array.isArray(clip.reframe)) {
clip.reframe.forEach((keyframe, k) => {
if (keyframe.tMs < clip.inMs || keyframe.tMs > clip.outMs) {
fail(
"range_out_of_asset",
`${path}.reframe[${k}].tMs`,
"reframe keyframe lies outside the clip's source range",
);
}
});
}
if (
(profile.kind === "free_social" || context.goal === "teaser") &&
context.paidOnly.some(
(segment) =>
segment.assetId === clip.assetId &&
segment.startMs < clip.outMs &&
clip.inMs < segment.endMs,
)
) {
fail("paid_only_segment", path, "clip uses material reserved for the paid release");
}
});
// Transitions: known clips, adjacent in time, at most one in and one out per clip.
const blendBetween = new Map<string, number>();
const outgoing = new Set<string>();
const incoming = new Set<string>();
const order = sortedVideoClips(edl);
const nextOf = new Map<string, string>();
for (let i = 0; i + 1 < order.length; i++) nextOf.set(order[i]!.id, order[i + 1]!.id);
edl.transitions.forEach((transition, index) => {
const path = `transitions[${index}]`;
const from = clipsById.get(transition.fromClipId);
const to = clipsById.get(transition.toClipId);
if (!from) fail("unknown_clip", `${path}.fromClipId`, `no clip ${transition.fromClipId}`);
if (!to) fail("unknown_clip", `${path}.toClipId`, `no clip ${transition.toClipId}`);
if (!from || !to) return;
if (nextOf.get(from.id) !== to.id) {
fail("schema", path, "a transition must join a clip to the clip that follows it");
return;
}
if (outgoing.has(from.id) || incoming.has(to.id)) {
fail("schema", path, "a clip can have only one transition on each side");
return;
}
outgoing.add(from.id);
incoming.add(to.id);
if (BLENDED_TRANSITIONS.has(transition.kind) && transition.durationMs > 0) {
const shortest = Math.min(clipLengthMs(from), clipLengthMs(to));
if (transition.durationMs >= shortest) {
fail("overlap", `${path}.durationMs`, "transition is longer than one of its clips");
return;
}
blendBetween.set(`${from.id}>${to.id}`, transition.durationMs);
}
});
// Overlaps on the video track: only blended transitions may overlap, by exactly their length.
for (let i = 0; i + 1 < order.length; i++) {
const a = order[i]!;
const b = order[i + 1]!;
const overlap = a.atMs + clipLengthMs(a) - b.atMs;
const blend = blendBetween.get(`${a.id}>${b.id}`);
const path = `tracks.video[${edl.tracks.video.indexOf(b)}].atMs`;
if (blend !== undefined) {
if (Math.abs(overlap - blend) > 1) {
fail(
"overlap",
path,
`clips ${a.id} and ${b.id} must overlap by the ${blend} ms transition`,
);
}
} else if (overlap > 1) {
fail("overlap", path, `clip ${b.id} starts ${overlap} ms before ${a.id} ends`);
}
}
for (let i = 0; i + 2 < order.length; i++) {
const a = order[i]!;
const c = order[i + 2]!;
if (a.atMs + clipLengthMs(a) > c.atMs + 1) {
fail("overlap", `tracks.video[${edl.tracks.video.indexOf(c)}].atMs`, "three clips overlap");
}
}
// Audio clips.
const audioByKind = new Map<string, Array<{ start: number; end: number; index: number }>>();
edl.tracks.audio.forEach((clip, index) => {
const path = `tracks.audio[${index}]`;
claimId(clip.id, `${path}.id`);
if (clip.outMs <= clip.inMs) fail("range_empty", path, "outMs must be after inMs");
if (!context.artifactIds.has(clip.artifactId)) {
fail(
"unknown_artifact",
`${path}.artifactId`,
`artifact ${clip.artifactId} is not available`,
);
}
if (clip.fadeInMs + clip.fadeOutMs > clip.outMs - clip.inMs && clip.outMs > clip.inMs) {
fail("range_empty", path, "fades are longer than the clip");
}
if (clip.kind === "music") {
if (profile.audio.music === "platform_library_only") {
fail(
"music_license",
`${path}.artifactId`,
"this platform only allows music added from its own library",
);
} else if (profile.audio.music === "licensed_only") {
const license = context.music.get(clip.artifactId);
if (!license?.commercialUse) {
fail(
"music_license",
`${path}.artifactId`,
license ? "music is not licensed for commercial use" : "music license is unknown",
);
}
}
}
if (clip.kind !== "sfx") {
const list = audioByKind.get(clip.kind) ?? [];
list.push({ start: clip.atMs, end: clip.atMs + (clip.outMs - clip.inMs), index });
audioByKind.set(clip.kind, list);
}
});
for (const list of audioByKind.values()) {
list.sort((a, b) => a.start - b.start || a.index - b.index);
for (let i = 0; i + 1 < list.length; i++) {
if (list[i]!.end > list[i + 1]!.start) {
fail("overlap", `tracks.audio[${list[i + 1]!.index}].atMs`, "audio clips overlap");
}
}
}
// Templates.
const checkTemplate = (templateId: string, path: string) => {
if (!context.templateIds.has(templateId)) {
fail("unknown_template", path, `template ${templateId} is not available`);
}
};
if (edl.intro) checkTemplate(edl.intro.templateId, "intro.templateId");
if (edl.outro) checkTemplate(edl.outro.templateId, "outro.templateId");
// Overlays.
edl.overlays.forEach((overlay, index) => {
const path = `overlays[${index}]`;
claimId(overlay.id, `${path}.id`);
if (overlay.durationMs <= 0)
fail("range_empty", `${path}.durationMs`, "overlay has no duration");
if (overlay.motion) checkTemplate(overlay.motion.templateId, `${path}.motion.templateId`);
if (overlay.box && !boxInsideSafeZone(overlay.box, profile)) {
fail("overlay_outside_safe_zone", `${path}.box`, "overlay leaves the profile safe zone");
}
if (overlay.kind === "watermark" && !context.brandKit?.watermark) {
fail("unknown_artifact", path, "the brand kit has no watermark");
}
if ((overlay.kind === "text" || overlay.kind === "lower_third") && !overlay.text?.trim()) {
if (!overlay.motion) fail("range_empty", `${path}.text`, "text overlay has no text");
}
});
// Captions preset.
const presetId = edl.captions?.presetId;
if (presetId && context.brandKit) {
if (!context.brandKit.captionPresets.some((preset) => preset.id === presetId)) {
fail("unknown_template", "captions.presetId", `caption preset ${presetId} is not in the kit`);
}
}
// Redactions.
edl.redactions.faces.forEach((redaction, index) => {
const path = `redactions.faces[${index}]`;
claimId(redaction.id, `${path}.id`);
const target = redaction.target;
if ("trackId" in target && !context.faceTrackIds.has(target.trackId)) {
fail("unknown_track", `${path}.target.trackId`, `face track ${target.trackId} is unknown`);
}
if ("personId" in target && !context.personIds.has(target.personId)) {
fail("unknown_person", `${path}.target.personId`, `person ${target.personId} is unknown`);
}
});
edl.redactions.marks.forEach((mark, index) => {
if (!context.markIds.has(mark.markId)) {
fail("unknown_mark", `redactions.marks[${index}].markId`, `mark ${mark.markId} is unknown`);
}
});
edl.redactions.regions.forEach((region, index) => {
const path = `redactions.regions[${index}]`;
claimId(region.id, `${path}.id`);
if (!clipsById.has(region.clipId)) {
fail("unknown_clip", `${path}.clipId`, `no clip ${region.clipId}`);
}
});
// Duration against both the plan's and the profile's hard bounds.
const duration = edlDurationMs(edl);
const minMs = Math.max(edl.output.durationHardMinMs, profile.duration.hardMinMs);
const maxMs = Math.min(edl.output.durationHardMaxMs, profile.duration.hardMaxMs);
if (duration < minMs) {
fail("duration_below_min", "output", `duration ${duration} ms is below ${minMs} ms`);
}
if (duration > maxMs) {
fail("duration_above_max", "output", `duration ${duration} ms is above ${maxMs} ms`);
}
return errors.length > 0 ? { ok: false, errors } : { ok: true, edl };
} }

531
packages/media/src/gate.test.ts Normal file
View File

@ -0,0 +1,531 @@
import type { FfprobeReport, GateReport } from "@rakazo/contracts";
import { GateReportSchema, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
import { describe, expect, it } from "vitest";
import { edlDurationMs } from "./edl.js";
import type { GateInput } from "./gate.js";
import {
checkCopy,
checkTechnical,
evaluateGate,
findMetadataLeaks,
METADATA_ALLOWED_TAGS,
planAutoFix,
} from "./gate.js";
import { testEdl, testPaidProfile, testProfile } from "./test-fixtures.js";
const cleanTags = {
"format.major_brand": "isom",
"format.minor_version": "512",
"format.compatible_brands": "isomiso2avc1mp41",
"format.title": "Example",
"stream.0.language": "und",
"stream.0.handler_name": "VideoHandler",
"stream.0.vendor_id": "[0][0][0][0]",
"stream.0.encoder": "Lavc libx264",
"stream.1.handler_name": "SoundHandler",
};
function probe(overrides: Partial<FfprobeReport> = {}): FfprobeReport {
return {
durationMs: 4000,
width: 1080,
height: 1920,
fps: 30,
videoCodec: "h264",
audioCodec: "aac",
sizeBytes: 2_000_000,
integratedLufs: -14.3,
truePeakDb: -1.2,
tags: cleanTags,
...overrides,
};
}
function gateInput(overrides: Partial<GateInput> = {}): GateInput {
return {
profile: testProfile(),
edl: testEdl(),
ffprobe: probe(),
content: [
{ tMs: 0, scores: { explicit_nudity: 0.01 } },
{ tMs: 500, scores: { explicit_nudity: 0.02, violence: 0.1 } },
],
faces: [],
trackingGaps: [],
marks: [],
people: [],
copy: { title: "A day out", caption: "Short caption #one", tags: ["two"] },
aiAlterations: [],
bundleHasDisclosure: false,
watermarkBox: { x: 0.7, y: 0.12, w: 0.15, h: 0.05 },
sampledFrames: 8,
autoFixRound: 0,
now: new Date("2026-10-01T12:00:00Z"),
...overrides,
};
}
const blockCodes = (report: GateReport) =>
report.issues.filter((i) => i.severity === "block").map((i) => i.code);
describe("metadata", () => {
it("allows only harmless container tags", () => {
expect(METADATA_ALLOWED_TAGS).toContain("title");
expect(findMetadataLeaks(cleanTags)).toEqual([]);
expect(
findMetadataLeaks({
...cleanTags,
"format.creation_time": "2026-01-01T00:00:00Z",
"format.location": "+00.0000+000.0000/",
"format.com.apple.quicktime.make": "Example",
"format.com.apple.quicktime.location.ISO6709": "+00+000/",
"format.make": "Example",
"format.model": "Example",
"format.software": "Example 1.0",
"stream.0.creation_time": "2026-01-01T00:00:00Z",
GPSLatitude: "0",
"format.encoder": "Lavf60.16.100",
"stream:0:encoder": "x264 core 164",
}),
).toEqual([
"GPSLatitude",
"format.com.apple.quicktime.location.ISO6709",
"format.com.apple.quicktime.make",
"format.creation_time",
"format.encoder",
"format.location",
"format.make",
"format.model",
"format.software",
"stream.0.creation_time",
"stream:0:encoder",
]);
expect(findMetadataLeaks({ title: "x", major_brand: "isom" })).toEqual([]);
});
});
describe("evaluateGate", () => {
it("passes a clean render and produces a schema-valid report", () => {
const report = evaluateGate(gateInput());
expect(report.issues).toEqual([]);
expect(report.passed).toBe(true);
expect(GateReportSchema.parse(report)).toEqual(report);
expect(report.checkedAt).toBe("2026-10-01T12:00:00.000Z");
});
it("blocks technical problems", () => {
const edl = testEdl();
expect(checkTechnical(testProfile(), edl, probe())).toEqual([]);
const cases: Array<[Partial<FfprobeReport>, string]> = [
[{ width: 1920, height: 1080 }, "resolution_mismatch"],
[{ width: 1920, height: 1080 }, "aspect_not_allowed"],
[{ fps: 25 }, "fps_mismatch"],
[{ durationMs: 70_000 }, "duration_too_long"],
[{ durationMs: 500 }, "duration_too_short"],
[{ sizeBytes: 600_000_000 }, "file_too_large"],
[{ videoCodec: "vp9" }, "codec_mismatch"],
[{ integratedLufs: -20 }, "loudness_off_target"],
[{ integratedLufs: null }, "loudness_unmeasured"],
[{ truePeakDb: 0 }, "true_peak_exceeded"],
[{ tags: { ...cleanTags, "format.creation_time": "2020" } }, "metadata_leak"],
];
for (const [patch, code] of cases) {
const report = evaluateGate(gateInput({ ffprobe: probe(patch) }));
expect(report.passed, code).toBe(false);
expect(blockCodes(report), code).toContain(code);
}
const drift = checkTechnical(testProfile(), edl, probe({ durationMs: 4500 }));
expect(drift).toEqual([expect.objectContaining({ code: "duration_drift", severity: "warn" })]);
expect(
checkTechnical(testProfile(), edl, probe({ audioCodec: null, integratedLufs: null })),
).toEqual([]);
});
it("blocks classifier hits with timestamps, category and score", () => {
const report = evaluateGate(
gateInput({
content: [
{ tMs: 0, scores: { explicit_nudity: 0.05 } },
{ tMs: 1500, scores: { explicit_nudity: 0.4 } },
{ tMs: 2000, scores: { explicit_nudity: 0.6 } },
{ tMs: 3500, scores: { violence: 0.9, drugs: 1 } },
],
}),
);
expect(report.passed).toBe(false);
expect(report.issues).toEqual([
expect.objectContaining({
check: "content",
code: "content_threshold",
category: "explicit_nudity",
atMs: 1500,
endMs: 2000,
score: 0.6,
}),
expect.objectContaining({ category: "violence", atMs: 3500, severity: "block" }),
]);
});
it("only blocks above the ratio when the profile says so", () => {
const profile = testProfile({
content: { thresholds: { violence: 0.5 }, blockOn: "ratio", blockRatio: 0.25 },
});
const samples = Array.from({ length: 10 }, (_, i) => ({
tMs: i * 500,
scores: { violence: i < 2 ? 0.9 : 0 },
}));
const below = evaluateGate(gateInput({ profile, content: samples }));
expect(below.passed).toBe(true);
expect(below.issues[0]).toMatchObject({ severity: "warn", category: "violence" });
const above = samples.map((s, i) => ({ ...s, scores: { violence: i < 4 ? 0.9 : 0 } }));
expect(evaluateGate(gateInput({ profile, content: above })).passed).toBe(false);
});
it("blocks anonymization gaps, uncovered and unknown faces, and visible marks", () => {
const people = [
{ id: "person-1", anonymizeByDefault: "blur", consentRecordedAt: null, ageVerifiedAt: null },
{ id: "person-2", anonymizeByDefault: "none", consentRecordedAt: null, ageVerifiedAt: null },
];
const box = { x: 0.4, y: 0.2, w: 0.1, h: 0.1 };
const report = evaluateGate(
gateInput({
people,
faces: [
{ tMs: 100, box, personId: "person-1", coverage: 1 },
{ tMs: 200, box, personId: "person-1", coverage: 0.9 },
{ tMs: 300, box, personId: "person-2", coverage: 0 },
{ tMs: 400, box, personId: null, coverage: 0 },
],
trackingGaps: [
{ startMs: 1000, endMs: 1400, personId: "person-1" },
{ startMs: 2000, endMs: 2600, personId: "person-1" },
],
marks: [
{ markId: "mark-1", tMs: 100, residualVisibility: 0.01 },
{ markId: "mark-1", tMs: 300, residualVisibility: 0.3 },
],
}),
);
expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([
expect.objectContaining({ code: "face_not_covered", personId: "person-1", atMs: 200 }),
expect.objectContaining({ code: "unknown_face", atMs: 400 }),
expect.objectContaining({
code: "tracking_gap",
atMs: 2000,
endMs: 2600,
personId: "person-1",
}),
expect.objectContaining({ code: "mark_visible", markId: "mark-1", atMs: 300 }),
]);
// Without protected people an unmatched face is not an anonymization issue.
const open = evaluateGate(gateInput({ faces: [{ tMs: 0, box, personId: null, coverage: 0 }] }));
expect(open.passed).toBe(true);
});
it("requires consent and age verification only where the profile does", () => {
const people = [
{
id: "p-ok",
anonymizeByDefault: "none",
consentRecordedAt: "2026-01-01T00:00:00Z",
ageVerifiedAt: "2026-01-01T00:00:00Z",
},
{
id: "p-no-consent",
anonymizeByDefault: "none",
consentRecordedAt: null,
ageVerifiedAt: "2026-01-01T00:00:00Z",
},
{
id: "p-no-age",
anonymizeByDefault: "none",
consentRecordedAt: "2026-01-01T00:00:00Z",
ageVerifiedAt: null,
},
];
const faces = [{ tMs: 1000, box: { x: 0, y: 0, w: 0.1, h: 0.1 }, personId: null, coverage: 0 }];
expect(evaluateGate(gateInput({ people, faces })).passed).toBe(true);
const profile = testPaidProfile();
const edl = testEdl({
output: { ...testEdl().output, profileId: profile.id, width: 1920, height: 1080 },
});
const report = evaluateGate(
gateInput({ profile, edl, people, faces, ffprobe: probe({ width: 1920, height: 1080 }) }),
);
expect(report.issues.filter((i) => i.check === "consent")).toEqual([
expect.objectContaining({ code: "age_unverified", personId: "p-no-age" }),
expect.objectContaining({ code: "consent_missing", personId: "p-no-consent" }),
expect.objectContaining({ code: "unknown_performer", atMs: 1000 }),
]);
expect(report.passed).toBe(false);
});
it("requires the AI label when the profile does and warns about the watermark", () => {
const report = evaluateGate(
gateInput({
aiAlterations: ["synthetic_identity"],
watermarkBox: { x: 0.9, y: 0.01, w: 0.1, h: 0.05 },
}),
);
expect(report.issues).toEqual([
expect.objectContaining({ check: "disclosure", code: "ai_label_missing", severity: "block" }),
expect.objectContaining({
check: "brand",
code: "watermark_outside_safe_zone",
severity: "warn",
}),
]);
expect(
evaluateGate(gateInput({ aiAlterations: ["inpaint"], bundleHasDisclosure: true })).passed,
).toBe(true);
const edl = testEdl({ metadata: { aiAltered: true, aiAlterations: ["voice_disguise"] } });
expect(blockCodes(evaluateGate(gateInput({ edl })))).toEqual(["ai_label_missing"]);
const relaxed = testProfile({ disclosure: { aiAlteredLabelRequired: false } });
expect(evaluateGate(gateInput({ profile: relaxed, edl })).passed).toBe(true);
});
});
describe("checkCopy", () => {
const profile = testProfile({
text: {
titleMaxChars: 20,
captionMaxChars: 60,
hashtagLimit: 2,
linkPolicy: "bio_only",
bannedPatterns: [
{ pattern: "free\\s+money", reason: "misleading claim" },
{ pattern: "([a-z", reason: "broken pattern" },
],
},
});
it("checks banned patterns case-insensitively and reports invalid ones without crashing", () => {
const issues = checkCopy(profile, {
title: "FREE Money",
caption: null,
tags: ["free money"],
});
expect(issues.map((i) => [i.code, i.message])).toEqual([
["banned_pattern", "title: misleading claim"],
["banned_pattern", "tags[0]: misleading claim"],
["invalid_banned_pattern", "profile pattern cannot be checked: broken pattern"],
]);
});
it("checks lengths, hashtags and links", () => {
const codes = (copy: { title: string | null; caption: string | null; tags: string[] }) =>
checkCopy(profile, copy)
.map((i) => i.code)
.filter((c) => c !== "invalid_banned_pattern");
expect(codes({ title: "x".repeat(21), caption: "y".repeat(61), tags: [] })).toEqual([
"title_too_long",
"caption_too_long",
]);
expect(codes({ title: null, caption: "#a #b", tags: ["#c"] })).toEqual(["too_many_hashtags"]);
expect(codes({ title: null, caption: "#a #A", tags: ["a"] })).toEqual([]);
expect(codes({ title: null, caption: "see https://example.com/x", tags: [] })).toEqual([
"link_bio_only",
]);
expect(codes({ title: null, caption: "visit example.shop today", tags: [] })).toEqual([
"link_bio_only",
]);
expect(codes({ title: "Link in bio", caption: "Ready? Go.", tags: [] })).toEqual([]);
const forbidden = testProfile({
text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "forbidden" },
});
expect(
checkCopy(forbidden, { title: "www.example.com", caption: null, tags: [] })[0]!.code,
).toBe("link_forbidden");
const allowed = testProfile({
text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "allowed" },
});
expect(checkCopy(allowed, { title: null, caption: "https://example.com", tags: [] })).toEqual(
[],
);
});
});
describe("planAutoFix", () => {
const edl = testEdl({
tracks: {
video: [
{
id: "c1",
assetId: "asset-a",
inMs: 0,
outMs: 2000,
atMs: 0,
reframe: "auto",
segmentId: "s1",
},
{
id: "c2",
assetId: "asset-a",
inMs: 5000,
outMs: 7000,
atMs: 2000,
reframe: "auto",
segmentId: "s2",
},
{
id: "c3",
assetId: "asset-a",
inMs: 8000,
outMs: 10_000,
atMs: 4000,
reframe: "auto",
segmentId: "s3",
},
],
},
transitions: [{ fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 }],
overlays: [
{ id: "wm", kind: "watermark", atMs: 0, durationMs: 6000 },
{ id: "cta", kind: "cta_end_card", atMs: 4500, durationMs: 1500, text: "Link in bio" },
],
redactions: {
faces: [{ id: "f1", target: { personId: "person-1" }, mode: "blur", paddingPct: 20 }],
marks: [{ markId: "mark-1", treatment: "inpaint" }],
},
metadata: { aiAltered: true, aiAlterations: ["inpaint"] },
});
const report = (issues: GateReport["issues"], round = 0): GateReport => ({
profileId: "example-short-vertical",
passed: !issues.some((i) => i.severity === "block"),
checkedAt: "2026-10-01T12:00:00.000Z",
sampledFrames: 12,
issues,
autoFixRound: round,
});
const contentHit = (atMs: number, endMs = atMs) =>
({
check: "content",
severity: "block",
code: "content_threshold",
message: "violence",
atMs,
endMs,
category: "violence",
score: 0.9,
}) as const;
const assertNoNewOverlays = (fixed: typeof edl) => {
const before = new Map(edl.overlays.map((o) => [o.id, o]));
for (const overlay of fixed.overlays) {
const original = before.get(overlay.id);
expect(original, `overlay ${overlay.id} was added`).toBeDefined();
expect(overlay.kind).toBe(original!.kind);
expect(overlay.text).toBe(original!.text);
expect(overlay.durationMs).toBeLessThanOrEqual(original!.durationMs);
}
expect(fixed.overlays.length).toBeLessThanOrEqual(edl.overlays.length);
};
it("swaps an offending clip for a clean alternative segment", () => {
const fix = planAutoFix(edl, report([contentHit(2500, 3000)]), [
{ segmentId: "s2", assetId: "asset-a", startMs: 5000, endMs: 7000 },
{ segmentId: "s9", assetId: "asset-a", startMs: 5500, endMs: 9000 },
{ segmentId: "s8", assetId: "asset-b", startMs: 1000, endMs: 4000 },
])!;
expect(fix.changes).toEqual(["clip c2: swapped to segment s8 (content_threshold)"]);
expect(fix.edl.tracks.video.find((c) => c.id === "c2")).toMatchObject({
assetId: "asset-b",
inMs: 1000,
outMs: 3000,
segmentId: "s8",
atMs: 2000,
});
expect(edlDurationMs(fix.edl)).toBe(6000);
assertNoNewOverlays(fix.edl);
});
it("trims or drops the clip when no alternative fits, and repacks the timeline", () => {
const trimmed = planAutoFix(edl, report([contentHit(3800)]), [])!;
expect(trimmed.changes).toEqual(["clip c2: trimmed the end at 6300 ms (content_threshold)"]);
expect(trimmed.edl.tracks.video.map((c) => [c.id, c.atMs, c.inMs, c.outMs])).toEqual([
["c1", 0, 0, 2000],
["c2", 2000, 5000, 6300],
["c3", 3300, 8000, 10_000],
]);
assertNoNewOverlays(trimmed.edl);
expect(trimmed.edl.overlays.find((o) => o.id === "wm")!.durationMs).toBe(5300);
const dropped = planAutoFix(edl, report([contentHit(2300, 3700)]), [])!;
expect(dropped.changes).toEqual(["clip c2: removed (content_threshold)"]);
expect(dropped.edl.tracks.video.map((c) => c.id)).toEqual(["c1", "c3"]);
expect(dropped.edl.transitions).toEqual([]);
assertNoNewOverlays(dropped.edl);
});
it("cuts tracking gaps, widens face redactions, adds a blur for unknown faces, and covers marks", () => {
const fix = planAutoFix(
edl,
report([
{
check: "anonymization",
severity: "block",
code: "face_not_covered",
message: "",
atMs: 100,
personId: "person-1",
},
{ check: "anonymization", severity: "block", code: "unknown_face", message: "", atMs: 300 },
{
check: "anonymization",
severity: "block",
code: "mark_visible",
message: "",
atMs: 300,
markId: "mark-1",
},
{
check: "anonymization",
severity: "block",
code: "tracking_gap",
message: "",
atMs: 4100,
endMs: 4800,
},
]),
[],
)!;
expect(fix.edl.redactions.faces).toEqual([
expect.objectContaining({ id: "f1", paddingPct: 45 }),
expect.objectContaining({
id: "fix-face-unassigned",
target: { unassignedFaces: true },
mode: "blur",
}),
]);
expect(fix.edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "cover" }]);
expect(fix.edl.metadata.aiAlterations).toEqual([]);
expect(fix.edl.tracks.video.find((c) => c.id === "c3")).toMatchObject({ inMs: 9300 });
assertNoNewOverlays(fix.edl);
});
it("stops at the round limit, on passing reports, and when nothing safe applies", () => {
expect(planAutoFix(edl, report([contentHit(2500)], MAX_AUTO_FIX_ROUNDS), [])).toBeNull();
expect(planAutoFix(edl, report([]), [])).toBeNull();
const textOnly = report([
{ check: "text", severity: "block", code: "banned_pattern", message: "" },
]);
expect(planAutoFix(edl, textOnly, [])).toBeNull();
const consent = report([
{ check: "consent", severity: "block", code: "consent_missing", message: "", personId: "p" },
]);
expect(planAutoFix(edl, consent, [])).toBeNull();
});
it("runs at most three rounds in a loop", () => {
let current = edl;
let rounds = 0;
for (let round = 0; round < 10; round++) {
const fix = planAutoFix(current, report([contentHit(500)], round), []);
if (!fix) break;
current = fix.edl;
rounds += 1;
assertNoNewOverlays(current);
}
expect(rounds).toBe(MAX_AUTO_FIX_ROUNDS);
});
});

650
packages/media/src/gate.ts
View File

@ -2,16 +2,74 @@ import type {
AiAlteration, AiAlteration,
ContentCategory, ContentCategory,
Edl, Edl,
EdlVideoClip,
FaceRedaction,
FfprobeReport, FfprobeReport,
GateIssue, GateIssue,
GateReport, GateReport,
NormalizedBox, NormalizedBox,
PlatformProfile, PlatformProfile,
} from "@rakazo/contracts"; } from "@rakazo/contracts";
import { notImplemented } from "./not-implemented.js"; import { CONTENT_CATEGORIES, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts";
import {
BLENDED_TRANSITIONS,
boxInsideSafeZone,
clipLengthMs,
edlDurationMs,
locateOutputTimeAll,
matchesAspect,
sortedVideoClips,
} from "./edl.js";
/** Tags a clean export may carry. Everything else is a metadata leak. */ /**
export const METADATA_ALLOWED_TAGS: readonly string[] = []; * Tags a clean export may carry. Everything else is a metadata leak.
*
* - `title`: the only descriptive tag the compiler writes, and only when set.
* - `major_brand`, `minor_version`, `compatible_brands`: the MP4 file-type
* box, always written by the muxer; it names the format, not the author.
* - `handler_name`, `vendor_id`, `language`: per-stream boilerplate the MP4
* muxer writes ("VideoHandler", "[0][0][0][0]", "und").
* - `encoder`: the MP4 sample description always names the codec. It is
* allowed only without a version number (bitexact output writes
* "Lavc libx264"); a versioned value such as "Lavf60.16.100" is a leak.
*
* `creation_time`, `location`, `com.apple.quicktime.*`, `make`, `model`,
* `software`, GPS keys and anything unknown always count as leaks.
*/
export const METADATA_ALLOWED_TAGS: readonly string[] = [
"title",
"major_brand",
"minor_version",
"compatible_brands",
"handler_name",
"vendor_id",
"language",
"encoder",
];
const ALLOWED = new Set(METADATA_ALLOWED_TAGS);
/** "Lavf60.16.100", "Lavc60.31", "x264 core 164", "HandBrake 1.7.0". */
const VERSIONED = /\d+\.\d+|lav[cf]\d|core\s*\d/i;
/** Tag name without a "format." / "stream.0." style prefix, lower-cased. */
function tagName(key: string): string {
return key
.trim()
.toLowerCase()
.replace(/^(?:format|streams?(?:[.:/]\d+)?)[.:/]/, "");
}
export function findMetadataLeaks(tags: Record<string, string>): string[] {
return Object.entries(tags)
.filter(([key, value]) => {
const name = tagName(key);
if (!ALLOWED.has(name)) return true;
if (name === "encoder") return VERSIONED.test(value);
return false;
})
.map(([key]) => key)
.sort();
}
export interface GateInput { export interface GateInput {
profile: PlatformProfile; profile: PlatformProfile;
@ -44,28 +102,370 @@ export interface GateInput {
now: Date; now: Date;
} }
/** Deterministic gate (spec 6). Any blocking issue fails the report. */ /** Longest tracking gap the anonymizer may bridge (spec 7). */
export function evaluateGate(_input: GateInput): GateReport { export const MAX_TRACKING_GAP_MS = 500;
return notImplemented("evaluateGate"); /** Residual visibility of a treated mark above which the gate blocks. */
export const MAX_MARK_RESIDUAL = 0.05;
/** Loudness tolerance around the target, in LU. */
export const LOUDNESS_TOLERANCE_LU = 2;
const TRUE_PEAK_TOLERANCE_DB = 0.5;
const ms = (value: number) => Math.max(0, Math.round(value));
/** Merge time-sorted hits into ranges when they are at most `gapMs` apart. */
function ranges<T extends { tMs: number }>(hits: T[], gapMs: number): T[][] {
const sorted = [...hits].sort((a, b) => a.tMs - b.tMs);
const out: T[][] = [];
for (const hit of sorted) {
const last = out.at(-1);
if (last && hit.tMs - last.at(-1)!.tMs <= gapMs) last.push(hit);
else out.push([hit]);
}
return out;
} }
export function checkTechnical( export function checkTechnical(
_profile: PlatformProfile, profile: PlatformProfile,
_edl: Edl, edl: Edl,
_probe: FfprobeReport, probe: FfprobeReport,
): GateIssue[] { ): GateIssue[] {
return notImplemented("checkTechnical"); const issues: GateIssue[] = [];
const block = (code: string, message: string) =>
issues.push({ check: "technical", severity: "block", code, message });
if (probe.width !== edl.output.width || probe.height !== edl.output.height) {
block(
"resolution_mismatch",
`output is ${probe.width}x${probe.height}, plan expects ${edl.output.width}x${edl.output.height}`,
);
}
if (!matchesAspect(probe.width, probe.height, profile.video.aspectRatios)) {
block("aspect_not_allowed", `${probe.width}x${probe.height} is not an allowed aspect ratio`);
}
if (Math.max(probe.width, probe.height) > Math.max(profile.video.width, profile.video.height)) {
block("resolution_too_high", "output is larger than the profile allows");
}
if (
Math.abs(probe.fps - edl.output.fps) > edl.output.fps * 0.01 ||
probe.fps > profile.video.fps * 1.001
) {
block("fps_mismatch", `output runs at ${probe.fps} fps`);
}
const frameMs = 1000 / Math.max(1, edl.output.fps);
const expected = edlDurationMs(edl);
if (probe.durationMs < profile.duration.hardMinMs - frameMs) {
block("duration_too_short", `${probe.durationMs} ms is below ${profile.duration.hardMinMs} ms`);
}
if (probe.durationMs > profile.duration.hardMaxMs + frameMs) {
block("duration_too_long", `${probe.durationMs} ms is above ${profile.duration.hardMaxMs} ms`);
}
if (Math.abs(probe.durationMs - expected) > Math.max(2 * frameMs, 100)) {
issues.push({
check: "technical",
severity: "warn",
code: "duration_drift",
message: `output is ${probe.durationMs} ms, plan is ${expected} ms`,
});
}
if (probe.sizeBytes > profile.video.maxFileSizeMb * 1_000_000) {
block("file_too_large", `file is larger than ${profile.video.maxFileSizeMb} MB`);
}
const codec = probe.videoCodec.toLowerCase();
const expectedCodec = profile.video.codec === "hevc" ? ["hevc", "h265"] : ["h264", "avc1"];
if (!expectedCodec.includes(codec)) {
block("codec_mismatch", `video codec ${probe.videoCodec} is not ${profile.video.codec}`);
}
if (probe.audioCodec !== null && probe.audioCodec.toLowerCase() !== "aac") {
issues.push({
check: "technical",
severity: "warn",
code: "audio_codec",
message: `audio codec ${probe.audioCodec} is not AAC`,
});
}
const target = edl.targetLufs ?? profile.audio.targetLufs;
if (probe.audioCodec !== null) {
if (probe.integratedLufs === null) {
block("loudness_unmeasured", "loudness was not measured");
} else if (Math.abs(probe.integratedLufs - target) > LOUDNESS_TOLERANCE_LU) {
block("loudness_off_target", `${probe.integratedLufs} LUFS, target ${target} LUFS`);
}
if (
probe.truePeakDb !== null &&
probe.truePeakDb > profile.audio.truePeakDb + TRUE_PEAK_TOLERANCE_DB
) {
block("true_peak_exceeded", `true peak ${probe.truePeakDb} dBTP`);
}
}
const leaks = findMetadataLeaks(probe.tags);
if (leaks.length > 0) block("metadata_leak", `metadata not removed: ${leaks.join(", ")}`);
return issues;
} }
export function findMetadataLeaks(_tags: Record<string, string>): string[] { const URL_PATTERN =
return notImplemented("findMetadataLeaks"); /\b(?:https?:\/\/|www\.)\S+|\b[a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:com|net|org|io|de|at|ch|eu|ly|co|me|link|shop|store|app|tv|gg|xyz)\b(?:\/\S*)?/i;
const HASHTAG_PATTERN = /#[\p{L}\p{N}_]+/gu;
/** Profile patterns compile case-insensitively; an invalid one is reported, never thrown. */
function compilePattern(pattern: string): RegExp | null {
try {
return new RegExp(pattern, "i");
} catch {
return null;
}
} }
export function checkCopy( export function checkCopy(
_profile: PlatformProfile, profile: PlatformProfile,
_copy: { title: string | null; caption: string | null; tags: string[] }, copy: { title: string | null; caption: string | null; tags: string[] },
): GateIssue[] { ): GateIssue[] {
return notImplemented("checkCopy"); const issues: GateIssue[] = [];
const block = (code: string, message: string) =>
issues.push({ check: "text", severity: "block", code, message });
const { text } = profile;
const title = copy.title ?? "";
const caption = copy.caption ?? "";
if (text.titleMaxChars !== undefined && [...title].length > text.titleMaxChars) {
block(
"title_too_long",
`title has ${[...title].length} characters, limit ${text.titleMaxChars}`,
);
}
if ([...caption].length > text.captionMaxChars) {
block(
"caption_too_long",
`caption has ${[...caption].length} characters, limit ${text.captionMaxChars}`,
);
}
const hashtags = new Set<string>();
for (const tag of copy.tags) {
const clean = tag.replace(/^#+/, "").trim().toLowerCase();
if (clean) hashtags.add(clean);
}
for (const match of caption.matchAll(HASHTAG_PATTERN))
hashtags.add(match[0].slice(1).toLowerCase());
if (hashtags.size > text.hashtagLimit) {
block("too_many_hashtags", `${hashtags.size} hashtags, limit ${text.hashtagLimit}`);
}
const fields: Array<[string, string]> = [
["title", title],
["caption", caption],
...copy.tags.map((tag, i): [string, string] => [`tags[${i}]`, tag]),
];
for (const banned of text.bannedPatterns) {
const regex = compilePattern(banned.pattern);
if (!regex) {
block("invalid_banned_pattern", `profile pattern cannot be checked: ${banned.reason}`);
continue;
}
for (const [field, value] of fields) {
if (value && regex.test(value)) block("banned_pattern", `${field}: ${banned.reason}`);
}
}
if (text.linkPolicy !== "allowed") {
for (const [field, value] of fields) {
if (!value || !URL_PATTERN.test(value)) continue;
if (text.linkPolicy === "forbidden") block("link_forbidden", `${field} contains a link`);
else block("link_bio_only", `${field} contains a link; links belong in the bio`);
}
}
return issues;
}
function checkContent(input: GateInput): GateIssue[] {
const { profile } = input;
const issues: GateIssue[] = [];
const thresholds = profile.content.thresholds;
const gapMs = (1000 / profile.content.sampleFps) * 1.5;
const sampled = Math.max(input.content.length, input.sampledFrames, 1);
for (const category of CONTENT_CATEGORIES) {
const threshold = thresholds[category];
if (threshold === undefined) continue;
const hits = input.content.flatMap((sample) => {
const score = sample.scores[category];
return score !== undefined && score > threshold ? [{ tMs: sample.tMs, score }] : [];
});
if (hits.length === 0) continue;
const blocking =
profile.content.blockOn === "any_frame" ||
hits.length / sampled > (profile.content.blockRatio ?? 0);
for (const range of ranges(hits, gapMs)) {
const score = Math.max(...range.map((h) => h.score));
issues.push({
check: "content",
severity: blocking ? "block" : "warn",
code: "content_threshold",
message: `${category} ${score.toFixed(2)} above ${threshold}`,
atMs: ms(range[0]!.tMs),
endMs: ms(range.at(-1)!.tMs),
category,
score,
});
}
}
return issues;
}
function checkAnonymization(input: GateInput): GateIssue[] {
const issues: GateIssue[] = [];
const protectedIds = new Set(
input.people.filter((p) => p.anonymizeByDefault !== "none").map((p) => p.id),
);
const gapMs = 1000;
const uncovered = input.faces.filter(
(face) => face.personId !== null && protectedIds.has(face.personId) && face.coverage < 1,
);
const byPerson = new Map<string, typeof uncovered>();
for (const face of uncovered) {
const list = byPerson.get(face.personId!) ?? [];
list.push(face);
byPerson.set(face.personId!, list);
}
for (const [personId, faces] of [...byPerson.entries()].sort((a, b) =>
a[0].localeCompare(b[0]),
)) {
for (const range of ranges(faces, gapMs)) {
const coverage = Math.min(...range.map((f) => f.coverage));
issues.push({
check: "anonymization",
severity: "block",
code: "face_not_covered",
message: `face covered ${(coverage * 100).toFixed(0)} %`,
atMs: ms(range[0]!.tMs),
endMs: ms(range.at(-1)!.tMs),
personId,
score: coverage,
});
}
}
if (protectedIds.size > 0) {
const unknown = input.faces.filter((face) => face.personId === null);
for (const range of ranges(unknown, gapMs)) {
issues.push({
check: "anonymization",
severity: "block",
code: "unknown_face",
message: "a face that cannot be matched appears next to a protected person",
atMs: ms(range[0]!.tMs),
endMs: ms(range.at(-1)!.tMs),
});
}
}
for (const gap of input.trackingGaps) {
if (gap.endMs - gap.startMs <= MAX_TRACKING_GAP_MS) continue;
issues.push({
check: "anonymization",
severity: "block",
code: "tracking_gap",
message: `face tracking lost for ${Math.round(gap.endMs - gap.startMs)} ms (needs review)`,
atMs: ms(gap.startMs),
endMs: ms(gap.endMs),
...(gap.personId ? { personId: gap.personId } : {}),
});
}
const visibleMarks = input.marks.filter((mark) => mark.residualVisibility > MAX_MARK_RESIDUAL);
const byMark = new Map<string, typeof visibleMarks>();
for (const mark of visibleMarks) {
const list = byMark.get(mark.markId) ?? [];
list.push(mark);
byMark.set(mark.markId, list);
}
for (const [markId, marks] of [...byMark.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
for (const range of ranges(marks, gapMs)) {
const residual = Math.max(...range.map((m) => m.residualVisibility));
issues.push({
check: "anonymization",
severity: "block",
code: "mark_visible",
message: `treated mark still ${(residual * 100).toFixed(0)} % visible`,
atMs: ms(range[0]!.tMs),
endMs: ms(range.at(-1)!.tMs),
markId,
score: residual,
});
}
}
return issues;
}
function checkConsent(input: GateInput): GateIssue[] {
const { consent } = input.profile;
if (!consent.requirePerformerConsent && !consent.requireAgeVerification) return [];
const issues: GateIssue[] = [];
for (const person of [...input.people].sort((a, b) => a.id.localeCompare(b.id))) {
if (consent.requirePerformerConsent && !person.consentRecordedAt) {
issues.push({
check: "consent",
severity: "block",
code: "consent_missing",
message: "no recorded consent",
personId: person.id,
});
}
if (consent.requireAgeVerification && !person.ageVerifiedAt) {
issues.push({
check: "consent",
severity: "block",
code: "age_unverified",
message: "age is not verified",
personId: person.id,
});
}
}
for (const range of ranges(
input.faces.filter((face) => face.personId === null),
1000,
)) {
issues.push({
check: "consent",
severity: "block",
code: "unknown_performer",
message: "a visible face is not assigned to a person with consent",
atMs: ms(range[0]!.tMs),
endMs: ms(range.at(-1)!.tMs),
});
}
return issues;
}
/** Deterministic gate (spec 6). Any blocking issue fails the report. */
export function evaluateGate(input: GateInput): GateReport {
const { profile } = input;
const issues: GateIssue[] = [
...checkTechnical(profile, input.edl, input.ffprobe),
...checkContent(input),
...checkAnonymization(input),
...checkConsent(input),
...checkCopy(profile, input.copy),
];
const alterations = new Set([...input.aiAlterations, ...input.edl.metadata.aiAlterations]);
if (
alterations.size > 0 &&
profile.disclosure.aiAlteredLabelRequired &&
!input.bundleHasDisclosure
) {
issues.push({
check: "disclosure",
severity: "block",
code: "ai_label_missing",
message: `AI alterations (${[...alterations].sort().join(", ")}) need the platform label`,
});
}
if (input.watermarkBox && !boxInsideSafeZone(input.watermarkBox, profile)) {
issues.push({
check: "brand",
severity: "warn",
code: "watermark_outside_safe_zone",
message: "watermark is covered by platform interface",
});
}
return {
profileId: profile.id,
passed: !issues.some((issue) => issue.severity === "block"),
checkedAt: input.now.toISOString(),
sampledFrames: Math.max(0, Math.round(input.sampledFrames)),
issues,
autoFixRound: Math.min(MAX_AUTO_FIX_ROUNDS, Math.max(0, Math.round(input.autoFixRound))),
};
} }
export interface AutoFixResult { export interface AutoFixResult {
@ -73,15 +473,227 @@ export interface AutoFixResult {
changes: string[]; changes: string[];
} }
const PADDING_STEP = 25;
const CUT_MARGIN_MS = 500;
const MIN_KEEP_MS = 500;
/** Lay clips out back to back again, keeping blended transitions' overlap. */
function repack(edl: Edl): Edl {
const clips = sortedVideoClips(edl);
const ids = new Set(clips.map((c) => c.id));
const transitions = edl.transitions.filter((t) => ids.has(t.fromClipId) && ids.has(t.toClipId));
const adjacent = new Set<string>();
for (let i = 0; i + 1 < clips.length; i++) adjacent.add(`${clips[i]!.id}>${clips[i + 1]!.id}`);
const kept = transitions.filter((t) => adjacent.has(`${t.fromClipId}>${t.toClipId}`));
const blend = new Map(
kept
.filter((t) => BLENDED_TRANSITIONS.has(t.kind) && t.durationMs > 0)
.map((t) => [`${t.fromClipId}>${t.toClipId}`, t.durationMs]),
);
let cursor = 0;
const packed: EdlVideoClip[] = clips.map((clip, i) => {
const previous = clips[i - 1];
let overlap = previous ? (blend.get(`${previous.id}>${clip.id}`) ?? 0) : 0;
overlap = Math.min(overlap, clipLengthMs(clip) - 1, cursor);
const atMs = Math.max(0, cursor - Math.max(0, overlap));
cursor = atMs + clipLengthMs(clip);
return { ...clip, atMs };
});
const regions = edl.redactions.regions.filter((r) => ids.has(r.clipId));
const next = {
...edl,
tracks: { ...edl.tracks, video: packed },
transitions: kept.filter((t) => {
if (!BLENDED_TRANSITIONS.has(t.kind)) return true;
const from = packed.find((c) => c.id === t.fromClipId)!;
const to = packed.find((c) => c.id === t.toClipId)!;
return t.durationMs < Math.min(clipLengthMs(from), clipLengthMs(to));
}),
redactions: { ...edl.redactions, regions },
};
// Overlays never grow or move; they are only clamped to the new end.
const total = edlDurationMs(next);
next.overlays = edl.overlays.flatMap((overlay) => {
if (overlay.atMs >= total) return [];
return [{ ...overlay, durationMs: Math.min(overlay.durationMs, total - overlay.atMs) }];
});
return next;
}
/** /**
* Compliance-only repairs: trim or drop offending clips, change crop, widen * Compliance-only repairs: trim or drop offending clips, change crop, widen
* redaction padding, swap to another segment. Never adds obfuscation aimed at * redaction padding, swap to another segment. Never adds obfuscation aimed at
* classifiers. Returns null when the round limit is reached or nothing safe applies. * classifiers. Returns null when the round limit is reached or nothing safe applies.
*
* - Content blocks and tracking gaps: the clip showing that moment is swapped
* for an alternative segment of at least the same length that does not
* overlap the offending source range, else trimmed (when the range sits at
* one end) or dropped. The timeline is then repacked.
* - Uncovered faces: padding of the person's face redaction grows by 25
* points; a missing redaction is added as blur.
* - Unknown faces: the unassigned-face redaction grows or is added as blur.
* - Visible marks: the mark treatment falls back to cover.
*/ */
export function planAutoFix( export function planAutoFix(
_edl: Edl, edl: Edl,
_report: GateReport, report: GateReport,
_alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>, alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>,
): AutoFixResult | null { ): AutoFixResult | null {
return notImplemented("planAutoFix"); if (report.autoFixRound >= MAX_AUTO_FIX_ROUNDS || report.passed) return null;
const blocks = report.issues.filter((issue) => issue.severity === "block");
const changes: string[] = [];
let clips: EdlVideoClip[] = edl.tracks.video.map((clip) => ({ ...clip }));
let faces: FaceRedaction[] = edl.redactions.faces.map((f) => ({ ...f }));
let marks = edl.redactions.marks.map((m) => ({ ...m }));
let clipsChanged = false;
// Source ranges to remove, per clip.
const offending = new Map<string, Array<{ start: number; end: number; reason: string }>>();
for (const issue of blocks) {
const timeline =
issue.check === "content" ||
issue.code === "tracking_gap" ||
(issue.code === "mark_visible" &&
!edl.redactions.marks.some((m) => m.markId === issue.markId));
if (!timeline || issue.atMs === undefined) continue;
const endMs = issue.endMs ?? issue.atMs;
const step = Math.max(1, Math.floor(1000 / edl.output.fps));
const times: number[] = [];
for (let t = issue.atMs; t < endMs; t += step) times.push(t);
times.push(endMs);
for (const t of times) {
for (const hit of locateOutputTimeAll(edl, t)) {
const list = offending.get(hit.clipId) ?? [];
const last = list.at(-1);
if (last && last.reason === issue.code && hit.sourceMs - last.end <= step * 4) {
last.end = Math.max(last.end, hit.sourceMs);
} else {
list.push({ start: hit.sourceMs, end: hit.sourceMs, reason: issue.code });
}
offending.set(hit.clipId, list);
}
}
}
const usedAlternatives = new Set<string>();
for (const [clipId, found] of [...offending.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
const clip = clips.find((c) => c.id === clipId);
if (!clip) continue;
const start = Math.max(clip.inMs, Math.min(...found.map((r) => r.start)) - CUT_MARGIN_MS);
const end = Math.min(clip.outMs, Math.max(...found.map((r) => r.end)) + CUT_MARGIN_MS);
const length = clip.outMs - clip.inMs;
const alternative = alternatives
.filter(
(a) =>
!usedAlternatives.has(a.segmentId) &&
a.segmentId !== clip.segmentId &&
a.endMs - a.startMs >= length &&
!(a.assetId === clip.assetId && a.startMs < end && start < a.endMs),
)
.sort((a, b) => a.segmentId.localeCompare(b.segmentId))[0];
clipsChanged = true;
if (alternative) {
usedAlternatives.add(alternative.segmentId);
Object.assign(clip, {
assetId: alternative.assetId,
segmentId: alternative.segmentId,
inMs: alternative.startMs,
outMs: alternative.startMs + length,
reframe: clip.reframe === "none" ? "none" : "auto",
});
changes.push(
`clip ${clip.id}: swapped to segment ${alternative.segmentId} (${found[0]!.reason})`,
);
} else if (start - clip.inMs >= MIN_KEEP_MS && end >= clip.outMs - 1) {
clip.outMs = start;
changes.push(`clip ${clip.id}: trimmed the end at ${start} ms (${found[0]!.reason})`);
} else if (clip.outMs - end >= MIN_KEEP_MS && start <= clip.inMs + 1) {
clip.inMs = end;
changes.push(`clip ${clip.id}: trimmed the start to ${end} ms (${found[0]!.reason})`);
} else {
clips = clips.filter((c) => c.id !== clip.id);
changes.push(`clip ${clip.id}: removed (${found[0]!.reason})`);
}
}
for (const clip of clips) {
if (Array.isArray(clip.reframe)) {
const inside = clip.reframe.filter((k) => k.tMs >= clip.inMs && k.tMs <= clip.outMs);
clip.reframe = inside.length > 0 ? inside : "auto";
}
}
const widen = (match: (f: FaceRedaction) => boolean, add: () => FaceRedaction, label: string) => {
const existing = faces.filter(match);
if (existing.length === 0) {
faces = [...faces, add()];
changes.push(`${label}: added blur redaction`);
return;
}
for (const face of existing) {
if (face.paddingPct >= 100) continue;
face.paddingPct = Math.min(100, face.paddingPct + PADDING_STEP);
changes.push(`${label}: redaction ${face.id} padding ${face.paddingPct} %`);
}
};
const widened = new Set<string>();
for (const issue of blocks) {
if (issue.code === "face_not_covered" && issue.personId && !widened.has(issue.personId)) {
const personId = issue.personId;
widened.add(personId);
widen(
(f) => "personId" in f.target && f.target.personId === personId,
() => ({
id: `fix-face-${widened.size}`,
target: { personId },
mode: "blur",
paddingPct: 20 + PADDING_STEP,
featherPct: 10,
minSizePct: 2,
}),
`person ${personId}`,
);
}
if (issue.code === "unknown_face" && !widened.has("(unassigned)")) {
widened.add("(unassigned)");
widen(
(f) => "unassignedFaces" in f.target,
() => ({
id: "fix-face-unassigned",
target: { unassignedFaces: true },
mode: "blur",
paddingPct: 20 + PADDING_STEP,
featherPct: 10,
minSizePct: 2,
}),
"unassigned faces",
);
}
if (issue.code === "mark_visible" && issue.markId) {
const markId = issue.markId;
marks = marks.map((mark) => {
if (mark.markId !== markId || mark.treatment === "cover") return mark;
changes.push(`mark ${markId}: ${mark.treatment} replaced by cover`);
return { ...mark, treatment: "cover" as const };
});
}
}
if (changes.length === 0 || clips.length === 0) return null;
let next: Edl = {
...edl,
tracks: { ...edl.tracks, video: clips },
redactions: { ...edl.redactions, faces, marks },
};
if (clipsChanged) next = repack(next);
const stillAltered = new Set<AiAlteration>(
next.metadata.aiAlterations.filter(
(a) => a !== "inpaint" || marks.some((m) => m.treatment === "inpaint"),
),
);
const explicit = edl.metadata.aiAltered && edl.metadata.aiAlterations.length === 0;
next.metadata = {
...next.metadata,
aiAlterations: [...stillAltered],
aiAltered: stillAltered.size > 0 || explicit,
};
return { edl: next, changes };
} }

320
packages/media/src/marketing.test.ts Normal file
View File

@ -0,0 +1,320 @@
import type { MetricCsvMapping, MotionTemplateInfo } from "@rakazo/contracts";
import { MetricCsvMappingSchema } from "@rakazo/contracts";
import { describe, expect, it } from "vitest";
import {
buildTrackingUrl,
contrastRatio,
introFits,
normalizeHashtags,
parseCsvRecords,
parseMetricsCsv,
parseMoneyToCents,
proposeIntroVariants,
uploadChecklist,
validateBrandKit,
} from "./marketing.js";
import { testKit, testPaidProfile, testProfile } from "./test-fixtures.js";
const utm = { source: "example-short-vertical", medium: "short", campaign: "spring launch" };
describe("buildTrackingUrl", () => {
it("appends UTM parameters and keeps the existing query and fragment", () => {
expect(buildTrackingUrl("https://example.com/shop", utm)).toBe(
"https://example.com/shop?utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch",
);
expect(
buildTrackingUrl("https://example.com/p?ref=a%20b&x=1&utm_source=old#top", {
...utm,
content: "thumb-2",
term: "ä&b",
}),
).toBe(
"https://example.com/p?ref=a%20b&x=1&utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch&utm_content=thumb-2&utm_term=%C3%A4%26b#top",
);
expect(buildTrackingUrl("http://example.com", utm)).toMatch(
/^http:\/\/example\.com\/\?utm_source=/,
);
});
it("rejects non-http schemes and invalid URLs", () => {
expect(() => buildTrackingUrl("javascript:alert(1)", utm)).toThrow(/http/);
expect(() => buildTrackingUrl("ftp://example.com", utm)).toThrow(/http/);
expect(() => buildTrackingUrl("not a url", utm)).toThrow(/valid/);
});
});
describe("normalizeHashtags", () => {
it("normalizes, dedupes and limits", () => {
expect(
normalizeHashtags(["#Summer", "summer", "##new day", "über-cool!", "", "#"], 10),
).toEqual(["#Summer", "#newday", "#übercool"]);
expect(normalizeHashtags(["a", "b", "c"], 2)).toEqual(["#a", "#b"]);
expect(normalizeHashtags(["a"], 0)).toEqual([]);
});
});
describe("CSV", () => {
const mapping = (overrides: Partial<MetricCsvMapping> = {}): MetricCsvMapping =>
MetricCsvMappingSchema.parse({
name: "Example",
columns: {
publication: "Link",
observedAt: "Date",
views: "Views",
clicks: "Clicks",
conversions: "Sales",
revenue: "Revenue",
},
publicationKey: "tracking_url",
...overrides,
});
it("parses RFC 4180 records with quotes, delimiters and newlines in fields", () => {
expect(parseCsvRecords('a,"b,c","say ""hi""\nthere"\r\n1,2,3\n', ",")).toEqual([
["a", "b,c", 'say "hi"\nthere'],
["1", "2", "3"],
]);
expect(parseCsvRecords("x;y\n", ";")).toEqual([["x", "y"]]);
expect(parseCsvRecords("", ",")).toEqual([]);
});
it("imports metrics with BOM, comma delimiter and major-unit revenue", () => {
const csv =
'Link,Date,Views,Clicks,Sales,Revenue\n"https://example.com/?utm_content=a,b",2026-09-01,"1,200",30,2,"19.98"\n';
const result = parseMetricsCsv(csv, mapping());
expect(result.skipped).toEqual([]);
expect(result.rows).toEqual([
{
row: 2,
publicationKey: "https://example.com/?utm_content=a,b",
observedAt: new Date("2026-09-01T00:00:00Z"),
views: 1200,
clicks: 30,
conversions: 2,
revenueCents: 1998,
currency: "EUR",
},
]);
});
it("handles semicolons, tabs, decimal commas, currencies and skipped rows", () => {
const semicolon = [
"link;date;views;clicks;sales;revenue;currency",
"pub-1;01.09.2026;1.500;12;1;1.234,50 €;",
"pub-2;2026/09/02;10;1;0;12,5;usd",
"pub-3;31.02.2026;1;0;0;0;",
"pub-4;2026-09-03;ten;0;0;0;",
";2026-09-03;1;0;0;0;",
"pub-5;2026-09-04;1;0;0;-5;",
"pub-6;2026-09-04;1;0;0;1,2,3;",
"pub-7;2026-09-04;1;0;0;5;EURO",
";;;;;;",
"pub-8;2026-09-05T10:30:00Z;7;;;$3.10;",
].join("\r\n");
const result = parseMetricsCsv(
semicolon,
mapping({
delimiter: ";",
columns: {
publication: "LINK",
observedAt: "date",
views: "views",
clicks: "clicks",
conversions: "sales",
revenue: "revenue",
currency: "currency",
},
}),
);
expect(result.rows.map((r) => [r.publicationKey, r.views, r.revenueCents, r.currency])).toEqual(
[
["pub-1", 1500, 123_450, "EUR"],
["pub-2", 10, 1250, "USD"],
["pub-8", 7, 310, "USD"],
],
);
expect(result.rows[2]!.observedAt).toEqual(new Date("2026-09-05T10:30:00Z"));
expect(result.skipped).toEqual([
{ row: 4, reason: "date is not recognized" },
{ row: 5, reason: "views is not a whole number" },
{ row: 6, reason: "publication is empty" },
{ row: 7, reason: "revenue is negative" },
{ row: 8, reason: "revenue is not a number" },
{ row: 9, reason: "currency is not a 3-letter code" },
]);
const tabs = parseMetricsCsv(
"Link\tDate\tRevenue\nx\t2026-09-01\t250\n",
mapping({
delimiter: "\t",
revenueUnit: "cents",
columns: { publication: "Link", observedAt: "Date", revenue: "Revenue" },
}),
);
expect(tabs.rows[0]).toMatchObject({ views: 0, clicks: 0, conversions: 0, revenueCents: 250 });
});
it("reports missing columns and empty files", () => {
expect(parseMetricsCsv("Other,Date\n", mapping())).toEqual({
rows: [],
skipped: [{ row: 1, reason: "missing columns: Link, Views, Clicks, Sales, Revenue" }],
});
expect(parseMetricsCsv("", mapping()).skipped).toEqual([{ row: 1, reason: "file is empty" }]);
});
it("converts money in either notation to cents", () => {
expect(parseMoneyToCents("1,234.56")).toEqual({ cents: 123_456, currency: null });
expect(parseMoneyToCents("1.234,56")).toEqual({ cents: 123_456, currency: null });
expect(parseMoneyToCents("1,234")).toEqual({ cents: 123_400, currency: null });
expect(parseMoneyToCents("0.995")).toEqual({ cents: 100, currency: null });
expect(parseMoneyToCents("2.5")).toEqual({ cents: 250, currency: null });
expect(parseMoneyToCents("CHF 7")).toEqual({ cents: 700, currency: "CHF" });
expect(parseMoneyToCents("12.345.678,9")).toEqual({ cents: 1_234_567_890, currency: null });
expect(parseMoneyToCents("1,5,0")).toBeNull();
});
});
describe("intro", () => {
const templates: MotionTemplateInfo[] = [
{
id: "intro-glitch",
kind: "intro",
name: "Glitch energy",
minDurationMs: 1000,
maxDurationMs: 5000,
params: [{ name: "primaryColor", type: "color", required: true }],
},
{
id: "intro-elegant",
kind: "intro",
name: "Elegant fade",
minDurationMs: 2000,
maxDurationMs: 3000,
params: [
{ name: "brandName", type: "string", required: true },
{ name: "sound", type: "string", required: false },
],
},
{
id: "intro-kinetic",
kind: "intro",
name: "Kinetic type",
minDurationMs: 1500,
maxDurationMs: 4000,
params: [],
},
{
id: "intro-logo",
kind: "intro",
name: "Minimal logo reveal",
minDurationMs: 1500,
maxDurationMs: 4000,
params: [{ name: "logoArtifactId", type: "string", required: true }],
},
{
id: "outro-a",
kind: "outro",
name: "Outro",
minDurationMs: 1000,
maxDurationMs: 3000,
params: [],
},
];
it("keeps shorts hook-first and allows short intros on long-form profiles", () => {
expect(introFits(testProfile(), 1500)).toBe(false);
expect(introFits(testProfile(), 0)).toBe(true);
expect(introFits(testPaidProfile(), 3000)).toBe(true);
expect(introFits(testPaidProfile(), 5000)).toBe(false);
});
it("proposes three distinct variants ordered by tone", () => {
const kit = testKit({ tone: "calm, premium", logoArtifactIds: ["logo-1"] });
const variants = proposeIntroVariants(kit, templates, 3);
expect(variants.map((v) => v.template.templateId)).toEqual([
"intro-elegant",
"intro-logo",
"intro-glitch",
]);
expect(new Set(variants.map((v) => v.template.templateId)).size).toBe(3);
for (const variant of variants) {
expect(variant.template.durationMs).toBeGreaterThanOrEqual(1500);
expect(variant.template.durationMs).toBeLessThanOrEqual(4000);
}
expect(variants[0]!.template.params).toEqual({ brandName: "Studio Example", sound: "music-1" });
expect(variants[0]!.rationale).toContain('matches the tone "calm, premium"');
expect(variants[1]!.template.params).toEqual({ logoArtifactId: "logo-1" });
expect(proposeIntroVariants(kit, templates, 3)).toEqual(variants);
const energetic = proposeIntroVariants(testKit({ tone: "bold and energetic" }), templates, 3);
expect(energetic[0]!.template).toMatchObject({
templateId: "intro-glitch",
params: { primaryColor: "#112233" },
});
expect(proposeIntroVariants(kit, [], 3)).toEqual([]);
expect(
proposeIntroVariants(kit, templates.slice(0, 1), 3).map((v) => v.template.durationMs),
).toEqual([2500, 3000, 1500]);
});
});
describe("validateBrandKit", () => {
it("flags low contrast, duplicate presets, unlicensed music and fonts", () => {
expect(validateBrandKit(testKit())).toEqual([]);
expect(contrastRatio("#FFFFFF", "#000000")).toBeCloseTo(21, 5);
const preset = testKit().captionPresets[0]!;
const warnings = validateBrandKit(
testKit({
colors: { ...testKit().colors, text: "#777777", background: "#888888" },
captionPresets: [preset, { ...preset, backgroundColor: "#EEEEEE" }],
musicBeds: [
{ artifactId: "m", title: "Borrowed", license: "unknown", commercialUse: false },
],
fonts: [{ family: "Nameless", artifactId: "f", role: "body" }],
}),
);
expect(warnings).toEqual([
"text on background has contrast 1.3:1, below 4.5:1",
'caption preset id "clean" is used twice',
'caption preset "clean" has contrast 1.2:1, below 4.5:1',
'music bed "Borrowed" is not licensed for commercial use',
'font "Nameless" has no license note',
]);
});
});
describe("uploadChecklist", () => {
it("lists the AI label, age restriction, link policy and captions where they apply", () => {
const paid = uploadChecklist({
profile: testPaidProfile(),
aiAltered: true,
hasCaptions: true,
trackingUrl: "https://example.com/?utm_source=x",
});
expect(paid).toContain("# Upload checklist: Example paid video");
expect(paid).toContain("AI-generated or altered content label");
expect(paid).toContain("age-restricted");
expect(paid).toContain("Add the tracking link: <https://example.com/?utm_source=x>");
expect(paid).toContain("SRT or VTT");
const short = uploadChecklist({
profile: testProfile({ captions: { recommended: true, required: true } }),
aiAltered: false,
hasCaptions: false,
trackingUrl: "https://example.com/",
});
expect(short).not.toContain("AI-generated");
expect(short).not.toContain("age-restricted");
expect(short).toContain("in the profile bio, not in the caption");
expect(short).toContain("Captions are required");
expect(short).toContain("at most 3 hashtags");
expect(short.split("\n").filter((l) => l.startsWith("- [ ] "))).toHaveLength(4);
const relaxed = uploadChecklist({
profile: testProfile({ disclosure: { aiAlteredLabelRequired: false } }),
aiAltered: true,
hasCaptions: true,
trackingUrl: null,
});
expect(relaxed).not.toContain("AI-generated");
});
});

510
packages/media/src/marketing.ts
View File

@ -6,15 +6,70 @@ import type {
PlatformProfile, PlatformProfile,
Utm, Utm,
} from "@rakazo/contracts"; } from "@rakazo/contracts";
import { notImplemented } from "./not-implemented.js";
/** Append UTM parameters to an http(s) target. Rejects other schemes. */ const UTM_KEYS: Array<[keyof Utm, string]> = [
export function buildTrackingUrl(_targetUrl: string, _utm: Utm): string { ["source", "utm_source"],
return notImplemented("buildTrackingUrl"); ["medium", "utm_medium"],
["campaign", "utm_campaign"],
["content", "utm_content"],
["term", "utm_term"],
];
/**
* Append UTM parameters to an http(s) target. Rejects other schemes. Existing
* query parameters keep their exact encoding; existing utm_* values for the
* keys being set are replaced, and the fragment stays last.
*/
export function buildTrackingUrl(targetUrl: string, utm: Utm): string {
let url: URL;
try {
url = new URL(targetUrl.trim());
} catch {
throw new Error("tracking target is not a valid URL");
}
if (url.protocol !== "http:" && url.protocol !== "https:") {
throw new Error("tracking target must use http or https");
}
const setting = UTM_KEYS.filter(([key]) => utm[key]?.trim());
const replaced = new Set(setting.map(([, name]) => name));
const kept = url.search
.replace(/^\?/, "")
.split("&")
.filter((part) => {
if (!part) return false;
const rawKey = part.split("=")[0] ?? "";
let key = rawKey;
try {
key = decodeURIComponent(rawKey.replace(/\+/g, " "));
} catch {
// keep undecodable keys as they are
}
return !replaced.has(key);
});
const added = setting.map(
([key, name]) => `${name}=${encodeURIComponent((utm[key] as string).trim())}`,
);
const query = [...kept, ...added].join("&");
return `${url.origin}${url.pathname}${query ? `?${query}` : ""}${url.hash}`;
} }
export function normalizeHashtags(_tags: string[], _limit: number): string[] { /**
return notImplemented("normalizeHashtags"); * Normalize to "#tag": strip leading #, drop spaces and punctuation (letters,
* digits and _ stay, in any script), dedupe case-insensitively, and cap.
*/
export function normalizeHashtags(tags: string[], limit: number): string[] {
const seen = new Set<string>();
const out: string[] = [];
for (const raw of tags) {
const tag = raw.normalize("NFC").replace(/[^\p{L}\p{N}_]/gu, "");
if (!tag) continue;
const key = tag.toLocaleLowerCase();
if (seen.has(key)) continue;
seen.add(key);
out.push(`#${tag}`);
if (out.length >= Math.max(0, limit)) break;
}
return out.slice(0, Math.max(0, limit));
} }
export interface ParsedMetricRow { export interface ParsedMetricRow {
@ -28,39 +83,450 @@ export interface ParsedMetricRow {
currency: string; currency: string;
} }
/** RFC 4180 CSV parsing with the saved column mapping. */ /** RFC 4180 records; quoted fields may hold delimiters, quotes ("") and newlines. */
export function parseCsvRecords(csv: string, delimiter: string): string[][] {
const text = csv.replace(/^/, "");
const records: string[][] = [];
let record: string[] = [];
let field = "";
let quoted = false;
let i = 0;
while (i < text.length) {
const char = text[i]!;
if (quoted) {
if (char === '"') {
if (text[i + 1] === '"') {
field += '"';
i += 2;
continue;
}
quoted = false;
i += 1;
continue;
}
field += char;
i += 1;
continue;
}
if (char === '"' && field.length === 0) {
quoted = true;
i += 1;
} else if (char === delimiter) {
record.push(field);
field = "";
i += 1;
} else if (char === "\r" || char === "\n") {
record.push(field);
records.push(record);
record = [];
field = "";
i += char === "\r" && text[i + 1] === "\n" ? 2 : 1;
} else {
field += char;
i += 1;
}
}
if (field.length > 0 || record.length > 0) {
record.push(field);
records.push(record);
}
return records;
}
const CURRENCY_SYMBOLS: Record<string, string> = { "€": "EUR", $: "USD", "£": "GBP", "¥": "JPY" };
/** Whole, non-negative count with optional thousands separators. */
function parseCount(raw: string): number | null {
const value = raw.trim().replace(/[\s  ']/g, "");
if (value === "") return 0;
if (/^\d+$/.test(value)) return Number(value);
if (/^\d{1,3}([.,]\d{3})+$/.test(value) && new Set(value.match(/[.,]/g)).size === 1) {
return Number(value.replace(/[.,]/g, ""));
}
return null;
}
/**
* Money in major units to cents. With both "," and "." the later one is the
* decimal separator; a single separator followed by one or two digits is
* decimal, by exactly three digits after 1–999 it groups thousands ("1,234"),
* otherwise it is decimal ("0.995"). Extra decimals round half up.
*/
export function parseMoneyToCents(raw: string): { cents: number; currency: string | null } | null {
let value = raw.trim();
let currency: string | null = null;
for (const [symbol, code] of Object.entries(CURRENCY_SYMBOLS)) {
if (value.includes(symbol)) {
currency = code;
value = value.replace(symbol, "");
}
}
const code = value.match(/[A-Za-z]{3}/)?.[0];
if (code) {
currency = code.toUpperCase();
value = value.replace(code, "");
}
value = value.replace(/[\s  ']/g, "");
if (value === "") return { cents: 0, currency };
if (!/^\d[\d.,]*$/.test(value)) return null;
const lastComma = value.lastIndexOf(",");
const lastDot = value.lastIndexOf(".");
let integer = value;
let fraction = "";
if (lastComma >= 0 && lastDot >= 0) {
const decimal = Math.max(lastComma, lastDot);
integer = value.slice(0, decimal);
fraction = value.slice(decimal + 1);
} else if (lastComma >= 0 || lastDot >= 0) {
const separator = lastComma >= 0 ? "," : ".";
const parts = value.split(separator);
const tail = parts.at(-1)!;
const decimal =
parts.length === 2 &&
tail.length >= 1 &&
(tail.length <= 2 || !/^[1-9]\d{0,2}$/.test(parts[0]!));
if (decimal) {
integer = parts[0]!;
fraction = tail;
} else if (parts.slice(1).every((part) => part.length === 3)) {
integer = parts.join("");
} else {
return null;
}
}
integer = integer.replace(/[.,]/g, "");
if (!/^\d+$/.test(integer) || !/^\d*$/.test(fraction)) return null;
const twoDigits = `${fraction}00`.slice(0, 2);
const roundUp = fraction.length > 2 && Number(fraction[2]) >= 5 ? 1 : 0;
return { cents: Number(integer) * 100 + Number(twoDigits) + roundUp, currency };
}
/** ISO dates and times, YYYY/MM/DD, and DD.MM.YYYY. Ambiguous slash dates are rejected. */
function parseDate(raw: string): Date | null {
const value = raw.trim();
let match = value.match(/^(\d{4})[-/](\d{2})[-/](\d{2})$/);
if (match) return utcDate(Number(match[1]), Number(match[2]), Number(match[3]));
match = value.match(/^(\d{1,2})\.(\d{1,2})\.(\d{4})$/);
if (match) return utcDate(Number(match[3]), Number(match[2]), Number(match[1]));
if (/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(value)) {
const date = new Date(value.replace(" ", "T"));
return Number.isNaN(date.getTime()) ? null : date;
}
return null;
}
function utcDate(year: number, month: number, day: number): Date | null {
const date = new Date(Date.UTC(year, month - 1, day));
return date.getUTCFullYear() === year &&
date.getUTCMonth() === month - 1 &&
date.getUTCDate() === day
? date
: null;
}
/** RFC 4180 CSV parsing with the saved column mapping. Rows count from the header as row 1. */
export function parseMetricsCsv( export function parseMetricsCsv(
_csv: string, csv: string,
_mapping: MetricCsvMapping, mapping: MetricCsvMapping,
): { rows: ParsedMetricRow[]; skipped: Array<{ row: number; reason: string }> } { ): { rows: ParsedMetricRow[]; skipped: Array<{ row: number; reason: string }> } {
return notImplemented("parseMetricsCsv"); const records = parseCsvRecords(csv, mapping.delimiter ?? ",");
const header = records[0];
const rows: ParsedMetricRow[] = [];
const skipped: Array<{ row: number; reason: string }> = [];
if (!header) return { rows, skipped: [{ row: 1, reason: "file is empty" }] };
const names = header.map((name) => name.trim());
const column = (name: string | undefined): number => {
if (!name) return -1;
const exact = names.indexOf(name.trim());
if (exact >= 0) return exact;
const lower = name.trim().toLowerCase();
return names.findIndex((n) => n.toLowerCase() === lower);
};
const columns = mapping.columns;
const index = {
publication: column(columns.publication),
observedAt: column(columns.observedAt),
views: column(columns.views),
clicks: column(columns.clicks),
conversions: column(columns.conversions),
revenue: column(columns.revenue),
currency: column(columns.currency),
};
const missing = (Object.keys(index) as Array<keyof typeof index>).filter(
(key) => columns[key] !== undefined && index[key] < 0,
);
if (missing.length > 0) {
return {
rows,
skipped: [
{ row: 1, reason: `missing columns: ${missing.map((k) => columns[k]).join(", ")}` },
],
};
}
const defaultCurrency = (mapping.defaultCurrency ?? "EUR").toUpperCase();
for (const [offset, record] of records.slice(1).entries()) {
const row = offset + 2;
if (record.every((field) => field.trim() === "")) continue;
const cell = (i: number) => (i >= 0 ? (record[i] ?? "") : "");
const publicationKey = cell(index.publication).trim();
if (!publicationKey) {
skipped.push({ row, reason: "publication is empty" });
continue;
}
const observedAt = parseDate(cell(index.observedAt));
if (!observedAt) {
skipped.push({ row, reason: "date is not recognized" });
continue;
}
const counts = {
views: parseCount(cell(index.views)),
clicks: parseCount(cell(index.clicks)),
conversions: parseCount(cell(index.conversions)),
};
const invalid = (Object.keys(counts) as Array<keyof typeof counts>).find(
(key) => counts[key] === null,
);
if (invalid) {
skipped.push({ row, reason: `${invalid} is not a whole number` });
continue;
}
let revenueCents = 0;
let currency: string | null = null;
const revenueRaw = cell(index.revenue);
if (revenueRaw.trim()) {
if (/^\s*-/.test(revenueRaw)) {
skipped.push({ row, reason: "revenue is negative" });
continue;
}
if (mapping.revenueUnit === "cents") {
const value = parseCount(revenueRaw);
if (value === null) {
skipped.push({ row, reason: "revenue is not a whole number" });
continue;
}
revenueCents = value;
} else {
const money = parseMoneyToCents(revenueRaw);
if (!money) {
skipped.push({ row, reason: "revenue is not a number" });
continue;
}
revenueCents = money.cents;
currency = money.currency;
}
}
const currencyCell = cell(index.currency).trim().toUpperCase();
if (currencyCell && !/^[A-Z]{3}$/.test(currencyCell)) {
skipped.push({ row, reason: "currency is not a 3-letter code" });
continue;
}
rows.push({
row,
publicationKey,
observedAt,
views: counts.views ?? 0,
clicks: counts.clicks ?? 0,
conversions: counts.conversions ?? 0,
revenueCents,
currency: currencyCell || currency || defaultCurrency,
});
}
return { rows, skipped };
} }
/** Whether the brand intro fits before the hook window; shorts put the hook first. */ const SHORT_FORM_MAX_MS = 180_000;
export function introFits(_profile: PlatformProfile, _introMs: number): boolean { const MAX_INTRO_MS = 4000;
return notImplemented("introFits");
/** Vertical profiles capped at three minutes: the hook must come before any branding. */
export function isShortForm(profile: PlatformProfile): boolean {
return (
profile.video.aspectRatios.includes("9:16") && profile.duration.hardMaxMs <= SHORT_FORM_MAX_MS
);
} }
/** Deterministic intro proposals from the brand kit, tone, and available templates. */ /**
* Whether the brand intro may open the video. Short-form vertical profiles
* always put the hook first (the brand appears as watermark or end card).
* Elsewhere the intro may take at most 4 s and a tenth of the maximum length.
*/
export function introFits(profile: PlatformProfile, introMs: number): boolean {
if (introMs <= 0) return true;
if (isShortForm(profile)) return false;
return introMs <= Math.min(MAX_INTRO_MS, profile.duration.hardMaxMs / 10);
}
const TONE_STYLES: Array<{ pattern: RegExp; keywords: string[] }> = [
{
pattern: /energ|bold|dynamic|loud|wild|sport|fast|laut|schnell|kraft/i,
keywords: ["glitch", "energy", "kinetic"],
},
{
pattern: /elegan|calm|luxur|soft|premium|ruhig|edel|sanft|warm/i,
keywords: ["elegant", "fade", "minimal"],
},
{
pattern: /minimal|clean|simple|schlicht|klar|modern/i,
keywords: ["minimal", "logo", "fade"],
},
{ pattern: /playful|fun|witzig|verspielt|bunt/i, keywords: ["kinetic", "typo", "glitch"] },
];
const PARAM_SOURCES: Record<string, (kit: BrandKit) => string | undefined> = {
primarycolor: (kit) => kit.colors.primary,
secondarycolor: (kit) => kit.colors.secondary,
accentcolor: (kit) => kit.colors.accent,
textcolor: (kit) => kit.colors.text,
backgroundcolor: (kit) => kit.colors.background,
brandname: (kit) => kit.name,
title: (kit) => kit.name,
name: (kit) => kit.name,
logoartifactid: (kit) => kit.logoArtifactIds[0],
logo: (kit) => kit.logoArtifactIds[0],
fontfamily: (kit) => (kit.fonts.find((f) => f.role === "heading") ?? kit.fonts[0])?.family,
soundartifactid: (kit) => kit.musicBeds.find((bed) => bed.commercialUse)?.artifactId,
sound: (kit) => kit.musicBeds.find((bed) => bed.commercialUse)?.artifactId,
};
const VARIANT_DURATIONS = [2500, 2000, 3000, 3500, 1500, 4000];
/**
* Deterministic intro proposals from the brand kit, tone, and available
* templates: intro templates are ranked by how well their id and name match
* the tone's style keywords (then by id), each proposal uses a distinct
* template while there are enough, and durations stay within 1.5–4 s and the
* template's own limits. Parameters are filled from the kit by name.
*/
export function proposeIntroVariants( export function proposeIntroVariants(
_kit: BrandKit, kit: BrandKit,
_templates: MotionTemplateInfo[], templates: MotionTemplateInfo[],
_count: number, count: number,
): Array<{ template: MotionTemplateRef; rationale: string }> { ): Array<{ template: MotionTemplateRef; rationale: string }> {
return notImplemented("proposeIntroVariants"); const intros = templates
.filter((t) => t.kind === "intro")
.sort((a, b) => a.id.localeCompare(b.id));
if (intros.length === 0 || count <= 0) return [];
const tone = `${kit.tone ?? ""} ${kit.audience ?? ""}`;
const keywords = TONE_STYLES.filter((style) => style.pattern.test(tone)).flatMap(
(style) => style.keywords,
);
const affinity = (template: MotionTemplateInfo) => {
const haystack = `${template.id} ${template.name}`.toLowerCase();
return keywords.reduce(
(score, keyword, index) => score + (haystack.includes(keyword) ? keywords.length - index : 0),
0,
);
};
const ranked = [...intros].sort((a, b) => affinity(b) - affinity(a) || a.id.localeCompare(b.id));
return Array.from({ length: count }, (_, index) => {
const template = ranked[index % ranked.length]!;
const round = Math.floor(index / ranked.length);
const wanted = VARIANT_DURATIONS[(index + round) % VARIANT_DURATIONS.length]!;
const durationMs = Math.min(
Math.max(wanted, 1500, template.minDurationMs),
Math.min(4000, template.maxDurationMs),
);
const params: Record<string, string | number | boolean> = {};
for (const param of template.params) {
const source = PARAM_SOURCES[param.name.toLowerCase().replace(/[^a-z]/g, "")];
const value = source?.(kit);
if (value !== undefined) params[param.name] = value.slice(0, 200);
}
const matched = affinity(template) > 0 && kit.tone ? ` matches the tone "${kit.tone}"` : "";
return {
template: { templateId: template.id, params, durationMs: Math.max(0, durationMs) },
rationale: `${template.name}${matched}, ${(durationMs / 1000).toFixed(1)} s`,
};
});
} }
function luminance(hex: string): number {
const channel = (offset: number) => {
const value = Number.parseInt(hex.slice(offset, offset + 2), 16) / 255;
return value <= 0.03928 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4;
};
return 0.2126 * channel(1) + 0.7152 * channel(3) + 0.0722 * channel(5);
}
/** WCAG contrast ratio of two #RRGGBB colours. */
export function contrastRatio(a: string, b: string): number {
const [light, dark] = [luminance(a), luminance(b)].sort((x, y) => y - x) as [number, number];
return (light + 0.05) / (dark + 0.05);
}
const MIN_CONTRAST = 4.5;
/** Structural brand-kit checks beyond the schema (contrast, duplicate presets). */ /** Structural brand-kit checks beyond the schema (contrast, duplicate presets). */
export function validateBrandKit(_kit: BrandKit): string[] { export function validateBrandKit(kit: BrandKit): string[] {
return notImplemented("validateBrandKit"); const warnings: string[] = [];
const ratio = contrastRatio(kit.colors.text, kit.colors.background);
if (ratio < MIN_CONTRAST) {
warnings.push(`text on background has contrast ${ratio.toFixed(1)}:1, below ${MIN_CONTRAST}:1`);
}
const seen = new Set<string>();
for (const preset of kit.captionPresets) {
if (seen.has(preset.id)) warnings.push(`caption preset id "${preset.id}" is used twice`);
seen.add(preset.id);
if (preset.backgroundColor) {
const presetRatio = contrastRatio(preset.textColor, preset.backgroundColor);
if (presetRatio < MIN_CONTRAST) {
warnings.push(
`caption preset "${preset.id}" has contrast ${presetRatio.toFixed(1)}:1, below ${MIN_CONTRAST}:1`,
);
}
}
}
for (const bed of kit.musicBeds) {
if (!bed.commercialUse)
warnings.push(`music bed "${bed.title}" is not licensed for commercial use`);
}
for (const font of kit.fonts) {
if (!font.license?.trim()) warnings.push(`font "${font.family}" has no license note`);
}
if (kit.watermark && !kit.watermark.artifactId) warnings.push("watermark has no image");
return warnings;
} }
/** Upload checklist markdown for an export bundle. */ /** Upload checklist markdown for an export bundle. */
export function uploadChecklist(_input: { export function uploadChecklist(input: {
profile: PlatformProfile; profile: PlatformProfile;
aiAltered: boolean; aiAltered: boolean;
hasCaptions: boolean; hasCaptions: boolean;
trackingUrl: string | null; trackingUrl: string | null;
}): string { }): string {
return notImplemented("uploadChecklist"); const { profile } = input;
const items: string[] = ["Upload the video file and choose the poster as cover image."];
if (input.aiAltered && profile.disclosure.aiAlteredLabelRequired) {
items.push(
"Set the platform's AI-generated or altered content label: this video contains AI alterations.",
);
}
if (profile.disclosure.ageRestrictedFlag) items.push("Mark the upload as age-restricted.");
if (profile.disclosure.paidPartnershipLabel) {
items.push("Turn on the paid partnership label if this post is sponsored.");
}
if (input.trackingUrl) {
const link = `<${input.trackingUrl}>`;
if (profile.text.linkPolicy === "allowed") items.push(`Add the tracking link: ${link}`);
else if (profile.text.linkPolicy === "bio_only") {
items.push(`Put the tracking link in the profile bio, not in the caption: ${link}`);
} else items.push("Do not add links: this platform does not allow them.");
} else if (profile.text.linkPolicy === "forbidden") {
items.push("Do not add links: this platform does not allow them.");
}
if (input.hasCaptions) {
items.push("Upload the SRT or VTT subtitles, or confirm the burned-in captions are readable.");
} else if (profile.captions.required) {
items.push("Captions are required on this platform: add them before publishing.");
}
const limits = [
profile.text.titleMaxChars ? `title up to ${profile.text.titleMaxChars} characters` : null,
`caption up to ${profile.text.captionMaxChars} characters`,
`at most ${profile.text.hashtagLimit} hashtags`,
].filter(Boolean);
items.push(`Paste the caption from caption.txt (${limits.join(", ")}).`);
return `# Upload checklist: ${profile.name.replace(/[\r\n]+/g, " ")}\n\n${items
.map((item) => `- [ ] ${item}`)
.join("\n")}\n`;
} }

4
packages/media/src/not-implemented.ts
View File

@ -1,4 +0,0 @@
/** Phase A scaffold marker. No call site may remain once Media Studio ships. */
export function notImplemented(name: string): never {
throw new Error(`not implemented: ${name}`);
}

334
packages/media/src/teaser.test.ts Normal file
View File

@ -0,0 +1,334 @@
import {
MediaProjectGoalSchema,
TEASER_FEATURES,
TEASER_LEARNING_MIN_SAMPLES,
} from "@rakazo/contracts";
import { describe, expect, it } from "vitest";
import { validateEdl } from "./edl.js";
import type { ScoredSegmentInput, SegmentFeatures } from "./teaser.js";
import {
autoPlanEdl,
defaultTeaserWeights,
fitTeaserWeights,
rankThumbnailFrames,
scoreSegment,
selectTeaserSegments,
speechHookScore,
} from "./teaser.js";
import { testContext, testKit, testPaidProfile, testProfile } from "./test-fixtures.js";
function segment(
id: string,
assetId: string,
startMs: number,
endMs: number,
features: SegmentFeatures,
extra: Partial<ScoredSegmentInput> = {},
): ScoredSegmentInput {
return {
id,
assetId,
startMs,
endMs,
capturedAt: null,
features,
paidOnly: false,
transcript: null,
...extra,
};
}
/** One 30 s asset building up to a climax at 20–24 s, plus a second asset. */
const segments: ScoredSegmentInput[] = [
segment("s1", "asset-a", 0, 4000, { motion: 0.2, sharpness: 0.8 }),
segment(
"s2",
"asset-a",
4000,
8000,
{ motion: 0.3, audioPeak: 0.3 },
{ transcript: "Wait, you won't believe what happens next?" },
),
segment("s3", "asset-a", 8000, 12_000, { motion: 0.5, faces: 0.6 }),
segment("s4", "asset-a", 12_000, 16_000, { motion: 0.6, audioPeak: 0.5 }),
segment("s5", "asset-a", 16_000, 20_000, { motion: 0.7, novelty: 0.6 }, { paidOnly: true }),
segment("s6", "asset-a", 20_000, 24_000, {
motion: 1,
audioPeak: 1,
faces: 1,
novelty: 1,
aesthetics: 1,
}),
segment("s7", "asset-a", 24_000, 28_000, { motion: 0.4 }),
segment("s8", "asset-b", 0, 3000, { motion: 0.5, aesthetics: 0.7 }),
];
describe("weights and scores", () => {
it("defines every feature for every goal", () => {
for (const goal of MediaProjectGoalSchema.options) {
const weights = defaultTeaserWeights(goal);
expect(Object.keys(weights).sort()).toEqual([...TEASER_FEATURES].sort());
expect(Object.values(weights).reduce((a, b) => a + b, 0)).toBeCloseTo(1, 6);
}
expect(defaultTeaserWeights("teaser").speechHook).toBeGreaterThan(
defaultTeaserWeights("highlight").speechHook,
);
});
it("scores as a weighted mean of clamped features", () => {
const weights = defaultTeaserWeights("teaser");
expect(scoreSegment({}, weights)).toBe(0);
const all = Object.fromEntries(TEASER_FEATURES.map((f) => [f, 1]));
expect(scoreSegment(all, weights)).toBeCloseTo(1, 6);
expect(scoreSegment({ motion: 5 }, weights)).toBeCloseTo(weights.motion, 6);
expect(scoreSegment({ motion: Number.NaN }, weights)).toBe(0);
expect(scoreSegment({ motion: 1 }, { ...weights, motion: 0 })).toBe(0);
});
it("detects German and English speech hooks", () => {
expect(speechHookScore("")).toBe(0);
expect(speechHookScore("the weather was fine")).toBe(0);
expect(speechHookScore("Wait, you won't believe this!", "en")).toBeGreaterThan(0.6);
expect(speechHookScore("Warum macht das niemand?", "de")).toBeGreaterThan(0.5);
expect(speechHookScore("Schau mal, 3 Tricks!", "de-DE")).toBeGreaterThan(0.6);
expect(speechHookScore("Here are 5 tips")).toBeGreaterThan(speechHookScore("here are tips"));
expect(speechHookScore("Wie geht das? Pass auf! 10 Sekunden, du!")).toBe(1);
});
});
describe("selectTeaserSegments", () => {
const profile = testProfile();
const select = (overrides: Partial<Parameters<typeof selectTeaserSegments>[0]> = {}) =>
selectTeaserSegments({
segments,
profile,
goal: "teaser",
targetMs: 15_000,
weights: defaultTeaserWeights("teaser"),
endCardMs: 2000,
...overrides,
});
it("opens with the hook inside the hook window", () => {
const selection = select();
expect(selection.hookSegmentId).toBe("s2");
expect(selection.clips[0]).toMatchObject({ segmentId: "s2", inMs: 4000 });
expect(selection.notes[0]).toContain("hook window");
});
it("never uses paid-only segments and ends before the payoff", () => {
const selection = select();
const ids = selection.clips.map((c) => c.segmentId);
expect(ids).not.toContain("s5");
expect(ids).not.toContain("s6");
expect(ids).not.toContain("s7");
const last = selection.clips.at(-1)!;
expect(last.assetId).toBe("asset-a");
expect(last.outMs).toBeLessThanOrEqual(20_000);
expect(selection.notes.some((n) => n.startsWith("open loop"))).toBe(true);
});
it("builds tension and fits the duration bounds minus the end card", () => {
const selection = select();
const total = selection.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0);
expect(total).toBeLessThanOrEqual(15_000 - 2000);
expect(total).toBeGreaterThanOrEqual(profile.duration.hardMinMs - 2000);
for (const clip of selection.clips) expect(clip.outMs).toBeGreaterThan(clip.inMs);
const tight = select({ targetMs: 5000 });
expect(tight.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0)).toBeLessThanOrEqual(3000);
});
it("returns nothing when only paid-only material exists", () => {
const paid = segments.map((s) => ({ ...s, paidOnly: true }));
expect(select({ segments: paid })).toEqual({
clips: [],
hookSegmentId: null,
notes: ["no usable segments"],
});
});
it("is deterministic", () => {
expect(select({ segments: [...segments].reverse() })).toEqual(select());
});
});
describe("fitTeaserWeights", () => {
const prior = defaultTeaserWeights("teaser");
const samples = (count: number) =>
Array.from({ length: count }, (_, i) => {
const hook = (i % 10) / 9;
const motion = ((i * 7) % 10) / 9;
return {
features: { speechHook: hook, motion },
views: 1000,
conversions: Math.round(1000 * (0.01 + 0.2 * hook)),
};
});
it("stays off below the minimum sample count", () => {
expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES - 1), prior)).toBeNull();
const withoutViews = [
...samples(TEASER_LEARNING_MIN_SAMPLES - 1),
{ features: {}, views: 0, conversions: 0 },
];
expect(fitTeaserWeights(withoutViews, prior)).toBeNull();
});
it("shifts weight toward features that drive conversion, deterministically", () => {
const learned = fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)!;
expect(learned).not.toBeNull();
expect(learned.speechHook).toBeGreaterThan(prior.speechHook);
expect(learned.motion).toBeLessThan(prior.motion);
expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)).toEqual(learned);
const more = fitTeaserWeights(samples(200), prior)!;
expect(more.speechHook).toBeGreaterThan(learned.speechHook);
});
});
describe("autoPlanEdl", () => {
const assets = new Map([
["asset-a", { durationMs: 30_000, width: 1920, height: 1080, hasSpeech: true }],
["asset-b", { durationMs: 8000, width: 1080, height: 1920, hasSpeech: false }],
]);
const context = testContext({
assets: new Map([
["asset-a", { kind: "video", durationMs: 30_000, width: 1920, height: 1080 }],
["asset-b", { kind: "video", durationMs: 8000, width: 1080, height: 1920 }],
]),
paidOnly: [{ assetId: "asset-a", startMs: 16_000, endMs: 20_000 }],
templateIds: new Set(["intro-minimal", "end-card-a"]),
goal: "teaser",
});
it("plans a valid teaser with hook first, captions, watermark, CTA and redactions", () => {
const kit = testKit({
intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 },
endCard: { templateId: "end-card-a", params: { style: "a" }, durationMs: 2000 },
lutArtifactId: "lut-1",
});
const edl = autoPlanEdl({
goal: "teaser",
profile: testProfile(),
brandKit: kit,
segments,
weights: defaultTeaserWeights("teaser"),
assets,
people: [
{ id: "person-1", anonymizeByDefault: "synthetic_identity" },
{ id: "person-2", anonymizeByDefault: "none" },
],
marks: [
{ id: "mark-1", treatment: "inpaint", assetId: "asset-a", personId: null },
{ id: "mark-2", treatment: "blur", assetId: "asset-z", personId: null },
{ id: "mark-3", treatment: "none", assetId: "asset-a", personId: null },
],
ctaText: "Full video: link in bio",
});
const result = validateEdl(edl, { ...context, brandKit: kit });
expect(result.ok ? [] : result.errors).toEqual([]);
// Short vertical profile: no intro, the hook comes first.
expect(edl.intro).toBeUndefined();
expect(edl.tracks.video[0]!.segmentId).toBe("s2");
expect(edl.tracks.video[0]!.reframe).toBe("auto");
expect(edl.tracks.video[0]!.color?.useBrandLut).toBe(true);
expect(edl.outro).toMatchObject({
templateId: "end-card-a",
params: { style: "a", cta: "Full video: link in bio" },
});
expect(edl.captions).toMatchObject({ mode: "both", presetId: "clean", maxCharsPerLine: 24 });
expect(edl.overlays.map((o) => o.kind)).toEqual(["watermark"]);
expect(edl.redactions.faces.map((f) => [f.target, f.mode])).toEqual([
[{ personId: "person-1" }, "synthetic_identity"],
[{ unassignedFaces: true }, "blur"],
]);
expect(edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "inpaint" }]);
expect(edl.metadata).toMatchObject({
aiAltered: true,
aiAlterations: ["synthetic_identity", "inpaint"],
});
});
it("uses a CTA text card without an end-card template and keeps the intro on long-form profiles", () => {
const profile = testPaidProfile();
const kit = testKit({
intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 },
watermark: undefined,
});
const teaser = autoPlanEdl({
goal: "teaser",
profile,
brandKit: kit,
segments,
weights: defaultTeaserWeights("teaser"),
assets,
people: [],
marks: [],
ctaText: "Watch the full video",
});
expect(teaser.intro?.templateId).toBe("intro-minimal");
expect(teaser.overlays).toEqual([
expect.objectContaining({ kind: "cta_end_card", text: "Watch the full video" }),
]);
expect(teaser.redactions.faces).toEqual([]);
expect(teaser.metadata.aiAltered).toBe(false);
const result = validateEdl(teaser, { ...context, profile, brandKit: kit });
expect(result.ok ? [] : result.errors).toEqual([]);
});
it("plans a full release in capture order including paid-only material on paid profiles", () => {
const profile = testPaidProfile();
const edl = autoPlanEdl({
goal: "full_release",
profile,
brandKit: null,
segments,
weights: defaultTeaserWeights("full_release"),
assets,
people: [],
marks: [],
ctaText: null,
});
expect(edl.tracks.video.map((c) => c.segmentId)).toEqual([
"s1",
"s2",
"s3",
"s4",
"s5",
"s6",
"s7",
"s8",
]);
expect(edl.tracks.video.at(-1)!.reframe).toBe("auto");
expect(edl.tracks.video[0]!.reframe).toBe("none");
const result = validateEdl(edl, { ...context, profile, brandKit: null, goal: "full_release" });
expect(result.ok ? [] : result.errors).toEqual([]);
expect(() =>
autoPlanEdl({
goal: "teaser",
profile,
brandKit: null,
segments: [],
weights: defaultTeaserWeights("teaser"),
assets,
people: [],
marks: [],
ctaText: null,
}),
).toThrow(/no usable segments/);
});
});
describe("rankThumbnailFrames", () => {
it("returns the best frames spread over the video", () => {
const candidates = [
{ tMs: 0, sharpness: 0.5, face: 0, thirds: 0.5, contrast: 0.5 },
{ tMs: 1000, sharpness: 1, face: 1, thirds: 1, contrast: 1 },
{ tMs: 1200, sharpness: 1, face: 1, thirds: 0.9, contrast: 1 },
{ tMs: 5000, sharpness: 0.8, face: 0.5, thirds: 0.5, contrast: 0.5 },
{ tMs: 9000, sharpness: 0.7, face: 0.5, thirds: 0.5, contrast: 0.5 },
];
expect(rankThumbnailFrames(candidates, 3)).toEqual([1000, 5000, 9000]);
expect(rankThumbnailFrames(candidates.slice(1, 3), 3)).toEqual([1000, 1200]);
expect(rankThumbnailFrames(candidates, 0)).toEqual([]);
});
});

599
packages/media/src/teaser.ts
View File

@ -1,4 +1,6 @@
import type { import type {
AiAlteration,
AnonymizeMode,
BrandKit, BrandKit,
Edl, Edl,
MediaProjectGoal, MediaProjectGoal,
@ -6,12 +8,88 @@ import type {
TeaserFeature, TeaserFeature,
TeaserWeights, TeaserWeights,
} from "@rakazo/contracts"; } from "@rakazo/contracts";
import { notImplemented } from "./not-implemented.js"; import {
AnonymizeModeSchema,
EdlSchema,
TEASER_FEATURES,
TEASER_LEARNING_MIN_SAMPLES,
} from "@rakazo/contracts";
import { introFits } from "./marketing.js";
export type SegmentFeatures = Partial<Record<TeaserFeature, number>>; export type SegmentFeatures = Partial<Record<TeaserFeature, number>>;
export function defaultTeaserWeights(_goal: MediaProjectGoal): TeaserWeights { const GOAL_WEIGHTS: Record<MediaProjectGoal, Record<TeaserFeature, number>> = {
return notImplemented("defaultTeaserWeights"); teaser: {
motion: 0.14,
sharpness: 0.06,
faces: 0.08,
gaze: 0.08,
audioPeak: 0.14,
speechHook: 0.24,
novelty: 0.12,
beatProximity: 0.06,
aesthetics: 0.08,
},
trailer: {
motion: 0.16,
sharpness: 0.08,
faces: 0.08,
gaze: 0.06,
audioPeak: 0.14,
speechHook: 0.12,
novelty: 0.12,
beatProximity: 0.12,
aesthetics: 0.12,
},
highlight: {
motion: 0.2,
sharpness: 0.08,
faces: 0.12,
gaze: 0.06,
audioPeak: 0.18,
speechHook: 0.08,
novelty: 0.1,
beatProximity: 0.08,
aesthetics: 0.1,
},
story: {
motion: 0.08,
sharpness: 0.08,
faces: 0.16,
gaze: 0.12,
audioPeak: 0.06,
speechHook: 0.22,
novelty: 0.12,
beatProximity: 0.04,
aesthetics: 0.12,
},
full_release: {
motion: 0.1,
sharpness: 0.14,
faces: 0.1,
gaze: 0.08,
audioPeak: 0.08,
speechHook: 0.1,
novelty: 0.12,
beatProximity: 0.06,
aesthetics: 0.22,
},
custom: {
motion: 0.12,
sharpness: 0.1,
faces: 0.1,
gaze: 0.1,
audioPeak: 0.12,
speechHook: 0.14,
novelty: 0.12,
beatProximity: 0.08,
aesthetics: 0.12,
},
};
/** Default per-goal weights over every teaser feature; each set sums to 1. */
export function defaultTeaserWeights(goal: MediaProjectGoal): TeaserWeights {
return { ...GOAL_WEIGHTS[goal] };
} }
export interface ScoredSegmentInput { export interface ScoredSegmentInput {
@ -25,13 +103,75 @@ export interface ScoredSegmentInput {
transcript: string | null; transcript: string | null;
} }
export function scoreSegment(_features: SegmentFeatures, _weights: TeaserWeights): number { const clamp01 = (value: number) => (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0);
return notImplemented("scoreSegment");
/** Weighted mean of 0..1 features; missing features count as 0. */
export function scoreSegment(features: SegmentFeatures, weights: TeaserWeights): number {
let total = 0;
let norm = 0;
for (const feature of TEASER_FEATURES) {
const weight = weights[feature] ?? 0;
norm += Math.abs(weight);
total += weight * clamp01(features[feature] ?? 0);
}
return norm === 0 ? 0 : total / norm;
} }
const HOOK_LEXICON = {
en: {
questionWords:
/^(what|why|how|who|when|where|which|did|do|does|can|could|would|should|is|are|have|ever)\b/,
announcements:
/\b(wait|watch|look|listen|check this|you won'?t believe|here'?s|secret|how to|never|finally|the moment|let me show|guess|imagine|stop|nobody|everyone|biggest|best|worst|first time|new|today)\b/,
address: /\b(you|your)\b/,
numbers: /\b(one|two|three|five|ten|hundred|thousand|million|first|second|third|top)\b/,
},
de: {
questionWords:
/^(was|warum|wieso|weshalb|wie|wer|wann|wo|welche[rsmn]?|hast|habt|kannst|könnt|willst|würdest|weißt|ist|sind)\b/,
announcements:
/\b(warte|wartet|schau|schaut|guck|hör zu|pass auf|achtung|du wirst nicht glauben|geheimnis|so geht|niemals|endlich|gleich|hier ist|zeig ich dir|stell dir vor|rate mal|niemand|alle|größte|beste|schlimmste|zum ersten mal|neu|heute)\b/,
address: /\b(du|dich|dir|dein|deine|ihr|euch|euer)\b/,
numbers: /\b(eins|zwei|drei|fünf|zehn|hundert|tausend|million|erste[rsn]?|zweite[rsn]?|top)\b/,
},
} as const;
/** 0..1 speech-hook strength from a transcript excerpt (questions, exclamations, numbers, announcements). */ /** 0..1 speech-hook strength from a transcript excerpt (questions, exclamations, numbers, announcements). */
export function speechHookScore(_text: string, _language?: string): number { export function speechHookScore(text: string, language?: string): number {
return notImplemented("speechHookScore"); const normalized = text.trim().toLocaleLowerCase(language);
if (!normalized) return 0;
const code = language?.slice(0, 2).toLowerCase();
const lexicons =
code === "de"
? [HOOK_LEXICON.de]
: code === "en"
? [HOOK_LEXICON.en]
: Object.values(HOOK_LEXICON);
let score = 0;
if (normalized.includes("?")) score += 0.35;
if (normalized.includes("!")) score += 0.25;
const hasNumber =
/\d/.test(normalized) || lexicons.some((lexicon) => lexicon.numbers.test(normalized));
if (hasNumber) score += 0.2;
if (lexicons.some((lexicon) => lexicon.announcements.test(normalized))) score += 0.3;
if (lexicons.some((lexicon) => lexicon.questionWords.test(normalized))) score += 0.15;
if (lexicons.some((lexicon) => lexicon.address.test(normalized))) score += 0.1;
return Math.min(1, Math.round(score * 1000) / 1000);
}
function segmentFeatures(segment: ScoredSegmentInput): SegmentFeatures {
if (segment.features.speechHook !== undefined || !segment.transcript) return segment.features;
return { ...segment.features, speechHook: speechHookScore(segment.transcript) };
}
/** Strength of a segment as an opener: speech hooks, audio peaks, motion, eye contact. */
function hookStrength(features: SegmentFeatures): number {
return (
0.45 * clamp01(features.speechHook ?? 0) +
0.25 * clamp01(features.audioPeak ?? 0) +
0.15 * clamp01(features.motion ?? 0) +
0.15 * clamp01(features.gaze ?? 0)
);
} }
export interface TeaserSelection { export interface TeaserSelection {
@ -41,12 +181,20 @@ export interface TeaserSelection {
notes: string[]; notes: string[];
} }
const MIN_CLIP_MS = 1000;
/** /**
* Pick clips for a goal and profile: a hook inside hookWindowMs, rising * Pick clips for a goal and profile: a hook inside hookWindowMs, rising
* tension, cut before the payoff (open loop), never paid-only segments, * tension, cut before the payoff (open loop), never paid-only segments,
* total within the profile's duration bounds. * total within the profile's duration bounds.
*
* The primary asset is the one holding the best segment overall; its payoff
* is its best segment (paid-only included, since that is usually the payoff).
* Nothing from the primary asset at or after the payoff is used. The hook
* opens the teaser, the rest follows in ascending score, and the lead-in to
* the payoff closes it, leaving `endCardMs` for the end card.
*/ */
export function selectTeaserSegments(_input: { export function selectTeaserSegments(input: {
segments: ScoredSegmentInput[]; segments: ScoredSegmentInput[];
profile: PlatformProfile; profile: PlatformProfile;
goal: MediaProjectGoal; goal: MediaProjectGoal;
@ -54,19 +202,227 @@ export function selectTeaserSegments(_input: {
weights: TeaserWeights; weights: TeaserWeights;
endCardMs: number; endCardMs: number;
}): TeaserSelection { }): TeaserSelection {
return notImplemented("selectTeaserSegments"); const { profile } = input;
const notes: string[] = [];
const scored = input.segments
.filter((s) => s.endMs > s.startMs)
.map((segment) => {
const features = segmentFeatures(segment);
return { segment, features, score: scoreSegment(features, input.weights) };
})
.sort(
(a, b) =>
b.score - a.score ||
a.segment.assetId.localeCompare(b.segment.assetId) ||
a.segment.startMs - b.segment.startMs ||
a.segment.id.localeCompare(b.segment.id),
);
const maxMs = Math.max(
0,
Math.min(input.targetMs, profile.duration.hardMaxMs) - Math.max(0, input.endCardMs),
);
const minMs = Math.max(0, profile.duration.hardMinMs - Math.max(0, input.endCardMs));
const usable = scored.filter((s) => !s.segment.paidOnly);
if (usable.length === 0 || maxMs < MIN_CLIP_MS) {
return { clips: [], hookSegmentId: null, notes: ["no usable segments"] };
}
const primaryAsset = scored[0]!.segment.assetId;
const payoff = scored.find((s) => s.segment.assetId === primaryAsset)!;
const pool = usable.filter(
(s) =>
s.segment.assetId !== primaryAsset ||
(s.segment.id !== payoff.segment.id && s.segment.endMs <= payoff.segment.startMs),
);
const openLoop = pool.length < usable.length || payoff.segment.paidOnly;
if (pool.length === 0) {
return { clips: [], hookSegmentId: null, notes: ["only the payoff is usable"] };
}
const hook = [...pool].sort(
(a, b) =>
hookStrength(b.features) - hookStrength(a.features) ||
b.score - a.score ||
a.segment.id.localeCompare(b.segment.id),
)[0]!;
const hookWindow = profile.duration.hookWindowMs;
const clipCap = Math.max(MIN_CLIP_MS, Math.min(5000, Math.round(maxMs / 4)));
const lengthOf = (s: ScoredSegmentInput, cap: number) => Math.min(s.endMs - s.startMs, cap);
// Lead-in: the last primary-asset segment before the payoff closes the teaser.
const leadIn = openLoop
? pool
.filter((s) => s.segment.assetId === primaryAsset && s !== hook)
.sort((a, b) => b.segment.endMs - a.segment.endMs)[0]
: undefined;
const chosen: typeof pool = [hook];
let total = lengthOf(hook.segment, Math.max(clipCap, Math.min(hookWindow, maxMs)));
const reserve = leadIn ? lengthOf(leadIn.segment, clipCap) : 0;
for (const candidate of pool) {
if (candidate === hook || candidate === leadIn) continue;
const length = lengthOf(candidate.segment, clipCap);
if (length < Math.min(MIN_CLIP_MS, candidate.segment.endMs - candidate.segment.startMs))
continue;
if (total + length + reserve > maxMs) continue;
chosen.push(candidate);
total += length;
}
if (leadIn && total + reserve <= maxMs) {
chosen.push(leadIn);
total += reserve;
}
const middle = chosen
.slice(1)
.filter((s) => s !== leadIn)
.sort((a, b) => a.score - b.score || a.segment.id.localeCompare(b.segment.id));
const ordered = [hook, ...middle, ...(chosen.includes(leadIn!) ? [leadIn!] : [])];
// Lengths: start from the cap, then grow toward the minimum duration if needed.
const lengths = ordered.map((s, i) =>
lengthOf(s.segment, i === 0 ? Math.max(clipCap, Math.min(hookWindow, maxMs)) : clipCap),
);
let sum = lengths.reduce((a, b) => a + b, 0);
for (let i = 0; i < ordered.length && sum < minMs; i++) {
const full = ordered[i]!.segment.endMs - ordered[i]!.segment.startMs;
const grow = Math.min(full - lengths[i]!, minMs - sum, maxMs - sum);
if (grow > 0) {
lengths[i]! += grow;
sum += grow;
}
}
while (sum > maxMs && lengths.length > 0) {
const over = sum - maxMs;
const last = lengths.length - 1;
if (lengths[last]! - over >= MIN_CLIP_MS) {
lengths[last]! -= over;
sum -= over;
} else {
sum -= lengths.pop()!;
ordered.pop();
}
}
const clips = ordered.map((s, i) => {
const length = lengths[i]!;
// The lead-in ends where the payoff would begin; other clips open at their start.
const isLeadIn = s === leadIn;
const outMs = isLeadIn ? s.segment.endMs : s.segment.startMs + length;
return {
segmentId: s.segment.id,
assetId: s.segment.assetId,
inMs: outMs - length,
outMs,
};
});
notes.push(
`hook: segment ${hook.segment.id} opens inside the ${hookWindow} ms hook window`,
...middle.map((s) => `build: segment ${s.segment.id} (score ${s.score.toFixed(2)})`),
);
if (leadIn && ordered.includes(leadIn)) {
notes.push(`open loop: segment ${leadIn.segment.id} cuts before payoff ${payoff.segment.id}`);
} else if (openLoop) {
notes.push(`open loop: payoff ${payoff.segment.id} is left out`);
}
if (input.segments.some((s) => s.paidOnly)) notes.push("paid-only segments excluded");
if (sum < minMs) notes.push(`only ${sum} ms of usable material; profile minimum is ${minMs} ms`);
return { clips, hookSegmentId: hook.segment.id, notes };
} }
/** Regularized logistic regression of conversion on features; null below TEASER_LEARNING_MIN_SAMPLES. */ const LEARNING_RATE = 0.5;
const LEARNING_ITERATIONS = 400;
const L2 = 0.05;
const SHRINKAGE = 50;
/**
* Regularized logistic regression of conversion on features; null below
* TEASER_LEARNING_MIN_SAMPLES samples with views.
*
* Each sample contributes `conversions` successes out of `views` trials,
* weighted so every publication counts equally. Full-batch gradient descent
* from zero with a fixed iteration count keeps it deterministic. Positive
* coefficients of features that vary in the data redistribute those features'
* prior mass; features without variation keep their prior. The result is
* shrunk toward the prior by n / (n + 50) so a small sample cannot overturn it.
*/
export function fitTeaserWeights( export function fitTeaserWeights(
_samples: Array<{ features: SegmentFeatures; views: number; conversions: number }>, samples: Array<{ features: SegmentFeatures; views: number; conversions: number }>,
_prior: TeaserWeights, prior: TeaserWeights,
): TeaserWeights | null { ): TeaserWeights | null {
return notImplemented("fitTeaserWeights"); const usable = samples.filter((s) => s.views > 0 && s.conversions >= 0);
if (usable.length < TEASER_LEARNING_MIN_SAMPLES) return null;
const k = TEASER_FEATURES.length;
const xs = usable.map((s) => TEASER_FEATURES.map((f) => clamp01(s.features[f] ?? 0)));
const ys = usable.map((s) => Math.min(1, s.conversions / s.views));
const beta = new Array<number>(k).fill(0);
let intercept = 0;
for (let iteration = 0; iteration < LEARNING_ITERATIONS; iteration++) {
const gradient = new Array<number>(k).fill(0);
let gradientIntercept = 0;
for (let i = 0; i < xs.length; i++) {
const x = xs[i]!;
let z = intercept;
for (let j = 0; j < k; j++) z += beta[j]! * x[j]!;
const error = 1 / (1 + Math.exp(-z)) - ys[i]!;
gradientIntercept += error;
for (let j = 0; j < k; j++) gradient[j]! += error * x[j]!;
}
intercept -= (LEARNING_RATE * gradientIntercept) / xs.length;
for (let j = 0; j < k; j++) {
beta[j]! -= LEARNING_RATE * (gradient[j]! / xs.length + L2 * beta[j]!);
}
}
// Only features that vary in the data are re-weighted; they share their prior mass.
const varies = TEASER_FEATURES.map((_, j) => xs.some((x) => x[j] !== xs[0]![j]));
const mass = TEASER_FEATURES.reduce(
(sum, f, j) => sum + (varies[j] ? Math.max(0, prior[f] ?? 0) : 0),
0,
);
const positive = beta.map((b, j) => (varies[j] ? Math.max(0, b) : 0));
const positiveTotal = positive.reduce((a, b) => a + b, 0);
const alpha = usable.length / (usable.length + SHRINKAGE);
return Object.fromEntries(
TEASER_FEATURES.map((feature, j) => {
const base = prior[feature] ?? 0;
const learned = varies[j] && positiveTotal > 0 ? (positive[j]! / positiveTotal) * mass : base;
const value = (1 - alpha) * base + alpha * learned;
return [feature, Math.round(value * 1e6) / 1e6];
}),
) as TeaserWeights;
} }
/** Deterministic first-cut EDL for a project and profile; the agent refines it. */ const CTA_CARD_MS = 2500;
export function autoPlanEdl(_input: {
function pickTarget(profile: PlatformProfile, goal: MediaProjectGoal, availableMs: number): number {
const { hardMinMs, hardMaxMs, recommended } = profile.duration;
const options = [...recommended]
.filter((r) => r >= hardMinMs && r <= hardMaxMs)
.sort((a, b) => a - b);
let target: number;
if (goal === "full_release" || goal === "custom") {
target = Math.min(hardMaxMs, availableMs);
} else if (options.length > 0) {
target = options[Math.floor((options.length - 1) / 2)]!;
} else {
target = Math.min(hardMaxMs, 30_000);
}
return Math.max(hardMinMs, Math.min(hardMaxMs, target));
}
/**
* Deterministic first-cut EDL for a project and profile; the agent refines it.
*
* Teaser-style goals use selectTeaserSegments; full releases and custom
* projects take segments in order (paid-only ones only for non-social
* profiles). Reframing is "auto" whenever the source aspect differs from the
* output, captions follow the profile when the material has speech, the
* brand intro only plays where introFits allows it, and a teaser ends on the
* brand end card (or a CTA text card). Person and mark defaults become
* redactions, and the AI-alteration metadata follows from them.
*/
export function autoPlanEdl(input: {
goal: MediaProjectGoal; goal: MediaProjectGoal;
profile: PlatformProfile; profile: PlatformProfile;
brandKit: BrandKit | null; brandKit: BrandKit | null;
@ -81,18 +437,223 @@ export function autoPlanEdl(_input: {
marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>;
ctaText: string | null; ctaText: string | null;
}): Edl { }): Edl {
return notImplemented("autoPlanEdl"); const { goal, profile, brandKit: kit } = input;
const W = profile.video.width;
const H = profile.video.height;
const segments = input.segments.filter((s) => {
const asset = input.assets.get(s.assetId);
return (
asset && s.endMs > s.startMs && (asset.durationMs === null || s.endMs <= asset.durationMs)
);
});
const available = segments.reduce((sum, s) => sum + (s.endMs - s.startMs), 0);
const targetMs = pickTarget(profile, goal, available);
const intro = kit?.intro && introFits(profile, kit.intro.durationMs) ? kit.intro : undefined;
let outro = goal === "teaser" ? undefined : kit?.outro;
const ctaText = input.ctaText?.trim() || null;
let ctaCard = false;
if (goal === "teaser") {
if (kit?.endCard) {
outro = {
...kit.endCard,
params: { ...kit.endCard.params, ...(ctaText ? { cta: ctaText.slice(0, 200) } : {}) },
};
} else if (ctaText) {
ctaCard = true;
}
}
const bookendsMs = (intro?.durationMs ?? 0) + (outro?.durationMs ?? 0);
const budget = Math.max(MIN_CLIP_MS, targetMs - bookendsMs);
let picks: Array<{ segmentId: string; assetId: string; inMs: number; outMs: number }>;
if (goal === "full_release" || goal === "custom") {
const allowPaid = profile.kind !== "free_social";
const ordered = segments
.filter((s) => allowPaid || !s.paidOnly)
.sort(
(a, b) =>
(a.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) -
(b.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) ||
a.assetId.localeCompare(b.assetId) ||
a.startMs - b.startMs,
);
picks = [];
let used = 0;
for (const s of ordered) {
const length = Math.min(s.endMs - s.startMs, budget - used);
if (length < MIN_CLIP_MS) break;
picks.push({
segmentId: s.id,
assetId: s.assetId,
inMs: s.startMs,
outMs: s.startMs + length,
});
used += length;
}
} else {
picks = selectTeaserSegments({
segments,
profile,
goal,
targetMs: budget,
weights: input.weights,
endCardMs: 0,
}).clips;
}
if (picks.length === 0) throw new Error("no usable segments for this plan");
let atMs = 0;
const lut = Boolean(kit?.lutArtifactId);
const video = picks.map((pick, index) => {
const asset = input.assets.get(pick.assetId)!;
const sameAspect =
!asset.width || !asset.height || Math.abs(asset.width / asset.height / (W / H) - 1) <= 0.01;
const clip = {
id: `c${index + 1}`,
assetId: pick.assetId,
inMs: pick.inMs,
outMs: pick.outMs,
atMs,
speed: 1,
reframe: sameAspect ? ("none" as const) : ("auto" as const),
...(lut ? { color: { useBrandLut: true } } : {}),
segmentId: pick.segmentId,
};
atMs += pick.outMs - pick.inMs;
return clip;
});
const mainMs = atMs;
const introLength = intro?.durationMs ?? 0;
const totalMs = introLength + mainMs + (outro?.durationMs ?? 0);
const usedAssets = new Set(picks.map((p) => p.assetId));
const hasSpeech = [...usedAssets].some((id) => input.assets.get(id)?.hasSpeech);
const preset = kit?.captionPresets[0];
const captions =
(profile.captions.recommended || profile.captions.required) && hasSpeech
? {
mode: profile.export.subtitles ? ("both" as const) : ("burned" as const),
...(preset ? { presetId: preset.id } : {}),
maxCharsPerLine: W < H ? 24 : 32,
wordHighlight: preset?.template === "karaoke",
position: "lower" as const,
}
: undefined;
const zones = profile.safeZones;
const overlays: Array<Record<string, unknown>> = [];
if (kit?.watermark) {
overlays.push({ id: "watermark", kind: "watermark", atMs: 0, durationMs: totalMs });
}
if (ctaCard && ctaText) {
const cardMs = Math.min(CTA_CARD_MS, mainMs);
const left = zones.leftPct / 100 + 0.02;
const right = zones.rightPct / 100 + 0.02;
overlays.push({
id: "cta",
kind: "cta_end_card",
atMs: introLength + mainMs - cardMs,
durationMs: cardMs,
text: ctaText.slice(0, 200),
box: { x: left, y: 0.42, w: Math.max(0.1, 1 - left - right), h: 0.12 },
});
}
const protectedPeople = input.people.flatMap((person) => {
const mode = AnonymizeModeSchema.safeParse(person.anonymizeByDefault);
return mode.success && mode.data !== "none" ? [{ id: person.id, mode: mode.data }] : [];
});
const faces: Array<{
id: string;
target: { personId: string } | { unassignedFaces: true };
mode: Exclude<AnonymizeMode, "none">;
}> = protectedPeople.map((person, index) => ({
id: `face-${index + 1}`,
target: { personId: person.id },
mode: person.mode,
}));
if (faces.length > 0) {
// Unknown faces next to a protected person are blurred too; the gate blocks them otherwise.
faces.push({ id: "face-unassigned", target: { unassignedFaces: true }, mode: "blur" });
}
const peopleIds = new Set(input.people.map((p) => p.id));
const marks = input.marks.flatMap((mark) => {
if (mark.treatment === "none") return [];
const relevant =
(mark.assetId !== null && usedAssets.has(mark.assetId)) ||
(mark.personId !== null && peopleIds.has(mark.personId)) ||
(mark.assetId === null && mark.personId === null);
if (!relevant) return [];
const treatment = mark.treatment as "inpaint" | "blur" | "cover" | "reframe_out";
return [{ markId: mark.id, treatment }];
});
const aiAlterations: AiAlteration[] = [];
if (faces.some((f) => f.mode === "synthetic_identity")) aiAlterations.push("synthetic_identity");
if (marks.some((m) => m.treatment === "inpaint")) aiAlterations.push("inpaint");
return EdlSchema.parse({
edlVersion: 1,
output: {
profileId: profile.id,
width: W,
height: H,
fps: profile.video.fps,
durationTargetMs: targetMs,
durationHardMinMs: profile.duration.hardMinMs,
durationHardMaxMs: profile.duration.hardMaxMs,
},
tracks: { video, audio: [] },
transitions: [],
...(captions ? { captions } : {}),
overlays,
...(intro ? { intro } : {}),
...(outro ? { outro } : {}),
redactions: { faces, marks, regions: [] },
metadata: { tags: [], aiAltered: aiAlterations.length > 0, aiAlterations },
});
} }
const THUMBNAIL_WEIGHTS = { sharpness: 0.4, face: 0.25, thirds: 0.2, contrast: 0.15 };
/**
* Best thumbnail frame times, spread out: candidates are ranked by sharpness,
* face presence, rule of thirds and contrast, and each pick must be at least
* a tenth of the candidate span (min 1 s) away from earlier picks while
* enough candidates remain.
*/
export function rankThumbnailFrames( export function rankThumbnailFrames(
_candidates: Array<{ candidates: Array<{
tMs: number; tMs: number;
sharpness: number; sharpness: number;
face: number; face: number;
thirds: number; thirds: number;
contrast: number; contrast: number;
}>, }>,
_count: number, count: number,
): number[] { ): number[] {
return notImplemented("rankThumbnailFrames"); if (count <= 0 || candidates.length === 0) return [];
const ranked = candidates
.map((c) => ({
tMs: c.tMs,
score:
THUMBNAIL_WEIGHTS.sharpness * clamp01(c.sharpness) +
THUMBNAIL_WEIGHTS.face * clamp01(c.face) +
THUMBNAIL_WEIGHTS.thirds * clamp01(c.thirds) +
THUMBNAIL_WEIGHTS.contrast * clamp01(c.contrast),
}))
.sort((a, b) => b.score - a.score || a.tMs - b.tMs);
const times = candidates.map((c) => c.tMs);
const spacing = Math.max(1000, (Math.max(...times) - Math.min(...times)) / 10);
const picked: number[] = [];
for (const candidate of ranked) {
if (picked.length >= count) break;
if (picked.every((t) => Math.abs(t - candidate.tMs) >= spacing)) picked.push(candidate.tMs);
}
for (const candidate of ranked) {
if (picked.length >= count) break;
if (!picked.includes(candidate.tMs)) picked.push(candidate.tMs);
}
return picked;
} }

135
packages/media/src/test-fixtures.ts Normal file
View File

@ -0,0 +1,135 @@
import type { BrandKit, Edl, PlatformProfile } from "@rakazo/contracts";
import { BrandKitSchema, EdlSchema, PlatformProfileSchema } from "@rakazo/contracts";
import type { EdlContext } from "./edl.js";
/* Synthetic fixtures shared by the unit tests. No real people, places, or platforms. */
export function testProfile(overrides: Record<string, unknown> = {}): PlatformProfile {
return PlatformProfileSchema.parse({
id: "example-short-vertical",
name: "Example short vertical",
kind: "free_social",
reviewedAt: "2026-09-01",
video: {
aspectRatios: ["9:16"],
width: 1080,
height: 1920,
fps: 30,
codec: "h264",
maxBitrateKbps: 8000,
maxFileSizeMb: 500,
},
duration: { hardMinMs: 1000, hardMaxMs: 60_000, recommended: [7000, 15_000, 30_000] },
safeZones: { topPct: 10, bottomPct: 20, leftPct: 5, rightPct: 12 },
captions: { recommended: true, maxLines: 2 },
audio: {},
text: {
titleMaxChars: 100,
captionMaxChars: 150,
hashtagLimit: 3,
linkPolicy: "bio_only",
bannedPatterns: [{ pattern: "free\\s+money", reason: "misleading claim" }],
},
content: { thresholds: { explicit_nudity: 0.2, violence: 0.5 } },
disclosure: { aiAlteredLabelRequired: true },
consent: { requirePerformerConsent: false, requireAgeVerification: false },
export: {},
...overrides,
});
}
export function testPaidProfile(): PlatformProfile {
return testProfile({
id: "example-paid-video",
name: "Example paid video",
kind: "paid_video",
video: {
aspectRatios: ["16:9"],
width: 1920,
height: 1080,
fps: 30,
codec: "h264",
maxBitrateKbps: 12_000,
maxFileSizeMb: 4000,
},
duration: { hardMinMs: 1000, hardMaxMs: 3_600_000, recommended: [] },
safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 },
text: { captionMaxChars: 2000, hashtagLimit: 10, linkPolicy: "allowed" },
consent: { requirePerformerConsent: true, requireAgeVerification: true },
disclosure: { aiAlteredLabelRequired: true, ageRestrictedFlag: true },
});
}
export function testKit(overrides: Partial<BrandKit> = {}): BrandKit {
return BrandKitSchema.parse({
name: "Studio Example",
colors: {
primary: "#112233",
secondary: "#445566",
accent: "#FFAA00",
text: "#FFFFFF",
background: "#000000",
},
fonts: [{ family: "Example Sans", artifactId: "font-1", role: "caption", license: "OFL" }],
watermark: { artifactId: "logo-1", position: "top_right", opacity: 0.6, scale: 0.15 },
captionPresets: [
{
id: "clean",
name: "Clean",
template: "clean",
fontFamily: "Example Sans",
sizePct: 5,
textColor: "#FFFFFF",
},
],
musicBeds: [{ artifactId: "music-1", title: "Bed", license: "own", commercialUse: true }],
...overrides,
});
}
/** Two 16:9 clips of a 10 s asset, 4 s total, on a vertical profile. */
export function testEdl(overrides: Record<string, unknown> = {}): Edl {
return EdlSchema.parse({
edlVersion: 1,
output: {
profileId: "example-short-vertical",
width: 1080,
height: 1920,
fps: 30,
durationTargetMs: 4000,
durationHardMinMs: 1000,
durationHardMaxMs: 60_000,
},
tracks: {
video: [
{ id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" },
{ id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 2000, reframe: "auto" },
],
},
metadata: {},
...overrides,
});
}
export function testContext(overrides: Partial<EdlContext> = {}): EdlContext {
return {
profile: testProfile(),
brandKit: testKit(),
assets: new Map([
["asset-a", { kind: "video", durationMs: 10_000, width: 1920, height: 1080 }],
["asset-b", { kind: "video", durationMs: 8000, width: 1920, height: 1080 }],
["photo-1", { kind: "photo", durationMs: null, width: 4000, height: 3000 }],
]),
artifactIds: new Set(["music-1", "music-2", "vo-1"]),
music: new Map([
["music-1", { commercialUse: true }],
["music-2", { commercialUse: false }],
]),
markIds: new Set(["mark-1"]),
personIds: new Set(["person-1"]),
faceTrackIds: new Set(["track-1"]),
templateIds: new Set(["intro-minimal", "outro-simple", "lower-third-a"]),
paidOnly: [],
...overrides,
};
}