diff --git a/packages/media/src/__snapshots__/compile.test.ts.snap b/packages/media/src/__snapshots__/compile.test.ts.snap new file mode 100644 index 0000000..94005f5 --- /dev/null +++ b/packages/media/src/__snapshots__/compile.test.ts.snap @@ -0,0 +1,491 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`compileRenderPlan > compiles 16:9 to 9:16 with auto-reframe, captions, watermark and loudnorm 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "subtitles_srt", + "path": "/work/render-1/captions.srt", + }, + { + "kind": "subtitles_vtt", + "path": "/work/render-1/captions.vtt", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "final", + "steps": [ + { + "content": "[Script Info] +ScriptType: v4.00+ +PlayResX: 1080 +PlayResY: 1920 +WrapStyle: 2 +ScaledBorderAndShadow: yes + +[V4+ Styles] +Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding +Style: Default,Example Sans,96,&H00FFFFFF,&H00FFFFFF,&H00000000,&H80000000,0,0,0,0,100,100,0,0,1,2,0,2,92,168,422,1 + +[Events] +Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text +Dialogue: 0,0:00:00.20,0:00:01.80,Default,,0,0,0,,Wait for this! +Dialogue: 0,0:00:02.10,0:00:03.40,Default,,0,0,0,,Here it is --> now. +", + "id": "captions-ass", + "kind": "write_file", + "path": "/work/render-1/captions.ass", + }, + { + "content": "1 +00:00:00,200 --> 00:00:01,800 +Wait for this! + +2 +00:00:02,100 --> 00:00:03,400 +Here it is -> now. +", + "id": "captions-srt", + "kind": "write_file", + "path": "/work/render-1/captions.srt", + }, + { + "content": "WEBVTT + +00:00:00.200 --> 00:00:01.800 +Wait for this! + +00:00:02.100 --> 00:00:03.400 +Here it <b>is</b> --> now. +", + "id": "captions-vtt", + "kind": "write_file", + "path": "/work/render-1/captions.vtt", + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "5", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-i", + "/brand/watermark.png", + "-filter_complex", + "[0:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,170.82\\,if(lt(t\\,1.5)\\,170.82+(167.81)*(t-0)/1.5\\,if(lt(t\\,2)\\,338.62+(19.97)*(t-1.5)/0.5\\,358.59))):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,1.5)\\,0+(0)*(t-0)/1.5\\,if(lt(t\\,2)\\,0+(0)*(t-1.5)/0.5\\,0))),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]crop=w=608:h=1080:x=if(lt(t\\,0)\\,1232\\,if(lt(t\\,2)\\,1232+(0)*(t-0)/2\\,1232)):y=if(lt(t\\,0)\\,0\\,if(lt(t\\,2)\\,0+(0)*(t-0)/2\\,0)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c2];[v_c1][a_c1][v_c2][a_c2]concat=n=2:v=1:a=1[cv1][ca2];[2:v]format=rgba,scale=162:-2,colorchannelmixer=aa=0.6[img3];[cv1][img3]overlay=x=main_w-overlay_w-152:y=214:enable=between(t\\,0\\,4)[ov4];[ov4]ass=filename=/work/render-1/captions.ass:fontsdir=/brand/fonts[cap5];[cap5]format=yuv420p[vout];[ca2]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "medium", + "-crf", + "20", + "-profile:v", + "high", + "-maxrate", + "8000k", + "-bufsize", + "16000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "192k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-metadata", + "title=Example: teaser", + "-t", + "4", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 4000, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "1.333", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; + +exports[`compileRenderPlan > compiles transitions, music ducking, voice-over and fades 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "draft", + "steps": [ + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "2", + "-t", + "2", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "3", + "-t", + "1", + "-i", + "/media/asset-a/proxy.mp4", + "-ss", + "4", + "-t", + "1", + "-i", + "/media/asset-b/original.mov", + "-ss", + "0", + "-t", + "8", + "-i", + "/brand/music/bed.wav", + "-ss", + "0", + "-t", + "2", + "-i", + "/work/render-1/voiceover.wav", + "-filter_complex", + "[0:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=2,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=2,asetpts=PTS-STARTPTS[a_c1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/0.5,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=4,setpts=PTS-STARTPTS[v_c2];[1:a]asetpts=PTS-STARTPTS,atempo=0.5,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=4,asetpts=PTS-STARTPTS[a_c2];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c3];[2:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c3];[3:v]crop=w=608:h=1080:x=656:y=0,scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[v_c4];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[a_c4];[v_c1][v_c2]xfade=transition=fade:duration=0.5:offset=1.5[xv1];[a_c1][a_c2]acrossfade=d=0.5:c1=tri:c2=tri[xa2];color=c=black:s=1080x1920:r=30,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=0.5,setpts=PTS-STARTPTS[gv3];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=0.5,asetpts=PTS-STARTPTS[ga4];[xv1][xa2][gv3][ga4][v_c3][a_c3][v_c4][a_c4]concat=n=4:v=1:a=1[cv5][ca6];[cv5]format=yuv420p[vout];[4:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,volume=-8dB,afade=t=in:st=0:d=0.5,afade=t=out:st=7:d=1[ac_bed];[5:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,adelay=1000|1000[ac_vo];[ca6][ac_vo]amix=inputs=2:duration=first:normalize=0[speech];[speech]asplit=2[speech_mix][speech_key];[ac_bed]anull[duck_bus];[duck_bus][speech_key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400:makeup=1[ducked];[speech_mix][ducked]amix=inputs=2:duration=first:normalize=0[mixed];[mixed]loudnorm=I=-16:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "28", + "-profile:v", + "high", + "-maxrate", + "3000k", + "-bufsize", + "6000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "128k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-t", + "8", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 8000, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "2.667", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; + +exports[`compileRenderPlan > compiles voice disguise, metadata stripping, intro/outro and text overlays 1`] = ` +{ + "outputs": [ + { + "kind": "video", + "path": "/work/render-1/output.mp4", + }, + { + "kind": "poster", + "path": "/work/render-1/poster.jpg", + }, + ], + "quality": "final", + "steps": [ + { + "fps": 30, + "height": 1920, + "id": "motion-1", + "kind": "motion", + "outPath": "/work/render-1/motion-1.mov", + "template": { + "durationMs": 1000, + "params": {}, + "templateId": "outro-simple", + }, + "width": 1080, + }, + { + "fps": 30, + "height": 1920, + "id": "motion-2", + "kind": "motion", + "outPath": "/work/render-1/motion-2.mov", + "template": { + "durationMs": 1000, + "params": {}, + "templateId": "lower-third-a", + }, + "width": 1080, + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "0", + "-t", + "3", + "-i", + "/media/asset-a/proxy.mp4", + "-i", + "/work/render-1/intro.mov", + "-i", + "/work/render-1/motion-1.mov", + "-i", + "/work/render-1/motion-2.mov", + "-filter_complex", + "[0:v]deshake,crop=w=406:h=720:x=if(lt(t\\,0)\\,373\\,if(lt(t\\,3)\\,373+(768)*(t-0)/3\\,1141)):y=if(lt(t\\,0)\\,180\\,if(lt(t\\,3)\\,180+(0)*(t-0)/3\\,180)),scale=1080:1920,setsar=1,setpts=(PTS-STARTPTS)/1,eq=brightness=0.05:contrast=1:saturation=1.1,colorbalance=rm=0.02:bm=-0.02,lut3d=file=/brand/look\\\\\\'s.cube,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=3,setpts=PTS-STARTPTS[v_c1];[0:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[a_c1];[a_c1]rubberband=pitch=0.793701:formant=preserved,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=3,asetpts=PTS-STARTPTS[vd1];[1:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1.5,setpts=PTS-STARTPTS[intro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1.5,asetpts=PTS-STARTPTS[intro_a];[2:v]scale=1080:1920:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=1080:1920:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=30,format=yuv420p,tpad=stop_mode=clone:stop_duration=2,trim=duration=1,setpts=PTS-STARTPTS[outro_v];anullsrc=r=48000:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo,apad,atrim=duration=1,asetpts=PTS-STARTPTS[outro_a];[intro_v][intro_a][v_c1][vd1][outro_v][outro_a]concat=n=3:v=1:a=1[prog_v][prog_a];[3:v]format=yuva420p,scale=1080:1920,setpts=PTS-STARTPTS+0/TB[mo2];[prog_v][mo2]overlay=x=0:y=0:eof_action=pass:enable=between(t\\,0\\,1)[ov3];color=c=0xFFAA00:s=1080x10:r=30:d=5.5[pb4];[ov3][pb4]overlay=x=-overlay_w+main_w*min(1\\,(t-1.5)/3):y=1504:enable=between(t\\,1.5\\,4.5)[ov5];[ov5]drawtext=fontfile=/brand/fonts/ExampleSans.ttf:text=Ana\\\\\\'s\\\\: \\[studio\\]\\; 100\\\\\\\\% real:fontsize=77:fontcolor=0xFFFFFF:x=(w-text_w)/2:y=h-text_h-560:box=1:boxcolor=0x112233@0.85:boxborderw=31:enable=between(t\\,2\\,3.5)[ov6];[ov6]format=yuv420p[vout];[prog_a]loudnorm=I=-14:TP=-1:LRA=11:linear=true,aresample=48000,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]", + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "medium", + "-crf", + "20", + "-profile:v", + "high", + "-maxrate", + "8000k", + "-bufsize", + "16000k", + "-pix_fmt", + "yuv420p", + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + "-bsf:v", + "filter_units=remove_types=6", + "-r", + "30", + "-c:a", + "aac", + "-b:a", + "192k", + "-ar", + "48000", + "-ac", + "2", + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + "-t", + "5.5", + "-movflags", + "+faststart", + "/work/render-1/output.mp4", + ], + "durationMs": 5500, + "id": "render", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/output.mp4", + ], + }, + { + "argv": [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + "2.5", + "-i", + "/work/render-1/output.mp4", + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + "/work/render-1/poster.jpg", + ], + "durationMs": 0, + "id": "poster", + "kind": "ffmpeg", + "produces": [ + "/work/render-1/poster.jpg", + ], + }, + ], + "workDir": "/work/render-1", +} +`; diff --git a/packages/media/src/clustering.test.ts b/packages/media/src/clustering.test.ts new file mode 100644 index 0000000..c2e333d --- /dev/null +++ b/packages/media/src/clustering.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it } from "vitest"; +import type { ClusterAsset } from "./clustering.js"; +import { + buildSessions, + calendarDay, + clusterFaces, + clusterThemes, + ruleLabel, +} from "./clustering.js"; + +const at = (iso: string) => new Date(iso); + +function asset( + id: string, + captured: string | null, + extra: Partial = {}, +): ClusterAsset { + return { + id, + kind: "photo", + capturedAt: captured ? at(captured) : null, + gpsLat: null, + gpsLon: null, + deviceModel: null, + albumNames: [], + embedding: null, + ...extra, + }; +} + +const vec = (...values: number[]) => new Float32Array(values); + +describe("buildSessions", () => { + it("splits on gaps longer than 45 minutes and sorts by capture time", () => { + const sessions = buildSessions([ + asset("c", "2026-05-01T10:40:00Z"), + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:20:00Z"), + asset("d", "2026-05-01T11:26:00Z"), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]); + expect(sessions[0]!.key).toBe("session:a"); + expect(sessions[0]!.startAt).toEqual(at("2026-05-01T10:00:00Z")); + expect(sessions[0]!.endAt).toEqual(at("2026-05-01T10:40:00Z")); + }); + + it("keeps a gap of exactly 45 minutes in one session", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:45:00Z"), + ]); + expect(sessions).toHaveLength(1); + }); + + it("puts undated assets into one trailing session", () => { + const sessions = buildSessions([ + asset("z", null), + asset("a", "2026-05-01T10:00:00Z"), + asset("m", null), + asset("bad", null, { capturedAt: new Date(Number.NaN) }), + ]); + expect(sessions.map((s) => s.key)).toEqual(["session:a", "session:undated"]); + expect(sessions[1]!.assetIds).toEqual(["bad", "m", "z"]); + expect(sessions[1]!.startAt).toBeNull(); + }); + + it("ignores duplicate ids and mixes photos and videos", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z"), + asset("v", "2026-05-01T10:05:00Z", { kind: "video" }), + asset("a", "2026-05-01T23:00:00Z"), + asset("l", "2026-05-01T10:06:00Z", { kind: "live_photo" }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "v", "l"]]); + }); + + it("splits on a location jump over 2 km and reports a rounded centroid", () => { + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z", { gpsLat: 10.0, gpsLon: 20.0 }), + asset("b", "2026-05-01T10:05:00Z"), + asset("c", "2026-05-01T10:10:00Z", { gpsLat: 10.004, gpsLon: 20.004 }), + asset("d", "2026-05-01T10:15:00Z", { gpsLat: 10.1, gpsLon: 20.0 }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b", "c"], ["d"]]); + expect(sessions[0]!.centroid).toEqual({ lat: 10, lon: 20 }); + expect(sessions[1]!.centroid).toEqual({ lat: 10.1, lon: 20 }); + }); + + it("breaks on a calendar-day change only with a gap over 3 hours, in the given time zone", () => { + const assets = [asset("a", "2026-05-01T22:30:00Z"), asset("b", "2026-05-02T02:00:00Z")]; + // Long gap rule disabled so only the day-break rule decides. + const options = { gapMinutes: 24 * 60 }; + expect(buildSessions(assets, options)).toHaveLength(2); + // In UTC-10 both are on 1 May: one session. + expect(buildSessions(assets, { ...options, timeZone: "Pacific/Honolulu" })).toHaveLength(1); + // Crossing midnight with a short gap stays one session. + expect( + buildSessions( + [asset("x", "2026-05-01T23:50:00Z"), asset("y", "2026-05-02T00:20:00Z")], + options, + ), + ).toHaveLength(1); + expect(calendarDay(at("2026-05-01T22:30:00Z"), "Europe/Berlin")).toBe("2026-05-02"); + }); + + it("merges neighbours with similar embeddings and a gap under 2 hours", () => { + const red = vec(1, 0, 0); + const blue = vec(0, 1, 0); + const sessions = buildSessions([ + asset("a", "2026-05-01T10:00:00Z", { embedding: red }), + asset("b", "2026-05-01T11:00:00Z", { embedding: vec(0.95, 0.05, 0) }), + asset("c", "2026-05-01T12:00:00Z", { embedding: blue }), + asset("d", "2026-05-01T15:00:00Z", { embedding: blue }), + ]); + expect(sessions.map((s) => s.assetIds)).toEqual([["a", "b"], ["c"], ["d"]]); + }); + + it("is deterministic for any input order", () => { + const assets = [ + asset("a", "2026-05-01T10:00:00Z"), + asset("b", "2026-05-01T10:00:00Z"), + asset("c", "2026-05-02T10:00:00Z"), + asset("d", null), + ]; + const forward = buildSessions(assets); + const reversed = buildSessions([...assets].reverse()); + expect(reversed).toEqual(forward); + expect(forward[0]!.assetIds).toEqual(["a", "b"]); + }); +}); + +describe("clusterThemes and clusterFaces", () => { + const items = [ + { id: "p3", vector: vec(0.9, 0.1, 0) }, + { id: "p1", vector: vec(1, 0, 0) }, + { id: "q1", vector: vec(0, 1, 0) }, + { id: "p2", vector: vec(0.95, 0.05, 0) }, + { id: "q2", vector: vec(0.05, 0.95, 0) }, + { id: "r1", vector: vec(0, 0, 1) }, + ]; + + it("groups by average linkage with stable keys, independent of input order", () => { + const themes = clusterThemes(items); + expect(themes).toEqual([ + { key: "theme:p1", ids: ["p1", "p2", "p3"] }, + { key: "theme:q1", ids: ["q1", "q2"] }, + { key: "theme:r1", ids: ["r1"] }, + ]); + expect(clusterThemes([...items].reverse())).toEqual(themes); + expect(clusterThemes(items, { minSize: 2 }).map((t) => t.key)).toEqual([ + "theme:p1", + "theme:q1", + ]); + expect(clusterThemes(items, { similarity: -1 })).toHaveLength(1); + expect(clusterThemes([])).toEqual([]); + }); + + it("uses average rather than single linkage", () => { + // b is close to a and c, but a and c are far apart: average linkage keeps c out. + const chain = [ + { id: "a", vector: vec(1, 0) }, + { id: "b", vector: vec(Math.cos(0.4), Math.sin(0.4)) }, + { id: "c", vector: vec(Math.cos(1.0), Math.sin(1.0)) }, + ]; + expect(clusterThemes(chain, { similarity: 0.8 }).map((g) => g.ids)).toEqual([ + ["a", "b"], + ["c"], + ]); + }); + + it("returns face groups with a mean centroid", () => { + const faces = clusterFaces([ + { id: "t1", vector: vec(1, 0) }, + { id: "t2", vector: vec(0.8, 0.2) }, + { id: "t3", vector: vec(0, 1) }, + ]); + expect(faces.map((f) => f.key)).toEqual(["person:t1", "person:t3"]); + expect(Array.from(faces[0]!.centroid)).toEqual([0.8999999761581421, 0.10000000149011612]); + expect(() => + clusterFaces([ + { id: "x", vector: vec(1) }, + { id: "y", vector: vec(1, 2) }, + ]), + ).toThrow(); + }); +}); + +describe("ruleLabel", () => { + it("combines the dominant album with a locale-aware date range", () => { + const label = ruleLabel( + { + startAt: at("2026-05-01T10:00:00Z"), + endAt: at("2026-05-03T10:00:00Z"), + albumNames: ["Trip", "Favorites", "Trip"], + }, + "en-US", + ); + expect(label.startsWith("Trip · ")).toBe(true); + expect(label).toMatch(/May 1\D+3, 2026/); + expect( + ruleLabel({ startAt: at("2026-05-01T10:00:00Z"), endAt: null, albumNames: [] }, "de-DE"), + ).toBe("01.05.2026"); + expect(ruleLabel({ startAt: null, endAt: null, albumNames: [" "] })).toBe(""); + expect(ruleLabel({ startAt: null, endAt: null, albumNames: ["B", "A"] })).toBe("A"); + }); +}); diff --git a/packages/media/src/clustering.ts b/packages/media/src/clustering.ts index 59b696e..31bf572 100644 --- a/packages/media/src/clustering.ts +++ b/packages/media/src/clustering.ts @@ -1,5 +1,5 @@ import type { MediaAssetKind } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import { cosineSimilarity, haversineKm, roundCoordinate } from "./geo.js"; export interface ClusterAsset { id: string; @@ -46,15 +46,157 @@ export interface SessionCluster { centroid: { lat: number; lon: number } | null; } +const MINUTE_MS = 60_000; +const HOUR_MS = 60 * MINUTE_MS; + +const dayFormatters = new Map(); + +/** Calendar day (YYYY-MM-DD) of an instant in an IANA time zone. */ +export function calendarDay(date: Date, timeZone: string): string { + let formatter = dayFormatters.get(timeZone); + if (!formatter) { + formatter = new Intl.DateTimeFormat("en-CA", { + timeZone, + year: "numeric", + month: "2-digit", + day: "2-digit", + }); + dayFormatters.set(timeZone, formatter); + } + return formatter.format(date); +} + +type DatedAsset = ClusterAsset & { capturedAt: Date }; +type PositionedAsset = ClusterAsset & { gpsLat: number; gpsLon: number }; + +function hasPosition(asset: ClusterAsset): asset is PositionedAsset { + return ( + asset.gpsLat !== null && + asset.gpsLon !== null && + Number.isFinite(asset.gpsLat) && + Number.isFinite(asset.gpsLon) + ); +} + +function isDated(asset: ClusterAsset): asset is DatedAsset { + return asset.capturedAt !== null && Number.isFinite(asset.capturedAt.getTime()); +} + +/** Mean of the given vectors, or null when none are present or dimensions disagree. */ +export function meanVector(vectors: ArrayLike[]): Float32Array | null { + const first = vectors[0]; + if (!first) return null; + const sum = new Float64Array(first.length); + for (const vector of vectors) { + if (vector.length !== first.length) return null; + for (let i = 0; i < vector.length; i++) sum[i] = sum[i]! + (vector[i] as number); + } + const mean = new Float32Array(first.length); + for (let i = 0; i < sum.length; i++) mean[i] = sum[i]! / vectors.length; + return mean; +} + +function sessionEmbedding(assets: ClusterAsset[]): Float32Array | null { + return meanVector(assets.flatMap((asset) => (asset.embedding ? [asset.embedding] : []))); +} + +function finishSession(assets: ClusterAsset[], key: string): SessionCluster { + const dated = assets.filter(isDated); + const positioned = assets.filter(hasPosition); + const mean = (values: number[]) => values.reduce((sum, v) => sum + v, 0) / values.length; + return { + key, + assetIds: assets.map((asset) => asset.id), + startAt: dated[0]?.capturedAt ?? null, + endAt: dated.at(-1)?.capturedAt ?? null, + centroid: + positioned.length === 0 + ? null + : { + lat: roundCoordinate(mean(positioned.map((a) => a.gpsLat))), + lon: roundCoordinate(mean(positioned.map((a) => a.gpsLon))), + }, + }; +} + /** * Deterministic session grouping (spec 4.4). Assets without capturedAt form * one trailing "undated" session; exact duplicates (same id) are ignored. + * + * A new session starts on a gap longer than gapMinutes, a jump of more than + * maxDistanceKm from the session's last known position, or a calendar-day + * change (in timeZone) with a gap longer than dayBreakHours. Adjacent sessions + * are then merged when their mean embeddings reach mergeSimilarity and the gap + * between them stays below mergeMaxGapHours. Keys derive from the earliest + * asset id, so they survive re-clustering as long as that asset stays first. */ export function buildSessions( - _assets: ClusterAsset[], - _options?: Partial, + assets: ClusterAsset[], + options?: Partial, ): SessionCluster[] { - return notImplemented("buildSessions"); + const opts = { ...DEFAULT_SESSION_OPTIONS, ...options }; + const unique = new Map(); + for (const asset of assets) if (!unique.has(asset.id)) unique.set(asset.id, asset); + + const dated = [...unique.values()] + .filter(isDated) + .sort((a, b) => a.capturedAt.getTime() - b.capturedAt.getTime() || a.id.localeCompare(b.id)); + const undated = [...unique.values()] + .filter((asset) => !isDated(asset)) + .sort((a, b) => a.id.localeCompare(b.id)); + + const split: DatedAsset[][] = []; + let lastPosition: { lat: number; lon: number } | null = null; + for (const asset of dated) { + const current = split.at(-1); + const previous = current?.at(-1); + let startNew = true; + if (current && previous) { + const gap = asset.capturedAt.getTime() - previous.capturedAt.getTime(); + const jumped = + hasPosition(asset) && + lastPosition !== null && + haversineKm(lastPosition, { lat: asset.gpsLat, lon: asset.gpsLon }) > opts.maxDistanceKm; + const dayChanged = + gap > opts.dayBreakHours * HOUR_MS && + calendarDay(asset.capturedAt, opts.timeZone) !== + calendarDay(previous.capturedAt, opts.timeZone); + startNew = gap > opts.gapMinutes * MINUTE_MS || jumped || dayChanged; + } + if (startNew || !current) { + split.push([asset]); + lastPosition = null; + } else { + current.push(asset); + } + if (hasPosition(asset)) lastPosition = { lat: asset.gpsLat, lon: asset.gpsLon }; + } + + // Merge visually similar neighbours that are close in time. + const merged: DatedAsset[][] = []; + for (const session of split) { + const last = merged.at(-1); + if (last) { + const gap = session[0]!.capturedAt.getTime() - last.at(-1)!.capturedAt.getTime(); + const a = sessionEmbedding(last); + const b = sessionEmbedding(session); + if ( + a && + b && + a.length === b.length && + gap < opts.mergeMaxGapHours * HOUR_MS && + cosineSimilarity(a, b) >= opts.mergeSimilarity + ) { + last.push(...session); + continue; + } + } + merged.push([...session]); + } + + const sessions = merged.map((group) => finishSession(group, `session:${group[0]!.id}`)); + if (undated.length > 0) sessions.push(finishSession(undated, "session:undated")); + return sessions; } export interface VectorItem { @@ -62,26 +204,144 @@ export interface VectorItem { vector: Float32Array; } -/** Agglomerative average-linkage clustering; returns groups with stable keys. */ +/** + * Average-linkage agglomerative clustering cut at a similarity threshold, + * using the nearest-neighbour chain algorithm (O(n²) time and memory). Items + * are sorted by id first and ties resolve to the lower index, so the result + * does not depend on input order. Callers batch very large libraries. + */ +function agglomerate(items: VectorItem[], threshold: number): VectorItem[][] { + const unique = new Map(); + for (const item of items) if (!unique.has(item.id)) unique.set(item.id, item); + const sorted = [...unique.values()].sort((a, b) => a.id.localeCompare(b.id)); + const n = sorted.length; + if (n === 0) return []; + const dim = sorted[0]!.vector.length; + if (sorted.some((item) => item.vector.length !== dim)) throw new Error("vector length mismatch"); + + const sim = new Float64Array(n * n); + for (let i = 0; i < n; i++) { + for (let j = i + 1; j < n; j++) { + const value = cosineSimilarity(sorted[i]!.vector, sorted[j]!.vector); + sim[i * n + j] = value; + sim[j * n + i] = value; + } + } + const size = new Array(n).fill(1); + const members: number[][] = sorted.map((_, i) => [i]); + const active = new Set(members.map((_, i) => i)); + const finished: number[][] = []; + const chain: number[] = []; + + while (active.size > 0) { + if (chain.length === 0) chain.push(Math.min(...active)); + const top = chain.at(-1)!; + const below = chain.length >= 2 ? chain.at(-2)! : -1; + let best = -1; + let bestSim = Number.NEGATIVE_INFINITY; + for (const j of active) { + if (j === top) continue; + const s = sim[top * n + j]!; + if (s > bestSim || (s === bestSim && j < best)) { + best = j; + bestSim = s; + } + } + // Prefer the chain predecessor on ties so the chain cannot cycle. + if (below >= 0 && sim[top * n + below]! === bestSim) best = below; + if (best < 0 || bestSim < threshold) { + // Average linkage is reducible: merges never raise similarity to this cluster again. + active.delete(top); + finished.push(members[top]!); + chain.length = 0; + continue; + } + if (best !== below) { + chain.push(best); + continue; + } + chain.length -= 2; + const a = Math.min(top, below); + const b = Math.max(top, below); + for (const k of active) { + if (k === a || k === b) continue; + const value = + (size[a]! * sim[a * n + k]! + size[b]! * sim[b * n + k]!) / (size[a]! + size[b]!); + sim[a * n + k] = value; + sim[k * n + a] = value; + } + size[a] = size[a]! + size[b]!; + members[a] = [...members[a]!, ...members[b]!]; + active.delete(b); + } + return finished.map((group) => group.sort((x, y) => x - y).map((i) => sorted[i]!)); +} + +function sortGroups(groups: T[]): T[] { + return groups.sort((a, b) => b.ids.length - a.ids.length || a.key.localeCompare(b.key)); +} + +/** + * Agglomerative average-linkage clustering; returns groups with stable keys + * (from the smallest member id), largest first. Default similarity 0.8. + */ export function clusterThemes( - _items: VectorItem[], - _options?: { similarity?: number; minSize?: number }, + items: VectorItem[], + options?: { similarity?: number; minSize?: number }, ): Array<{ key: string; ids: string[] }> { - return notImplemented("clusterThemes"); + const minSize = options?.minSize ?? 1; + return sortGroups( + agglomerate(items, options?.similarity ?? 0.8) + .filter((group) => group.length >= minSize) + .map((group) => ({ key: `theme:${group[0]!.id}`, ids: group.map((item) => item.id) })), + ); } -/** Group face tracks into person suggestions by identity embedding. Local only. */ +/** Group face tracks into person suggestions by identity embedding. Local only. Default similarity 0.6. */ export function clusterFaces( - _tracks: VectorItem[], - _options?: { similarity?: number }, + tracks: VectorItem[], + options?: { similarity?: number }, ): Array<{ key: string; ids: string[]; centroid: Float32Array }> { - return notImplemented("clusterFaces"); + return sortGroups( + agglomerate(tracks, options?.similarity ?? 0.6).map((group) => ({ + key: `person:${group[0]!.id}`, + ids: group.map((item) => item.id), + centroid: meanVector(group.map((item) => item.vector))!, + })), + ); } -/** Rule-based title from date range, place centroid, and dominant album. No model involved. */ +/** + * Rule-based title from the date range and dominant album. Places are not + * named because only rounded coordinates are stored. Returns "" when there is + * nothing to say, so the caller shows its own localized placeholder. + */ export function ruleLabel( - _cluster: { startAt: Date | null; endAt: Date | null; albumNames: string[] }, - _locale?: string, + cluster: { startAt: Date | null; endAt: Date | null; albumNames: string[] }, + locale?: string, + timeZone = "UTC", ): string { - return notImplemented("ruleLabel"); + const counts = new Map(); + for (const raw of cluster.albumNames) { + const name = raw.trim(); + if (name) counts.set(name, (counts.get(name) ?? 0) + 1); + } + const album = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]; + + let dates = ""; + const start = cluster.startAt ?? cluster.endAt; + const end = cluster.endAt ?? cluster.startAt; + if (start && end) { + let formatter: Intl.DateTimeFormat; + try { + formatter = new Intl.DateTimeFormat(locale, { dateStyle: "medium", timeZone }); + } catch { + formatter = new Intl.DateTimeFormat("en", { dateStyle: "medium", timeZone }); + } + dates = + calendarDay(start, timeZone) === calendarDay(end, timeZone) + ? formatter.format(start) + : formatter.formatRange(start, end); + } + return [album?.[0], dates].filter(Boolean).join(" · "); } diff --git a/packages/media/src/compile.ffmpeg.test.ts b/packages/media/src/compile.ffmpeg.test.ts new file mode 100644 index 0000000..cd6051e --- /dev/null +++ b/packages/media/src/compile.ffmpeg.test.ts @@ -0,0 +1,322 @@ +import { execFile, execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import type { RenderPlan } from "@rakazo/contracts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { CompileInput } from "./compile.js"; +import { compileRenderPlan, escapeFilterText, escapeFilterValue } from "./compile.js"; +import { edlDurationMs } from "./edl.js"; +import { findMetadataLeaks } from "./gate.js"; +import { testEdl, testKit, testProfile } from "./test-fixtures.js"; + +const run = promisify(execFile); + +function onPath(binary: string): boolean { + try { + execFileSync(binary, ["-version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +const hasFfmpeg = onPath("ffmpeg") && onPath("ffprobe"); +const hasRubberband = + hasFfmpeg && + execFileSync("ffmpeg", ["-hide_banner", "-filters"]).toString().includes(" rubberband "); + +async function ffmpeg(args: string[]): Promise { + await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...args]); +} + +async function execute(plan: RenderPlan): Promise { + for (const step of plan.steps) { + if (step.kind === "write_file") await writeFile(step.path, step.content); + else if (step.kind === "ffmpeg") await run("ffmpeg", ["-loglevel", "error", ...step.argv]); + else throw new Error(`unexpected ${step.kind} step`); + } +} + +interface Probe { + durationMs: number; + width: number; + height: number; + tags: Record; +} + +async function probe(file: string): Promise { + const { stdout } = await run("ffprobe", [ + "-v", + "error", + "-show_format", + "-show_streams", + "-of", + "json", + file, + ]); + const json = JSON.parse(stdout) as { + format: { duration: string; tags?: Record }; + streams: Array<{ + codec_type: string; + width?: number; + height?: number; + tags?: Record; + }>; + }; + const tags: Record = {}; + for (const [key, value] of Object.entries(json.format.tags ?? {})) tags[`format.${key}`] = value; + json.streams.forEach((stream, index) => { + for (const [key, value] of Object.entries(stream.tags ?? {})) + tags[`stream.${index}.${key}`] = value; + }); + const video = json.streams.find((s) => s.codec_type === "video"); + return { + durationMs: Number(json.format.duration) * 1000, + width: video?.width ?? 0, + height: video?.height ?? 0, + tags, + }; +} + +describe.skipIf(!hasFfmpeg)("compiled plans run on ffmpeg", () => { + let dir: string; + let source: string; + let watermark: string; + let music: string; + let intro: string; + let lut: string; + + beforeAll(async () => { + dir = await mkdtemp(path.join(tmpdir(), "media-compile-")); + source = path.join(dir, "source.mp4"); + watermark = path.join(dir, "watermark.png"); + music = path.join(dir, "music.wav"); + intro = path.join(dir, "intro.mov"); + lut = path.join(dir, "brand look.cube"); + // Identity 2x2x2 LUT. + const lattice = []; + for (const b of [0, 1]) + for (const g of [0, 1]) for (const r of [0, 1]) lattice.push(`${r} ${g} ${b}`); + await writeFile(lut, `LUT_3D_SIZE 2\n${lattice.join("\n")}\n`); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc2=size=480x270:rate=30:duration=8", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=8", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-c:a", + "aac", + "-shortest", + // Tags a phone would write; the export must drop every one of them. + "-metadata", + "creation_time=2020-01-01T10:00:00Z", + "-metadata", + "location=+00.0000+000.0000/", + "-metadata", + "make=ExampleCam", + source, + ]); + await ffmpeg(["-f", "lavfi", "-i", "color=c=white:s=64x32", "-frames:v", "1", watermark]); + await ffmpeg(["-f", "lavfi", "-i", "sine=frequency=220:duration=6", music]); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc=size=320x180:rate=30:duration=1", + "-c:v", + "libx264", + "-preset", + "ultrafast", + intro, + ]); + }); + + afterAll(async () => { + if (dir) await rm(dir, { recursive: true, force: true }); + }); + + const baseInput = (overrides: Partial): CompileInput => ({ + edl: testEdl(), + profile: testProfile(), + brandKit: testKit(), + quality: "draft", + workDir: dir, + sources: new Map([ + [ + "asset-a", + { + path: source, + width: 480, + height: 270, + durationMs: 8000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + ]), + transcripts: new Map([ + [ + "asset-a", + [ + { startMs: 200, endMs: 600, text: "Wait" }, + { startMs: 650, endMs: 1200, text: "for" }, + { startMs: 1250, endMs: 1800, text: "this!" }, + { startMs: 5100, endMs: 5600, text: "Here" }, + { startMs: 5650, endMs: 6400, text: "it is." }, + ], + ], + ]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.3, cy: 0.5, strength: 1 }, + { tMs: 6000, cx: 0.7, cy: 0.5, strength: 1 }, + ], + ], + ]), + motionClips: new Map(), + files: { + lutPath: null, + watermarkPath: watermark, + fonts: new Map(), + audio: new Map([["music-1", music]]), + }, + videoEncoder: "libx264", + ...overrides, + }); + + it("renders 16:9 to 9:16 with auto-reframe, captions, watermark, ducked music and clean metadata", async () => { + const edl = testEdl({ + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 1500, reframe: "auto" }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: "music-1", + inMs: 0, + outMs: 3500, + atMs: 0, + gainDb: -6, + fadeOutMs: 500, + duckUnderSpeech: true, + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }], + captions: { mode: "both" }, + overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 3500 }], + metadata: { title: "Example teaser" }, + }); + const plan = compileRenderPlan(baseInput({ edl })); + await execute(plan); + const video = plan.outputs.find((o) => o.kind === "video")!.path; + const result = await probe(video); + expect(result.width).toBe(1080); + expect(result.height).toBe(1920); + expect(Math.abs(result.durationMs - edlDurationMs(edl))).toBeLessThanOrEqual(1000 / 30 + 1); + const keys = Object.keys(result.tags).map((k) => k.toLowerCase()); + expect(keys.some((k) => k.includes("creation_time") || k.includes("location"))).toBe(false); + expect(findMetadataLeaks(result.tags)).toEqual([]); + expect(result.tags["format.title"]).toBe("Example teaser"); + + const srt = await readFile(plan.outputs.find((o) => o.kind === "subtitles_srt")!.path, "utf8"); + expect(srt).toContain("Wait for this!"); + const poster = plan.outputs.find((o) => o.kind === "poster")!.path; + expect(existsSync(poster)).toBe(true); + expect((await probe(poster)).tags).toEqual({}); + }); + + it("renders an intro, voice disguise and an escaped text overlay", async () => { + const edl = testEdl({ + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 1000, + outMs: 3000, + atMs: 0, + reframe: "none", + stabilize: true, + color: { exposure: 0.3, temperature: -0.4, saturation: 1.2, useBrandLut: true }, + }, + ], + }, + intro: { templateId: "intro-minimal", durationMs: 1000 }, + overlays: [ + { + id: "t1", + kind: "text", + atMs: 1000, + durationMs: 1500, + text: "50% off: it's [0:v]; rm -rf / %{pts} \\ done", + box: { x: 0.1, y: 0.3, w: 0.8, h: 0.05 }, + }, + { id: "lt", kind: "lower_third", atMs: 1200, durationMs: 1000, text: "Lower third" }, + { id: "pb", kind: "progress_bar", atMs: 1000, durationMs: 2000 }, + ], + redactions: { voice: { method: hasRubberband ? "pitch" : "formant", semitones: 3 } }, + metadata: {}, + }); + const plan = compileRenderPlan( + baseInput({ + edl, + motionClips: new Map([["intro", intro]]), + quality: "final", + files: { ...baseInput({}).files, lutPath: lut }, + }), + ); + await execute(plan); + const result = await probe(plan.outputs[0]!.path); + expect(result.width).toBe(1080); + expect(Math.abs(result.durationMs - 3000)).toBeLessThanOrEqual(1000 / 30 + 1); + expect(findMetadataLeaks(result.tags)).toEqual([]); + expect(Object.keys(result.tags).some((k) => k.endsWith("title"))).toBe(false); + }); + + it("draws escaped text exactly like the literal text", async () => { + const text = "it's 50% [0:v]; a,b=c \\ %{pts} 'q'"; + const literal = path.join(dir, "literal.txt"); + await writeFile(literal, text); + const frame = async (filter: string) => { + const { stdout } = await run("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `color=c=black:s=320x80:d=0.1,${filter}`, + "-frames:v", + "1", + "-f", + "framemd5", + "-", + ]); + return stdout.split("\n").filter((line) => line && !line.startsWith("#")); + }; + const escaped = await frame(`drawtext=text=${escapeFilterText(text)}:fontcolor=white:x=4:y=20`); + const reference = await frame( + `drawtext=expansion=none:textfile=${escapeFilterValue(literal)}:fontcolor=white:x=4:y=20`, + ); + const blank = await frame("null"); + expect(escaped).toEqual(reference); + expect(escaped).not.toEqual(blank); + }); +}); diff --git a/packages/media/src/compile.test.ts b/packages/media/src/compile.test.ts new file mode 100644 index 0000000..1a14ebf --- /dev/null +++ b/packages/media/src/compile.test.ts @@ -0,0 +1,576 @@ +import type { RenderPlan } from "@rakazo/contracts"; +import { RenderPlanSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { CaptionWord, CompileInput } from "./compile.js"; +import { + atempoChain, + autoReframe, + buildCaptionCues, + compileRenderPlan, + escapeFilterText, + escapeFilterValue, + outputCaptionWords, + reframeExcluding, + toAss, + toSrt, + toVtt, +} from "./compile.js"; +import { testEdl, testKit, testProfile } from "./test-fixtures.js"; + +const words: CaptionWord[] = [ + { startMs: 200, endMs: 600, text: "Wait" }, + { startMs: 650, endMs: 1200, text: "for" }, + { startMs: 1250, endMs: 1800, text: "this!" }, + { startMs: 5100, endMs: 5600, text: "Here" }, + { startMs: 5650, endMs: 6400, text: "it is --> now." }, +]; + +function input(overrides: Partial = {}): CompileInput { + return { + edl: testEdl(), + profile: testProfile(), + brandKit: testKit(), + quality: "final", + workDir: "/work/render-1", + sources: new Map([ + [ + "asset-a", + { + path: "/media/asset-a/proxy.mp4", + width: 1920, + height: 1080, + durationMs: 10_000, + fps: 30, + hasAudio: true, + rotation: 0, + }, + ], + [ + "asset-b", + { + path: "/media/asset-b/original.mov", + width: 1080, + height: 1920, + durationMs: 8000, + fps: 60, + hasAudio: false, + rotation: 90, + }, + ], + ]), + transcripts: new Map([["asset-a", words]]), + subjects: new Map([ + [ + "asset-a", + [ + { tMs: 0, cx: 0.2, cy: 0.5, strength: 1 }, + { tMs: 1500, cx: 0.4, cy: 0.5, strength: 1 }, + { tMs: 5500, cx: 0.8, cy: 0.4, strength: 0.5 }, + ], + ], + ]), + motionClips: new Map(), + files: { + lutPath: "/brand/look's.cube", + watermarkPath: "/brand/watermark.png", + fonts: new Map([["font-1", "/brand/fonts/ExampleSans.ttf"]]), + audio: new Map([ + ["music-1", "/brand/music/bed.wav"], + ["vo-1", "/work/render-1/voiceover.wav"], + ]), + }, + videoEncoder: "libx264", + ...overrides, + }; +} + +function ffmpegStep(plan: RenderPlan, id = "render") { + const step = plan.steps.find((s) => s.id === id); + if (step?.kind !== "ffmpeg") throw new Error(`no ffmpeg step ${id}`); + return step; +} + +function graphOf(plan: RenderPlan): string { + const argv = ffmpegStep(plan).argv; + return argv[argv.indexOf("-filter_complex") + 1]!; +} + +describe("compileRenderPlan", () => { + it("compiles 16:9 to 9:16 with auto-reframe, captions, watermark and loudnorm", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + captions: { mode: "both", presetId: "clean" }, + overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 4000 }], + metadata: { title: "Example: teaser" }, + }), + }), + ); + expect(RenderPlanSchema.parse(plan)).toEqual(plan); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("crop=w=608:h=1080:x="); + expect(graph).toContain("scale=1080:1920"); + expect(graph).toContain("loudnorm=I=-14:TP=-1:LRA=11:linear=true"); + expect(graph).toContain("colorchannelmixer=aa=0.6"); + expect(graph).toContain("ass=filename=/work/render-1/captions.ass"); + expect(plan.outputs.map((o) => o.kind)).toEqual([ + "video", + "subtitles_srt", + "subtitles_vtt", + "poster", + ]); + const argv = ffmpegStep(plan).argv; + for (const flag of [ + "-map_metadata", + "-map_chapters", + "+bitexact", + "+faststart", + "yuv420p", + "bt709", + ]) { + expect(argv).toContain(flag); + } + expect(argv).toContain("title=Example: teaser"); + expect(argv.join(" ")).toContain("-c:a aac -b:a 192k -ar 48000"); + }); + + it("compiles transitions, music ducking, voice-over and fades", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" }, + { + id: "c2", + assetId: "asset-a", + inMs: 2000, + outMs: 4000, + atMs: 1500, + reframe: "none", + speed: 0.5, + }, + { + id: "c3", + assetId: "asset-a", + inMs: 3000, + outMs: 4000, + atMs: 6000, + reframe: "none", + }, + { + id: "c4", + assetId: "asset-b", + inMs: 4000, + outMs: 5000, + atMs: 7000, + reframe: "auto", + }, + ], + audio: [ + { + id: "bed", + kind: "music", + artifactId: "music-1", + inMs: 0, + outMs: 8000, + atMs: 0, + gainDb: -8, + fadeInMs: 500, + fadeOutMs: 1000, + duckUnderSpeech: true, + }, + { id: "vo", kind: "voiceover", artifactId: "vo-1", inMs: 0, outMs: 2000, atMs: 1000 }, + ], + }, + transitions: [ + { fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }, + { fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 }, + { fromClipId: "c3", toClipId: "c4", kind: "whip", durationMs: 0 }, + ], + targetLufs: -16, + }), + quality: "draft", + }), + ); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("xfade=transition=fade:duration=0.5:offset=1.5"); + expect(graph).toContain("acrossfade=d=0.5"); + expect(graph).toContain("atempo=0.5"); + expect(graph).toContain("color=c=black:s=1080x1920"); // gap from 5.5 s to 6 s + expect(graph).toContain("sidechaincompress"); + expect(graph).toContain("adelay=1000|1000"); + expect(graph).toContain("loudnorm=I=-16"); + expect(ffmpegStep(plan).argv).toEqual(expect.arrayContaining(["veryfast", "28", "3000k"])); + }); + + it("compiles voice disguise, metadata stripping, intro/outro and text overlays", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1500 }, + outro: { templateId: "outro-simple", durationMs: 1000 }, + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 0, + outMs: 3000, + atMs: 0, + reframe: [ + { tMs: 0, cx: 0.3, cy: 0.5, scale: 1 }, + { tMs: 3000, cx: 0.7, cy: 0.5, scale: 1.5 }, + ], + stabilize: true, + color: { exposure: 0.5, temperature: 0.2, saturation: 1.1, useBrandLut: true }, + }, + ], + }, + overlays: [ + { + id: "lt", + kind: "lower_third", + atMs: 2000, + durationMs: 1500, + text: "Ana's: [studio]; 100% real", + }, + { + id: "logo-anim", + kind: "motion", + atMs: 0, + durationMs: 1000, + motion: { templateId: "lower-third-a", durationMs: 1000 }, + }, + { id: "pb", kind: "progress_bar", atMs: 1500, durationMs: 3000 }, + ], + redactions: { voice: { method: "pitch", semitones: -4 } }, + metadata: {}, + }), + motionClips: new Map([["intro", "/work/render-1/intro.mov"]]), + }), + ); + expect(plan).toMatchSnapshot(); + const graph = graphOf(plan); + expect(graph).toContain("rubberband=pitch=0.793701:formant=preserved"); + expect(graph).toContain("deshake"); + expect(graph).toContain("lut3d=file=/brand/look\\\\\\'s.cube"); + expect(graph).toContain("concat=n=3:v=1:a=1[prog_v][prog_a]"); + expect(plan.steps.filter((s) => s.kind === "motion").map((s) => s.id)).toEqual([ + "motion-1", + "motion-2", + ]); + const argv = ffmpegStep(plan).argv; + expect(argv.some((a) => a.startsWith("title="))).toBe(false); + expect(argv).toEqual(expect.arrayContaining(["-map_metadata", "-1", "-map_chapters", "-1"])); + }); + + it("uses the asetrate chain for formant disguise and HEVC settings for HEVC profiles", () => { + const plan = compileRenderPlan( + input({ + profile: testProfile({ + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "hevc", + maxBitrateKbps: 6000, + maxFileSizeMb: 100, + }, + export: { poster: false }, + }), + edl: testEdl({ redactions: { voice: { method: "formant", semitones: 12 } }, metadata: {} }), + videoEncoder: "hevc_videotoolbox", + }), + ); + expect(graphOf(plan)).toContain("asetrate=96000,aresample=48000,atempo=0.5"); + const argv = ffmpegStep(plan).argv; + expect(argv).toEqual(expect.arrayContaining(["-tag:v", "hvc1", "-b:v", "4800k"])); + expect(argv).toContain("filter_units=remove_types=39|40"); + expect(plan.outputs.map((o) => o.kind)).toEqual(["video"]); + }); + + it("only ever uses SafeId labels and keeps text out of argv", () => { + const plan = compileRenderPlan( + input({ + edl: testEdl({ + overlays: [ + { id: "t1", kind: "text", atMs: 0, durationMs: 1000, text: "'; rm -rf / [0:v] %{pts}" }, + ], + }), + }), + ); + const graph = graphOf(plan); + for (const label of graph.matchAll(/\[([^\]\\]*)\]/g)) { + expect(label[1]).toMatch(/^[a-z0-9_:-]+$/); + } + expect(graph).toContain("text=\\\\\\'\\; rm -rf / \\[0\\\\:v\\] \\\\\\\\%{pts}"); + expect(ffmpegStep(plan).argv.every((arg) => !arg.includes("\0"))).toBe(true); + }); + + it("refuses unsafe or impossible input instead of guessing", () => { + expect(() => + compileRenderPlan( + input({ + edl: { + ...testEdl(), + overlays: [{ id: "BAD;ID", kind: "text", atMs: 0, durationMs: 1, text: "x" }], + } as never, + }), + ), + ).toThrow(); + expect(() => compileRenderPlan(input({ videoEncoder: "libx265" }))).toThrow( + /does not produce h264/, + ); + expect(() => compileRenderPlan(input({ sources: new Map() }))).toThrow(/no source/); + expect(() => + compileRenderPlan( + input({ + edl: testEdl({ overlays: [{ id: "wm", kind: "watermark", atMs: 0, durationMs: 1000 }] }), + files: { ...input().files, watermarkPath: null }, + }), + ), + ).toThrow(/watermark/); + }); + + it("moves very large filtergraphs into a script file", () => { + const video = Array.from({ length: 300 }, (_, i) => ({ + id: `c${i}`, + assetId: "asset-a", + inMs: 0, + outMs: 100, + atMs: i * 100, + reframe: Array.from({ length: 8 }, (_, k) => ({ + tMs: k * 12, + cx: 0.3 + k * 0.05, + cy: 0.5, + scale: 1, + })), + })); + const plan = compileRenderPlan(input({ edl: testEdl({ tracks: { video } }) })); + const script = plan.steps.find((s) => s.id === "filtergraph"); + expect(script?.kind).toBe("write_file"); + expect(ffmpegStep(plan).argv).toContain("-filter_complex_script"); + }); +}); + +describe("escaping", () => { + it("escapes every filtergraph and drawtext special character", () => { + expect(escapeFilterText("plain text")).toBe("plain text"); + expect(escapeFilterText("a:b")).toBe("a\\\\:b"); + expect(escapeFilterText("a,b;c")).toBe("a\\,b\\;c"); + expect(escapeFilterText("[0:v]")).toBe("\\[0\\\\:v\\]"); + expect(escapeFilterText("x=y")).toBe("x\\\\=y"); + expect(escapeFilterText("it's")).toBe("it\\\\\\'s"); + expect(escapeFilterText("100%")).toBe("100\\\\\\\\%"); + expect(escapeFilterText("a\\b")).toBe("a\\\\\\\\\\\\\\\\b"); + expect(escapeFilterText(" line one\r\nline two\u0007 ")).toBe("line one\nline two"); + }); + + /** av_get_token: one parser level; returns the token and what it left unconsumed. */ + function getToken(value: string, terms: string): { token: string; rest: string } { + let token = ""; + let i = 0; + while (i < value.length && !terms.includes(value[i]!)) { + const char = value[i++]!; + if (char === "\\" && i < value.length) token += value[i++]; + else if (char === "'") { + while (i < value.length && value[i] !== "'") token += value[i++]; + i++; + } else token += char; + } + return { token, rest: value.slice(i) }; + } + + function drawtextExpand(value: string): string { + let out = ""; + for (let i = 0; i < value.length; i++) { + if (value[i] === "\\" && i + 1 < value.length) out += value[++i]; + else if (value[i] === "%") throw new Error("expansion sequence reached drawtext"); + else out += value[i]; + } + return out; + } + + it("neutralizes injection attempts", () => { + const attempts = [ + "'; rm -rf /", + "[0:v]", + "%{pts}", + "%{eif:1:d}", + "x:fontfile=/etc/passwd", + "a',drawtext=text='b", + "a\\", + "50% off; [in]null[out]", + ]; + for (const attempt of attempts) { + const graphLevel = getToken(escapeFilterText(attempt), "[],;"); + expect(graphLevel.rest).toBe(""); + const optionLevel = getToken(graphLevel.token, ":"); + expect(optionLevel.rest).toBe(""); + expect(drawtextExpand(optionLevel.token)).toBe(attempt); + } + expect(escapeFilterText("%{pts}")).toBe("\\\\\\\\%{pts}"); + expect(escapeFilterValue("C:\\media\\a,b.cube")).toBe("C\\\\:\\\\\\\\media\\\\\\\\a\\,b.cube"); + expect(() => escapeFilterValue("a\0b")).toThrow(); + }); + + it("chains atempo within its clean range", () => { + expect(atempoChain(1)).toEqual([]); + expect(atempoChain(0.25)).toEqual(["atempo=0.5", "atempo=0.5"]); + expect(atempoChain(3)).toEqual(["atempo=2", "atempo=1.5"]); + expect(atempoChain(4)).toEqual(["atempo=2", "atempo=2"]); + }); +}); + +describe("captions", () => { + it("builds cues by line length, line count, pauses and sentence ends", () => { + const cues = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 1 }); + expect(cues.map((c) => c.text)).toEqual(["Wait for", "this!", "Here", "it is --> now."]); + const two = buildCaptionCues(words, { maxCharsPerLine: 10, maxLines: 2 }); + expect(two[0]).toMatchObject({ startMs: 200, endMs: 1800, text: "Wait for\nthis!" }); + expect(two[0]!.words).toHaveLength(3); + expect( + buildCaptionCues([{ startMs: 0, endMs: 0, text: " " }], { maxCharsPerLine: 10, maxLines: 2 }), + ).toEqual([]); + const short = buildCaptionCues([{ startMs: 1000, endMs: 1100, text: "Hi" }], { + maxCharsPerLine: 10, + maxLines: 2, + }); + expect(short[0]!.endMs).toBe(1500); + }); + + it("formats SRT and VTT", () => { + const cues = buildCaptionCues(words, { maxCharsPerLine: 32, maxLines: 2 }); + expect(toSrt(cues)).toBe( + "1\n00:00:00,200 --> 00:00:01,800\nWait for this!\n\n2\n00:00:05,100 --> 00:00:06,400\nHere it is -> now.\n", + ); + expect(toVtt(cues)).toBe( + "WEBVTT\n\n00:00:00.200 --> 00:00:01.800\nWait for this!\n\n00:00:05.100 --> 00:00:06.400\nHere it <b>is</b> --> now.\n", + ); + expect(toSrt([{ startMs: 3_723_004, endMs: 3_723_500, text: "a\n\nb", words: [] }])).toBe( + "1\n01:02:03,004 --> 01:02:03,500\na\nb\n", + ); + }); + + it("maps source words to output time across speed and intro", () => { + const edl = testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1000 }, + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 1000, + outMs: 2000, + atMs: 0, + reframe: "none", + speed: 2, + }, + { + id: "c2", + assetId: "asset-a", + inMs: 5000, + outMs: 6000, + atMs: 500, + reframe: "none", + muted: true, + }, + ], + }, + }); + expect(outputCaptionWords(edl, new Map([["asset-a", words]]))).toEqual([ + { startMs: 1125, endMs: 1400, text: "this!" }, + ]); + }); + + it("writes ASS without override injection and with karaoke timing", () => { + const cues = buildCaptionCues( + [ + { startMs: 0, endMs: 400, text: "{\\pos(0,0)}hi" }, + { startMs: 600, endMs: 900, text: "you\\N" }, + ], + { maxCharsPerLine: 32, maxLines: 2 }, + ); + const ass = toAss(cues, { + width: 1080, + height: 1920, + profile: testProfile(), + preset: { + ...testKit().captionPresets[0]!, + template: "karaoke", + highlightColor: "#FFAA00", + uppercase: true, + }, + fallbackTextColor: "#FFFFFF", + position: "lower", + maxCharsPerLine: 32, + wordHighlight: true, + }); + const dialogue = ass.split("\n").find((line) => line.startsWith("Dialogue:"))!; + expect(dialogue).toBe( + "Dialogue: 0,0:00:00.00,0:00:00.90,Default,,0,0,0,,{\\k40}\uFF5B\\\u2060POS(0,0)\uFF5DHI {\\k20}{\\k30}YOU\\\u2060N", + ); + expect(ass).toContain("Style: Default,Example Sans,96,&H0000AAFF,&H00FFFFFF"); + expect(ass).toContain(",2,92,168,422,1"); + }); +}); + +describe("reframing", () => { + const geometry = { + sourceWidth: 1920, + sourceHeight: 1080, + outWidth: 1080, + outHeight: 1920, + durationMs: 4000, + }; + + it("follows the subject smoothly and keeps the crop inside the frame", () => { + const path = autoReframe( + [ + { tMs: 0, cx: 0, cy: 0.5, strength: 1 }, + { tMs: 2000, cx: 0.5, cy: 0.5, strength: 1 }, + { tMs: 4000, cx: 1, cy: 0.5, strength: 1 }, + ], + geometry, + ); + const half = 608 / 1920 / 2; + for (const k of path) { + expect(k.cx).toBeGreaterThanOrEqual(half - 1e-4); + expect(k.cx).toBeLessThanOrEqual(1 - half + 1e-4); + expect(k.cy).toBe(0.5); + expect(k.scale).toBe(1); + } + expect(path[0]!.tMs).toBe(0); + expect(path.at(-1)!.tMs).toBe(4000); + for (let i = 1; i < path.length; i++) + expect(path[i]!.cx).toBeGreaterThanOrEqual(path[i - 1]!.cx); + expect(path.length).toBeLessThanOrEqual(64); + expect(autoReframe([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]); + }); + + it("keeps faces out of the frame or reports that it cannot", () => { + const left = reframeExcluding( + [{ tMs: 100, box: { x: 0.05, y: 0.3, w: 0.15, h: 0.3 } }], + geometry, + ); + expect(left).toHaveLength(1); + const k = left![0]!; + expect(k.tMs).toBe(100); + expect(k.scale).toBe(1); + expect(k.cx - 608 / 1920 / 2).toBeGreaterThanOrEqual(0.22 - 1e-6); + // Faces all over the frame: no crop can exclude them. + const everywhere = [0, 0.25, 0.5, 0.75].map((x) => ({ + tMs: 0, + box: { x, y: 0, w: 0.25, h: 1 }, + })); + expect(reframeExcluding(everywhere, geometry)).toBeNull(); + // A face across the middle forces a tighter crop. + const wide = reframeExcluding([{ tMs: 0, box: { x: 0.2, y: 0.4, w: 0.6, h: 0.2 } }], geometry); + expect(wide![0]!.scale).toBeGreaterThan(1); + expect(reframeExcluding([], geometry)).toEqual([{ tMs: 0, cx: 0.5, cy: 0.5, scale: 1 }]); + }); +}); diff --git a/packages/media/src/compile.ts b/packages/media/src/compile.ts index 913e831..ab16f88 100644 --- a/packages/media/src/compile.ts +++ b/packages/media/src/compile.ts @@ -1,23 +1,43 @@ import type { BrandKit, + CaptionStylePreset, Edl, + EdlAudioClip, + EdlOverlay, + EdlVideoClip, MediaRenderQuality, + MotionTemplateRef, NormalizedBox, PlatformProfile, ReframeKeyframe, RenderPlan, + RenderStep, } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import { EdlSchema } from "@rakazo/contracts"; +import { + BLENDED_TRANSITIONS, + clipLengthMs, + edlDurationMs, + introMs, + mainProgramMs, + sortedVideoClips, +} from "./edl.js"; export interface CompileSource { /** Redacted intermediate when redactions apply, otherwise original or proxy. */ path: string; + /** + * Coded size as stored. Rotation 90/270 swaps it for display, because + * ffmpeg auto-rotates while decoding. + */ width: number; height: number; durationMs: number; fps: number; hasAudio: boolean; rotation: 0 | 90 | 180 | 270; + /** A still image (photo). Also assumed when durationMs is 0. */ + still?: boolean; } export interface CaptionWord { @@ -38,28 +58,358 @@ export interface CompileInput { transcripts: Map; /** Saliency or subject points per asset for "auto" reframe. */ subjects: Map>; - /** Pre-rendered motion clips keyed by overlay id, "intro", "outro". */ + /** + * Pre-rendered motion clips keyed by overlay id, "intro", "outro". Missing + * ones become `motion` steps that the renderer runs first. + */ motionClips: Map; files: { lutPath: string | null; watermarkPath: string | null; + /** Brand-kit font artifact id (or family name) → font file. */ fonts: Map; /** artifactId → absolute path for audio clips. */ audio: Map; + /** First brand-kit logo, for logo overlays without a motion clip. */ + logoPath?: string | null; }; /** ffmpeg video encoder name, e.g. libx264, h264_vaapi, h264_videotoolbox. */ videoEncoder: string; + /** Output time of the poster frame. Defaults to a third into the main program. */ + posterAtMs?: number; +} + +// --------------------------------------------------------------------------- +// Escaping +// --------------------------------------------------------------------------- + +/* + * A filtergraph value passes three parsers: + * 1. the graph parser splits filters on `[ ] , ;` and honours `\` and `'`, + * 2. the option parser splits options on `:` (and `=`), again with `\` and `'`, + * 3. drawtext expands `%{...}` and treats `\x` as a literal `x`. + * Each level is escaped with a backslash, innermost first, so any user text + * arrives verbatim and can never open a new option, filter, or expansion. + */ +const escapeDrawtextLevel = (value: string) => value.replace(/[\\%]/g, "\\$&"); +const escapeOptionLevel = (value: string) => value.replace(/[\\':=]/g, "\\$&"); +const escapeGraphLevel = (value: string) => value.replace(/[\\'[\],;]/g, "\\$&"); + +/** Escape text for ffmpeg drawtext/filtergraph values. */ +export function escapeFilterText(text: string): string { + const normalized = text + .replace(/\r\n?/g, "\n") + // biome-ignore lint/suspicious/noControlCharactersInRegex: strip control characters on purpose + .replace(/[\u0000-\u0009\u000b-\u001f\u007f]/g, "") + .trim(); + return escapeGraphLevel(escapeOptionLevel(escapeDrawtextLevel(normalized))); +} + +/** Escape a file path or expression used as a filter option value. */ +export function escapeFilterValue(value: string): string { + if (value.includes("\0")) throw new Error("filter value contains NUL"); + return escapeGraphLevel(escapeOptionLevel(value)); +} + +// --------------------------------------------------------------------------- +// Small helpers +// --------------------------------------------------------------------------- + +/** Fixed-point number without exponent notation. */ +function num(value: number, digits = 3): string { + const fixed = value.toFixed(digits); + return fixed.includes(".") ? fixed.replace(/\.?0+$/, "") || "0" : fixed; +} + +const sec = (ms: number) => num(ms / 1000); + +function joinPath(dir: string, name: string): string { + return `${dir.replace(/[\\/]+$/, "")}/${name}`; +} + +function dirName(file: string): string { + const index = Math.max(file.lastIndexOf("/"), file.lastIndexOf("\\")); + return index > 0 ? file.slice(0, index) : "."; +} + +function brandColor(hex: string | undefined, fallback: string): string { + return hex ? `0x${hex.slice(1).toUpperCase()}` : fallback; +} + +function even(value: number): number { + return Math.max(2, Math.floor(value / 2) * 2); +} + +const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value)); + +function enableBetween(startMs: number, endMs: number): string { + return escapeFilterValue(`between(t,${sec(startMs)},${sec(endMs)})`); +} + +/** atempo accepts 0.5..2 per instance for clean results; chain for the rest. */ +export function atempoChain(speed: number): string[] { + const factors: number[] = []; + let rest = speed; + while (rest > 2) { + factors.push(2); + rest /= 2; + } + while (rest < 0.5) { + factors.push(0.5); + rest /= 0.5; + } + if (Math.abs(rest - 1) > 1e-9 || factors.length === 0) factors.push(rest); + return factors.filter((f) => Math.abs(f - 1) > 1e-9).map((f) => `atempo=${num(f, 6)}`); +} + +// --------------------------------------------------------------------------- +// Reframing +// --------------------------------------------------------------------------- + +interface FrameGeometry { + sourceWidth: number; + sourceHeight: number; + outWidth: number; + outHeight: number; + durationMs: number; +} + +/** Largest crop with the output aspect inside the source, shrunk by scale. */ +function cropSize(geometry: FrameGeometry, scale: number): { w: number; h: number } { + const outAspect = geometry.outWidth / geometry.outHeight; + let w: number; + let h: number; + if (geometry.sourceWidth / geometry.sourceHeight > outAspect) { + h = geometry.sourceHeight; + w = h * outAspect; + } else { + w = geometry.sourceWidth; + h = w / outAspect; + } + const roundEven = (v: number) => Math.max(2, Math.round(v / 2) * 2); + return { + w: Math.min(geometry.sourceWidth, roundEven(w / scale)), + h: Math.min(geometry.sourceHeight, roundEven(h / scale)), + }; +} + +const AUTO_STEP_MS = 500; +const AUTO_WINDOW_MS = 750; +const AUTO_SMOOTHING = 0.35; +const AUTO_TOLERANCE = 0.01; +const MAX_KEYFRAMES = 64; + +function smooth(values: number[]): number[] { + const forward = [...values]; + for (let i = 1; i < forward.length; i++) { + forward[i] = forward[i - 1]! + AUTO_SMOOTHING * (forward[i]! - forward[i - 1]!); + } + const backward = [...forward]; + for (let i = backward.length - 2; i >= 0; i--) { + backward[i] = backward[i + 1]! + AUTO_SMOOTHING * (backward[i]! - backward[i + 1]!); + } + return backward; +} + +/** Keep only keyframes that linear interpolation cannot reproduce, at most MAX_KEYFRAMES. */ +function simplify(keyframes: ReframeKeyframe[]): ReframeKeyframe[] { + if (keyframes.length <= 2) return keyframes; + const kept: ReframeKeyframe[] = [keyframes[0]!]; + for (let i = 1; i < keyframes.length - 1; i++) { + const last = kept.at(-1)!; + const next = keyframes[i + 1]!; + const point = keyframes[i]!; + const f = (point.tMs - last.tMs) / (next.tMs - last.tMs); + const cx = last.cx + (next.cx - last.cx) * f; + const cy = last.cy + (next.cy - last.cy) * f; + if (Math.abs(cx - point.cx) > AUTO_TOLERANCE || Math.abs(cy - point.cy) > AUTO_TOLERANCE) { + kept.push(point); + } + } + kept.push(keyframes.at(-1)!); + if (kept.length <= MAX_KEYFRAMES) return kept; + const step = (kept.length - 1) / (MAX_KEYFRAMES - 1); + return Array.from({ length: MAX_KEYFRAMES }, (_, i) => kept[Math.round(i * step)]!); } /** - * Pure compiler from a validated EDL to ffmpeg argv steps plus caption files. - * No string ever reaches a shell; text is escaped for filtergraph and ASS, - * every label comes from SafeId values. + * Smoothed crop path that keeps the subject in frame for the output aspect. + * Subjects are weighted by strength in a ±750 ms window every 500 ms, the + * path is smoothed forwards and backwards (no lag), clamped so the crop stays + * inside the source, and reduced to the keyframes that matter. Times are the + * subjects' time base, from 0 to durationMs. */ -export function compileRenderPlan(_input: CompileInput): RenderPlan { - return notImplemented("compileRenderPlan"); +export function autoReframe( + subjects: Array<{ tMs: number; cx: number; cy: number; strength: number }>, + input: FrameGeometry, +): ReframeKeyframe[] { + const crop = cropSize(input, 1); + const halfW = crop.w / input.sourceWidth / 2; + const halfH = crop.h / input.sourceHeight / 2; + const clampX = (x: number) => clamp(x, halfW, 1 - halfW); + const clampY = (y: number) => clamp(y, halfH, 1 - halfH); + const usable = subjects + .filter((s) => s.strength > 0 && Number.isFinite(s.cx) && Number.isFinite(s.cy)) + .sort((a, b) => a.tMs - b.tMs); + if (usable.length === 0) return [{ tMs: 0, cx: clampX(0.5), cy: clampY(0.5), scale: 1 }]; + + const times: number[] = []; + for (let t = 0; t < input.durationMs; t += AUTO_STEP_MS) times.push(t); + times.push(Math.max(0, input.durationMs)); + const xs: number[] = []; + const ys: number[] = []; + for (const t of times) { + let weight = 0; + let x = 0; + let y = 0; + for (const subject of usable) { + const distance = Math.abs(subject.tMs - t); + if (distance > AUTO_WINDOW_MS) continue; + const w = subject.strength * (1 - distance / (AUTO_WINDOW_MS * 2)); + weight += w; + x += subject.cx * w; + y += subject.cy * w; + } + if (weight > 0) { + xs.push(x / weight); + ys.push(y / weight); + } else if (xs.length > 0) { + xs.push(xs.at(-1)!); + ys.push(ys.at(-1)!); + } else { + const nearest = usable.reduce((best, s) => + Math.abs(s.tMs - t) < Math.abs(best.tMs - t) ? s : best, + ); + xs.push(nearest.cx); + ys.push(nearest.cy); + } + } + const sx = smooth(xs); + const sy = smooth(ys); + const round = (v: number) => Math.round(v * 10_000) / 10_000; + return simplify( + times.map((tMs, i) => ({ + tMs: Math.round(tMs), + cx: round(clampX(sx[i]!)), + cy: round(clampY(sy[i]!)), + scale: 1, + })), + ); } +const REFRAME_SCALES = [1, 1.25, 1.5, 1.75, 2, 2.5, 3, 3.5, 4]; +const FACE_MARGIN = 0.02; + +/** + * Crop path that keeps every given face box outside the output frame, or + * null if impossible. The window is static for the whole shot (a moving + * window could sweep across a face between keyframes): it avoids the union of + * all face boxes plus a 2 % margin, prefers the widest crop, and then the + * position closest to the frame centre. + */ +export function reframeExcluding( + faces: Array<{ tMs: number; box: NormalizedBox }>, + input: FrameGeometry, +): ReframeKeyframe[] | null { + const tMs = faces.reduce((min, face) => Math.min(min, face.tMs), Number.POSITIVE_INFINITY); + const boxes = faces.map(({ box }) => ({ + x0: clamp(box.x - FACE_MARGIN, 0, 1), + y0: clamp(box.y - FACE_MARGIN, 0, 1), + x1: clamp(box.x + box.w + FACE_MARGIN, 0, 1), + y1: clamp(box.y + box.h + FACE_MARGIN, 0, 1), + })); + for (const scale of REFRAME_SCALES) { + const crop = cropSize(input, scale); + const w = crop.w / input.sourceWidth; + const h = crop.h / input.sourceHeight; + const xs = new Set([0.5, w / 2, 1 - w / 2]); + const ys = new Set([0.5, h / 2, 1 - h / 2]); + for (const b of boxes) { + xs.add(b.x0 - w / 2); + xs.add(b.x1 + w / 2); + ys.add(b.y0 - h / 2); + ys.add(b.y1 + h / 2); + } + const candidates: Array<{ cx: number; cy: number; distance: number }> = []; + for (const rawX of xs) { + for (const rawY of ys) { + const cx = clamp(rawX, w / 2, 1 - w / 2); + const cy = clamp(rawY, h / 2, 1 - h / 2); + const clear = boxes.every( + (b) => + cx + w / 2 <= b.x0 + 1e-9 || + cx - w / 2 >= b.x1 - 1e-9 || + cy + h / 2 <= b.y0 + 1e-9 || + cy - h / 2 >= b.y1 - 1e-9, + ); + if (clear) candidates.push({ cx, cy, distance: Math.hypot(cx - 0.5, cy - 0.5) }); + } + } + candidates.sort((a, b) => a.distance - b.distance || a.cx - b.cx || a.cy - b.cy); + const best = candidates[0]; + if (best) { + const round = (v: number) => Math.round(v * 10_000) / 10_000; + return [ + { + tMs: Number.isFinite(tMs) ? Math.round(tMs) : 0, + cx: round(best.cx), + cy: round(best.cy), + scale, + }, + ]; + } + } + return null; +} + +/** Piecewise-linear expression of `t` (seconds) through the given points. */ +function piecewise(points: Array<{ t: number; v: number }>): string { + if (points.length === 1) return num(points[0]!.v, 2); + let expression = num(points.at(-1)!.v, 2); + for (let i = points.length - 2; i >= 0; i--) { + const a = points[i]!; + const b = points[i + 1]!; + const span = Math.max(b.t - a.t, 0.001); + const segment = `${num(a.v, 2)}+(${num(b.v - a.v, 2)})*(t-${num(a.t)})/${num(span)}`; + expression = `if(lt(t,${num(b.t)}),${segment},${expression})`; + } + return `if(lt(t,${num(points[0]!.t)}),${num(points[0]!.v, 2)},${expression})`; +} + +/** crop+scale filters for keyframes in source time; uses the tightest scale of the path. */ +function cropFilters( + keyframes: ReframeKeyframe[], + clip: EdlVideoClip, + geometry: FrameGeometry, +): string[] { + const scale = Math.max(...keyframes.map((k) => k.scale)); + const crop = cropSize(geometry, scale); + const points = [...keyframes] + .sort((a, b) => a.tMs - b.tMs) + .map((k) => ({ + t: Math.max(0, (k.tMs - clip.inMs) / 1000), + x: clamp(k.cx * geometry.sourceWidth - crop.w / 2, 0, geometry.sourceWidth - crop.w), + y: clamp(k.cy * geometry.sourceHeight - crop.h / 2, 0, geometry.sourceHeight - crop.h), + })); + const x = piecewise(points.map((p) => ({ t: p.t, v: p.x }))); + const y = piecewise(points.map((p) => ({ t: p.t, v: p.y }))); + return [ + `crop=w=${crop.w}:h=${crop.h}:x=${escapeFilterValue(x)}:y=${escapeFilterValue(y)}`, + `scale=${geometry.outWidth}:${geometry.outHeight}`, + ]; +} + +function fitFilters(width: number, height: number): string[] { + return [ + `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`, + `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black`, + ]; +} + +// --------------------------------------------------------------------------- +// Captions +// --------------------------------------------------------------------------- + export interface CaptionCue { startMs: number; endMs: number; @@ -68,50 +418,952 @@ export interface CaptionCue { words: CaptionWord[]; } +const CUE_PAUSE_MS = 1000; +const CUE_MAX_MS = 7000; +const CUE_MIN_MS = 500; + +function cleanWord(word: CaptionWord): CaptionWord | null { + const text = word.text.replace(/\s+/g, " ").trim(); + if (!text || !(word.endMs >= word.startMs)) return null; + return { startMs: word.startMs, endMs: word.endMs, text }; +} + +/** Greedy line wrap; a single word longer than the limit gets its own line. */ +function wrapWords(words: CaptionWord[], maxChars: number): CaptionWord[][] { + const lines: CaptionWord[][] = []; + let line: CaptionWord[] = []; + let length = 0; + for (const word of words) { + const extra = line.length === 0 ? word.text.length : length + 1 + word.text.length; + if (line.length > 0 && extra > maxChars) { + lines.push(line); + line = [word]; + length = word.text.length; + } else { + line.push(word); + length = extra; + } + } + if (line.length > 0) lines.push(line); + return lines; +} + +/** + * Group words into cues of at most maxLines lines of maxCharsPerLine. A cue + * also ends at a pause over 1 s, after sentence punctuation, or at 7 s. Each + * cue is shown at least 500 ms unless the next one starts earlier. + */ export function buildCaptionCues( - _words: CaptionWord[], - _options: { maxCharsPerLine: number; maxLines: number }, + words: CaptionWord[], + options: { maxCharsPerLine: number; maxLines: number }, ): CaptionCue[] { - return notImplemented("buildCaptionCues"); + const clean = words + .map(cleanWord) + .filter((w): w is CaptionWord => w !== null) + .sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs); + const groups: CaptionWord[][] = []; + let current: CaptionWord[] = []; + for (const word of clean) { + const last = current.at(-1); + const first = current[0]; + const breaks = + last !== undefined && + first !== undefined && + (word.startMs - last.endMs > CUE_PAUSE_MS || + /[.!?…]["»”)]?$/.test(last.text) || + word.endMs - first.startMs > CUE_MAX_MS || + wrapWords([...current, word], options.maxCharsPerLine).length > options.maxLines); + if (breaks) { + groups.push(current); + current = []; + } + current.push(word); + } + if (current.length > 0) groups.push(current); + + return groups.map((group, index) => { + const startMs = group[0]!.startMs; + const next = groups[index + 1]?.[0]?.startMs ?? Number.POSITIVE_INFINITY; + const spoken = Math.max(...group.map((w) => w.endMs)); + const endMs = Math.max(spoken, Math.min(startMs + CUE_MIN_MS, next)); + return { + startMs, + endMs: Math.max(endMs, startMs + 1), + text: wrapWords(group, options.maxCharsPerLine) + .map((line) => line.map((w) => w.text).join(" ")) + .join("\n"), + words: group, + }; + }); } -export function toSrt(_cues: CaptionCue[]): string { - return notImplemented("toSrt"); +function clock(ms: number, separator: "," | "."): string { + const total = Math.max(0, Math.round(ms)); + const hours = Math.floor(total / 3_600_000); + const minutes = Math.floor((total % 3_600_000) / 60_000); + const seconds = Math.floor((total % 60_000) / 1000); + const millis = total % 1000; + const pad = (v: number, n = 2) => String(v).padStart(n, "0"); + return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}${separator}${pad(millis, 3)}`; } -export function toVtt(_cues: CaptionCue[]): string { - return notImplemented("toVtt"); +/** Cue text without blank lines, which would end a cue early in both formats. */ +function cueLines(text: string): string { + return text + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .join("\n"); } -/** Smoothed crop path that keeps the subject in frame for the output aspect. */ -export function autoReframe( - _subjects: Array<{ tMs: number; cx: number; cy: number; strength: number }>, - _input: { - sourceWidth: number; - sourceHeight: number; - outWidth: number; - outHeight: number; - durationMs: number; +export function toSrt(cues: CaptionCue[]): string { + return cues + .map( + (cue, index) => + `${index + 1}\n${clock(cue.startMs, ",")} --> ${clock(cue.endMs, ",")}\n${cueLines( + cue.text, + ).replace(/-->/g, "->")}\n`, + ) + .join("\n"); +} + +export function toVtt(cues: CaptionCue[]): string { + const escapeCue = (text: string) => + text.replace(/&/g, "&").replace(//g, ">"); + return `WEBVTT\n\n${cues + .map( + (cue) => + `${clock(cue.startMs, ".")} --> ${clock(cue.endMs, ".")}\n${escapeCue(cueLines(cue.text))}\n`, + ) + .join("\n")}`; +} + +/** ASS colour &HAABBGGRR from #RRGGBB. */ +function assColor(hex: string, alpha = 0): string { + const r = hex.slice(1, 3); + const g = hex.slice(3, 5); + const b = hex.slice(5, 7); + return `&H${alpha.toString(16).padStart(2, "0")}${b}${g}${r}`.toUpperCase(); +} + +function assTime(ms: number): string { + const total = Math.max(0, Math.round(ms / 10)); + const hours = Math.floor(total / 360_000); + const minutes = Math.floor((total % 360_000) / 6000); + const seconds = Math.floor((total % 6000) / 100); + const centis = total % 100; + const pad = (v: number) => String(v).padStart(2, "0"); + return `${hours}:${pad(minutes)}:${pad(seconds)}.${pad(centis)}`; +} + +/** Plain text for an ASS event: no override blocks, no escape sequences. */ +function assText(text: string): string { + return text.replace(/\\/g, "\\⁠").replace(/\{/g, "｛").replace(/\}/g, "｝"); +} + +export function toAss( + cues: CaptionCue[], + options: { + width: number; + height: number; + profile: PlatformProfile; + preset: CaptionStylePreset | null; + fallbackTextColor: string; + position: "lower" | "middle" | "upper"; + maxCharsPerLine: number; + wordHighlight: boolean; + language?: string; }, -): ReframeKeyframe[] { - return notImplemented("autoReframe"); +): string { + const { width, height, profile, preset } = options; + const zones = profile.safeZones; + const margin = Math.round(height * 0.02); + const alignment = { lower: 2, middle: 5, upper: 8 }[options.position]; + const marginV = + options.position === "upper" + ? Math.round((zones.topPct / 100) * height) + margin + : options.position === "lower" + ? Math.round((zones.bottomPct / 100) * height) + margin + : 0; + const marginL = Math.round((zones.leftPct / 100) * width) + margin; + const marginR = Math.round((zones.rightPct / 100) * width) + margin; + const textColor = preset?.textColor ?? options.fallbackTextColor; + const highlight = preset?.highlightColor ?? textColor; + const karaoke = options.wordHighlight || preset?.template === "karaoke"; + const bold = preset?.template === "bold_social"; + const fontName = (preset?.fontFamily ?? "Sans").replace(/[,\r\n]/g, " "); + const fontSize = Math.round(((preset?.sizePct ?? 5) / 100) * height); + const background = preset?.backgroundColor; + const style = [ + "Default", + fontName, + fontSize, + assColor(karaoke ? highlight : textColor), + assColor(textColor), + background ? assColor(background) : "&H00000000", + background ? assColor(background, 0x40) : "&H80000000", + bold ? -1 : 0, + 0, + 0, + 0, + 100, + 100, + 0, + 0, + background ? 3 : 1, + bold ? 4 : 2, + 0, + alignment, + marginL, + marginR, + marginV, + 1, + ].join(","); + const transform = (text: string) => + preset?.uppercase ? text.toLocaleUpperCase(options.language) : text; + const events = cues.map((cue) => { + let body: string; + if (karaoke && cue.words.length > 0) { + let cursor = cue.startMs; + body = wrapWords(cue.words, options.maxCharsPerLine) + .map((line) => + line + .map((word) => { + const gap = Math.max(0, Math.round((word.startMs - cursor) / 10)); + const length = Math.max(1, Math.round((word.endMs - word.startMs) / 10)); + cursor = Math.max(cursor, word.endMs); + return `${gap > 0 ? `{\\k${gap}}` : ""}{\\k${length}}${assText(transform(word.text))}`; + }) + .join(" "), + ) + .join("\\N"); + } else { + body = cueLines(transform(cue.text)).split("\n").map(assText).join("\\N"); + } + return `Dialogue: 0,${assTime(cue.startMs)},${assTime(cue.endMs)},Default,,0,0,0,,${body}`; + }); + return [ + "[Script Info]", + "ScriptType: v4.00+", + `PlayResX: ${width}`, + `PlayResY: ${height}`, + "WrapStyle: 2", + "ScaledBorderAndShadow: yes", + "", + "[V4+ Styles]", + "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding", + `Style: ${style}`, + "", + "[Events]", + "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text", + ...events, + "", + ].join("\n"); } -/** Crop path that keeps every given face box outside the output frame, or null if impossible. */ -export function reframeExcluding( - _faces: Array<{ tMs: number; box: NormalizedBox }>, - _input: { - sourceWidth: number; - sourceHeight: number; - outWidth: number; - outHeight: number; - durationMs: number; - }, -): ReframeKeyframe[] | null { - return notImplemented("reframeExcluding"); +/** Caption words on the output timeline, from each unmuted clip's transcript. */ +export function outputCaptionWords( + edl: Edl, + transcripts: Map, +): CaptionWord[] { + const offset = introMs(edl); + const words: CaptionWord[] = []; + for (const clip of sortedVideoClips(edl)) { + if (clip.muted) continue; + for (const word of transcripts.get(clip.assetId) ?? []) { + const start = Math.max(word.startMs, clip.inMs); + const end = Math.min(word.endMs, clip.outMs); + if (word.startMs < clip.inMs || start >= clip.outMs || end < start) continue; + words.push({ + startMs: Math.round(offset + clip.atMs + (start - clip.inMs) / clip.speed), + endMs: Math.round(offset + clip.atMs + (end - clip.inMs) / clip.speed), + text: word.text, + }); + } + } + return words; } -/** Escape text for ffmpeg drawtext/filtergraph values. */ -export function escapeFilterText(_text: string): string { - return notImplemented("escapeFilterText"); +// --------------------------------------------------------------------------- +// Compiler +// --------------------------------------------------------------------------- + +const AUDIO_FORMAT = "aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo"; +const XFADE_KIND = { crossfade: "fade", dip: "fadeblack", whip: "slideleft" } as const; +const MAX_INLINE_GRAPH = 60_000; + +function encoderFamily(encoder: string): "h264" | "hevc" | null { + if (/^(libx264|h264_)/.test(encoder)) return "h264"; + if (/^(libx265|hevc_)/.test(encoder)) return "hevc"; + return null; +} + +function encodingArgs(input: CompileInput): string[] { + const { profile, quality, videoEncoder } = input; + const family = encoderFamily(videoEncoder); + if (family !== profile.video.codec) { + throw new Error(`encoder ${videoEncoder} does not produce ${profile.video.codec}`); + } + const draft = quality === "draft"; + const maxKbps = draft + ? Math.min(profile.video.maxBitrateKbps, 3000) + : profile.video.maxBitrateKbps; + const args = ["-c:v", videoEncoder]; + if (videoEncoder === "libx264" || videoEncoder === "libx265") { + args.push("-preset", draft ? "veryfast" : "medium", "-crf", draft ? "28" : "20"); + if (videoEncoder === "libx264") args.push("-profile:v", "high"); + } else { + args.push("-b:v", `${Math.round(maxKbps * 0.8)}k`); + } + args.push("-maxrate", `${maxKbps}k`, "-bufsize", `${maxKbps * 2}k`); + if (!videoEncoder.endsWith("_vaapi")) args.push("-pix_fmt", "yuv420p"); + args.push( + "-colorspace", + "bt709", + "-color_primaries", + "bt709", + "-color_trc", + "bt709", + "-color_range", + "tv", + ); + // Drop SEI messages, which carry encoder name, version and settings. + args.push( + "-bsf:v", + family === "h264" ? "filter_units=remove_types=6" : "filter_units=remove_types=39|40", + ); + if (family === "hevc") args.push("-tag:v", "hvc1"); + args.push("-r", num(input.edl.output.fps, 3)); + args.push("-c:a", "aac", "-b:a", draft ? "128k" : "192k", "-ar", "48000", "-ac", "2"); + return args; +} + +/** Strip every inherited tag, chapter, and encoder string; only the plan's title is written. */ +function metadataArgs(title: string | undefined): string[] { + const args = [ + "-map_metadata", + "-1", + "-map_chapters", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-flags:a", + "+bitexact", + ]; + const clean = title?.replace(/[\0\r\n]+/g, " ").trim(); + if (clean) args.push("-metadata", `title=${clean}`); + return args; +} + +class Inputs { + readonly args: string[] = []; + private count = 0; + add(...args: string[]): number { + if (args.some((arg) => arg.includes("\0"))) throw new Error("argument contains NUL"); + this.args.push(...args); + return this.count++; + } +} + +function captionPreset(edl: Edl, kit: BrandKit | null): CaptionStylePreset | null { + const presets = kit?.captionPresets ?? []; + return presets.find((p) => p.id === edl.captions?.presetId) ?? presets[0] ?? null; +} + +function pickFont(kit: BrandKit | null, fonts: Map, role: string): string | null { + const preferred = kit?.fonts.find((font) => font.role === role) ?? kit?.fonts[0]; + if (preferred) { + const path = fonts.get(preferred.artifactId) ?? fonts.get(preferred.family); + if (path) return path; + } + return fonts.values().next().value ?? null; +} + +/** + * Pure compiler from a validated EDL to ffmpeg argv steps plus caption files. + * No string ever reaches a shell; text is escaped for filtergraph and ASS, + * every label comes from SafeId values. + * + * Loudness: a single loudnorm pass with linear=true targets the profile (or + * plan) loudness and true peak. Without measured values loudnorm runs in its + * dynamic mode; a renderer may run a measurement pass first and append the + * measured_* options to that filter. + * + * Face, mark, and region redactions are applied before compilation: sources + * already point at redacted intermediates. + */ +export function compileRenderPlan(input: CompileInput): RenderPlan { + const edl = EdlSchema.parse(input.edl); + const { profile, brandKit: kit, workDir } = input; + const W = edl.output.width; + const H = edl.output.height; + const F = edl.output.fps; + const totalMs = edlDurationMs(edl); + const intro = introMs(edl); + const mainMs = mainProgramMs(edl); + const steps: RenderStep[] = []; + const outputs: RenderPlan["outputs"] = []; + const inputs = new Inputs(); + const graph: string[] = []; + let motionCount = 0; + + const motionInput = (key: string, template: MotionTemplateRef, width: number, height: number) => { + let path = input.motionClips.get(key); + if (!path) { + motionCount += 1; + path = joinPath(workDir, `motion-${motionCount}.mov`); + steps.push({ + kind: "motion", + id: `motion-${motionCount}`, + template, + width, + height, + fps: F, + outPath: path, + }); + } + return inputs.add("-i", path); + }; + + const finishVideo = (lengthMs: number) => [ + `fps=${num(F, 3)}`, + "format=yuv420p", + "tpad=stop_mode=clone:stop_duration=2", + `trim=duration=${sec(lengthMs)}`, + "setpts=PTS-STARTPTS", + ]; + const finishAudio = (lengthMs: number) => [ + AUDIO_FORMAT, + "apad", + `atrim=duration=${sec(lengthMs)}`, + "asetpts=PTS-STARTPTS", + ]; + const silence = (label: string, lengthMs: number) => + graph.push(`anullsrc=r=48000:cl=stereo,${finishAudio(lengthMs).join(",")}[${label}]`); + + // Per-clip chains. + interface Part { + v: string; + a: string; + lengthMs: number; + } + const clipParts = new Map(); + for (const clip of sortedVideoClips(edl)) { + const source = input.sources.get(clip.assetId); + if (!source) throw new Error(`no source for asset in clip ${clip.id}`); + const still = source.still === true || source.durationMs === 0; + const sourceLength = clip.outMs - clip.inMs; + const lengthMs = clipLengthMs(clip); + const index = still + ? inputs.add( + "-loop", + "1", + "-framerate", + num(F, 3), + "-t", + sec(sourceLength), + "-i", + source.path, + ) + : inputs.add("-ss", sec(clip.inMs), "-t", sec(sourceLength), "-i", source.path); + const rotated = source.rotation === 90 || source.rotation === 270; + const geometry: FrameGeometry = { + sourceWidth: rotated ? source.height : source.width, + sourceHeight: rotated ? source.width : source.height, + outWidth: W, + outHeight: H, + durationMs: sourceLength, + }; + const video: string[] = []; + if (clip.stabilize && !still) video.push("deshake"); + if (clip.reframe === "none") { + video.push(...fitFilters(W, H)); + } else { + const keyframes = + clip.reframe === "auto" + ? autoReframe( + (input.subjects.get(clip.assetId) ?? []) + .filter( + (s) => + s.tMs >= clip.inMs - AUTO_WINDOW_MS && s.tMs <= clip.outMs + AUTO_WINDOW_MS, + ) + .map((s) => ({ ...s, tMs: s.tMs - clip.inMs })), + geometry, + ).map((k) => ({ ...k, tMs: k.tMs + clip.inMs })) + : clip.reframe; + video.push(...cropFilters(keyframes, clip, geometry)); + } + video.push("setsar=1", `setpts=(PTS-STARTPTS)/${num(clip.speed, 6)}`); + const color = clip.color; + if (color) { + const brightness = clamp(color.exposure * 0.1, -1, 1); + if (brightness !== 0 || color.contrast !== 1 || color.saturation !== 1) { + video.push( + `eq=brightness=${num(brightness)}:contrast=${num(color.contrast)}:saturation=${num(color.saturation)}`, + ); + } + if (color.temperature !== 0) { + const shift = num(color.temperature * 0.1); + const inverse = num(-color.temperature * 0.1); + video.push(`colorbalance=rm=${shift}:bm=${inverse}`); + } + if (color.useBrandLut) { + if (!input.files.lutPath) throw new Error(`clip ${clip.id} needs the brand LUT file`); + video.push(`lut3d=file=${escapeFilterValue(input.files.lutPath)}`); + } + } + video.push(...finishVideo(lengthMs)); + graph.push(`[${index}:v]${video.join(",")}[v_${clip.id}]`); + + if (source.hasAudio && !clip.muted && !still) { + const audio = ["asetpts=PTS-STARTPTS", ...atempoChain(clip.speed), "aresample=48000"]; + graph.push(`[${index}:a]${[...audio, ...finishAudio(lengthMs)].join(",")}[a_${clip.id}]`); + } else { + silence(`a_${clip.id}`, lengthMs); + } + clipParts.set(clip.id, { v: `v_${clip.id}`, a: `a_${clip.id}`, lengthMs }); + } + + // Main program: concatenate runs, blend where transitions ask for it, fill gaps with black. + const blends = new Map( + edl.transitions + .filter((t) => BLENDED_TRANSITIONS.has(t.kind) && t.durationMs > 0) + .map((t) => [`${t.fromClipId}>${t.toClipId}`, t] as const), + ); + let mixCount = 0; + const nextLabel = (prefix: string) => `${prefix}${++mixCount}`; + let accumulated: Part | null = null; + let queue: Part[] = []; + const flush = () => { + const parts = accumulated ? [accumulated, ...queue] : queue; + queue = []; + if (parts.length <= 1) { + accumulated = parts[0] ?? accumulated; + return; + } + const v = nextLabel("cv"); + const a = nextLabel("ca"); + graph.push( + `${parts.map((p) => `[${p.v}][${p.a}]`).join("")}concat=n=${parts.length}:v=1:a=1[${v}][${a}]`, + ); + accumulated = { v, a, lengthMs: parts.reduce((sum, p) => sum + p.lengthMs, 0) }; + }; + let cursor = 0; + let previous: EdlVideoClip | null = null; + for (const clip of sortedVideoClips(edl)) { + const part = clipParts.get(clip.id)!; + const blend = previous ? blends.get(`${previous.id}>${clip.id}`) : undefined; + if (blend) { + flush(); + const base = accumulated as Part | null; + if (!base) throw new Error("transition without a preceding clip"); + const v = nextLabel("xv"); + const a = nextLabel("xa"); + const kind = XFADE_KIND[blend.kind as keyof typeof XFADE_KIND]; + graph.push( + `[${base.v}][${part.v}]xfade=transition=${kind}:duration=${sec(blend.durationMs)}:offset=${sec( + base.lengthMs - blend.durationMs, + )}[${v}]`, + `[${base.a}][${part.a}]acrossfade=d=${sec(blend.durationMs)}:c1=tri:c2=tri[${a}]`, + ); + accumulated = { v, a, lengthMs: base.lengthMs + part.lengthMs - blend.durationMs }; + } else { + const gap = clip.atMs - cursor; + if (gap > 1) { + const v = nextLabel("gv"); + const a = nextLabel("ga"); + graph.push(`color=c=black:s=${W}x${H}:r=${num(F, 3)},${finishVideo(gap).join(",")}[${v}]`); + silence(a, gap); + queue.push({ v, a, lengthMs: gap }); + } + queue.push(part); + } + cursor = Math.max(cursor, clip.atMs + part.lengthMs); + previous = clip; + } + flush(); + const main = accumulated as Part | null; + if (!main) throw new Error("plan has no video clips"); + + let mainAudio = main.a; + const voice = edl.redactions.voice; + if (voice) { + const ratio = 2 ** (voice.semitones / 12); + const label = nextLabel("vd"); + const filter = + voice.method === "pitch" + ? `rubberband=pitch=${num(ratio, 6)}:formant=preserved` + : `asetrate=${num(48000 * ratio, 2)},aresample=48000,atempo=${num(1 / ratio, 6)}`; + graph.push(`[${mainAudio}]${filter},${finishAudio(main.lengthMs).join(",")}[${label}]`); + mainAudio = label; + } + + // Intro and outro. + const program: Part[] = []; + const bookend = (key: "intro" | "outro", ref: MotionTemplateRef | undefined) => { + if (!ref || ref.durationMs <= 0) return; + const index = motionInput(key, ref, W, H); + const v = `${key}_v`; + const a = `${key}_a`; + graph.push( + `[${index}:v]${[...fitFilters(W, H), "setsar=1", ...finishVideo(ref.durationMs)].join(",")}[${v}]`, + ); + silence(a, ref.durationMs); + program.push({ v, a, lengthMs: ref.durationMs }); + }; + bookend("intro", edl.intro); + program.push({ v: main.v, a: mainAudio, lengthMs: main.lengthMs }); + bookend("outro", edl.outro); + let videoLabel = main.v; + let programAudio = mainAudio; + if (program.length > 1) { + videoLabel = "prog_v"; + programAudio = "prog_a"; + graph.push( + `${program.map((p) => `[${p.v}][${p.a}]`).join("")}concat=n=${program.length}:v=1:a=1[prog_v][prog_a]`, + ); + } + + // Overlays on the output timeline. + const zones = profile.safeZones; + const margin = Math.round(W * 0.02); + const safeLeft = Math.round((zones.leftPct / 100) * W) + margin; + const safeRight = Math.round((zones.rightPct / 100) * W) + margin; + const safeTop = Math.round((zones.topPct / 100) * H) + margin; + const safeBottom = Math.round((zones.bottomPct / 100) * H) + margin; + const textColor = brandColor(kit?.colors.text, "white"); + const overlays = [...edl.overlays].sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)); + const imageOverlay = ( + path: string, + overlay: EdlOverlay, + widthPx: number, + opacity: number, + position: { x: string; y: string }, + ) => { + const index = inputs.add("-i", path); + const label = nextLabel("img"); + const out = nextLabel("ov"); + const alpha = opacity < 1 ? `,colorchannelmixer=aa=${num(opacity)}` : ""; + graph.push( + `[${index}:v]format=rgba,scale=${even(widthPx)}:-2${alpha}[${label}]`, + `[${videoLabel}][${label}]overlay=x=${escapeFilterValue(position.x)}:y=${escapeFilterValue( + position.y, + )}:enable=${enableBetween(overlay.atMs, overlay.atMs + overlay.durationMs)}[${out}]`, + ); + videoLabel = out; + }; + const boxPosition = (box: NormalizedBox) => ({ + x: String(Math.round(box.x * W)), + y: String(Math.round(box.y * H)), + }); + const cornerPosition = (corner: string) => ({ + x: corner.endsWith("left") + ? String(safeLeft) + : corner === "center" + ? "(main_w-overlay_w)/2" + : `main_w-overlay_w-${safeRight}`, + y: corner.startsWith("top") + ? String(safeTop) + : corner === "center" + ? "(main_h-overlay_h)/2" + : `main_h-overlay_h-${safeBottom}`, + }); + + for (const overlay of overlays) { + const endMs = overlay.atMs + overlay.durationMs; + const motionPath = input.motionClips.get(overlay.id); + if (overlay.motion || motionPath) { + const box = overlay.box; + const width = box ? even(box.w * W) : W; + const height = box ? even(box.h * H) : H; + const template = overlay.motion ?? { + templateId: "overlay", + params: {}, + durationMs: overlay.durationMs, + }; + const index = motionInput(overlay.id, template, width, height); + const label = nextLabel("mo"); + const out = nextLabel("ov"); + const position = box ? boxPosition(box) : { x: "0", y: "0" }; + graph.push( + `[${index}:v]format=yuva420p,scale=${width}:${height},setpts=PTS-STARTPTS+${sec(overlay.atMs)}/TB[${label}]`, + `[${videoLabel}][${label}]overlay=x=${position.x}:y=${position.y}:eof_action=pass:enable=${enableBetween( + overlay.atMs, + endMs, + )}[${out}]`, + ); + videoLabel = out; + continue; + } + if (overlay.kind === "watermark") { + const watermark = kit?.watermark; + if (!watermark || !input.files.watermarkPath) { + throw new Error(`overlay ${overlay.id} needs the brand watermark file`); + } + const width = overlay.box ? overlay.box.w * W : watermark.scale * W; + imageOverlay( + input.files.watermarkPath, + overlay, + width, + watermark.opacity, + overlay.box ? boxPosition(overlay.box) : cornerPosition(watermark.position), + ); + continue; + } + if (overlay.kind === "logo") { + if (!input.files.logoPath) throw new Error(`overlay ${overlay.id} needs a logo file`); + const width = overlay.box ? overlay.box.w * W : 0.15 * W; + imageOverlay( + input.files.logoPath, + overlay, + width, + 1, + overlay.box ? boxPosition(overlay.box) : cornerPosition("top_right"), + ); + continue; + } + if (overlay.kind === "progress_bar") { + const barHeight = Math.max(4, even(H * 0.006)); + const label = nextLabel("pb"); + const out = nextLabel("ov"); + const y = overlay.box ? Math.round(overlay.box.y * H) : H - safeBottom - barHeight; + const x = `-overlay_w+main_w*min(1,(t-${sec(overlay.atMs)})/${sec(Math.max(1, overlay.durationMs))})`; + graph.push( + `color=c=${brandColor(kit?.colors.accent, "white")}:s=${W}x${barHeight}:r=${num(F, 3)}:d=${sec(totalMs)}[${label}]`, + `[${videoLabel}][${label}]overlay=x=${escapeFilterValue(x)}:y=${y}:enable=${enableBetween( + overlay.atMs, + endMs, + )}[${out}]`, + ); + videoLabel = out; + continue; + } + // text, lower_third, cta_end_card + const text = overlay.text?.trim(); + if (!text) throw new Error(`overlay ${overlay.id} has no text and no motion clip`); + const lines = text.split(/\r?\n/).length; + const fontSize = overlay.box + ? Math.max(8, Math.round((overlay.box.h * H * 0.8) / lines)) + : Math.round(H * (overlay.kind === "cta_end_card" ? 0.05 : 0.04)); + const position = overlay.box + ? boxPosition(overlay.box) + : { + x: "(w-text_w)/2", + y: + overlay.kind === "lower_third" + ? `h-text_h-${safeBottom + Math.round(H * 0.08)}` + : overlay.kind === "cta_end_card" + ? "(h-text_h)/2" + : String(safeTop + Math.round(H * 0.05)), + }; + const font = pickFont( + kit, + input.files.fonts, + overlay.kind === "cta_end_card" ? "heading" : "body", + ); + const options = [ + ...(font ? [`fontfile=${escapeFilterValue(font)}`] : []), + `text=${escapeFilterText(text)}`, + `fontsize=${fontSize}`, + `fontcolor=${textColor}`, + `x=${escapeFilterValue(position.x)}`, + `y=${escapeFilterValue(position.y)}`, + ]; + if (overlay.kind !== "text") { + options.push( + "box=1", + `boxcolor=${brandColor(kit?.colors.primary, "black")}@0.85`, + `boxborderw=${Math.round(fontSize * 0.4)}`, + ); + } + options.push(`enable=${enableBetween(overlay.atMs, endMs)}`); + const out = nextLabel("ov"); + graph.push(`[${videoLabel}]drawtext=${options.join(":")}[${out}]`); + videoLabel = out; + } + + // Captions. + const captions = edl.captions; + if (captions && captions.mode !== "none") { + const cues = buildCaptionCues(outputCaptionWords(edl, input.transcripts), { + maxCharsPerLine: captions.maxCharsPerLine, + maxLines: profile.captions.maxLines, + }); + if (cues.length > 0) { + if (captions.mode === "burned" || captions.mode === "both") { + const preset = captionPreset(edl, kit); + const assPath = joinPath(workDir, "captions.ass"); + steps.push({ + kind: "write_file", + id: "captions-ass", + path: assPath, + content: toAss(cues, { + width: W, + height: H, + profile, + preset, + fallbackTextColor: kit?.colors.text ?? "#FFFFFF", + position: captions.position, + maxCharsPerLine: captions.maxCharsPerLine, + wordHighlight: captions.wordHighlight, + language: captions.language, + }), + }); + const font = pickFont(kit, input.files.fonts, "caption"); + const fontsDir = font ? `:fontsdir=${escapeFilterValue(dirName(font))}` : ""; + const out = nextLabel("cap"); + graph.push(`[${videoLabel}]ass=filename=${escapeFilterValue(assPath)}${fontsDir}[${out}]`); + videoLabel = out; + } + if (captions.mode === "sidecar" || captions.mode === "both") { + const srt = joinPath(workDir, "captions.srt"); + const vtt = joinPath(workDir, "captions.vtt"); + steps.push( + { kind: "write_file", id: "captions-srt", path: srt, content: toSrt(cues) }, + { kind: "write_file", id: "captions-vtt", path: vtt, content: toVtt(cues) }, + ); + outputs.push({ kind: "subtitles_srt", path: srt }, { kind: "subtitles_vtt", path: vtt }); + } + } + } + const finalVideo = input.videoEncoder.endsWith("_vaapi") + ? "format=nv12,hwupload" + : "format=yuv420p"; + graph.push(`[${videoLabel}]${finalVideo}[vout]`); + + // Audio: program (+ voice-over) is speech; clips marked duckUnderSpeech are compressed under it. + const speech: string[] = [programAudio]; + const ducked: string[] = []; + const beds: string[] = []; + const audioClips = [...edl.tracks.audio].sort( + (a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id), + ); + for (const clip of audioClips) { + const path = input.files.audio.get(clip.artifactId); + if (!path) throw new Error(`no file for audio clip ${clip.id}`); + const length = clip.outMs - clip.inMs; + const index = inputs.add("-ss", sec(clip.inMs), "-t", sec(length), "-i", path); + graph.push(`[${index}:a]${audioClipFilters(clip, length).join(",")}[ac_${clip.id}]`); + const label = `ac_${clip.id}`; + if (clip.kind === "voiceover") speech.push(label); + else if (clip.duckUnderSpeech) ducked.push(label); + else beds.push(label); + } + const mix = (labels: string[], out: string) => { + if (labels.length === 1) { + graph.push(`[${labels[0]}]anull[${out}]`); + } else { + graph.push( + `${labels.map((l) => `[${l}]`).join("")}amix=inputs=${labels.length}:duration=first:normalize=0[${out}]`, + ); + } + }; + const finalInputs: string[] = []; + if (ducked.length > 0) { + mix(speech, "speech"); + graph.push("[speech]asplit=2[speech_mix][speech_key]"); + mix(ducked, "duck_bus"); + graph.push( + "[duck_bus][speech_key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400:makeup=1[ducked]", + ); + finalInputs.push("speech_mix", "ducked"); + } else { + finalInputs.push(...speech); + } + finalInputs.push(...beds); + const targetLufs = edl.targetLufs ?? profile.audio.targetLufs; + const loudnorm = `loudnorm=I=${num(targetLufs, 1)}:TP=${num(profile.audio.truePeakDb, 1)}:LRA=11:linear=true`; + const mixed = finalInputs.length > 1 ? "mixed" : finalInputs[0]!; + if (finalInputs.length > 1) mix(finalInputs, "mixed"); + graph.push(`[${mixed}]${loudnorm},aresample=48000,${AUDIO_FORMAT}[aout]`); + + // Render step. + const graphText = graph.join(";"); + const graphArgs = + graphText.length > MAX_INLINE_GRAPH + ? ["-filter_complex_script", joinPath(workDir, "filtergraph.txt")] + : ["-filter_complex", graphText]; + if (graphText.length > MAX_INLINE_GRAPH) { + steps.push({ + kind: "write_file", + id: "filtergraph", + path: joinPath(workDir, "filtergraph.txt"), + content: graphText, + }); + } + const videoPath = joinPath(workDir, "output.mp4"); + steps.push({ + kind: "ffmpeg", + id: "render", + argv: [ + "-hide_banner", + "-nostdin", + "-y", + ...inputs.args, + ...graphArgs, + "-map", + "[vout]", + "-map", + "[aout]", + ...encodingArgs(input), + ...metadataArgs(edl.metadata.title), + "-t", + sec(totalMs), + "-movflags", + "+faststart", + videoPath, + ], + produces: [videoPath], + durationMs: totalMs, + }); + outputs.unshift({ kind: "video", path: videoPath }); + + if (profile.export.poster) { + const posterPath = joinPath(workDir, "poster.jpg"); + const at = clamp(input.posterAtMs ?? intro + mainMs / 3, 0, Math.max(0, totalMs - 1)); + steps.push({ + kind: "ffmpeg", + id: "poster", + argv: [ + "-hide_banner", + "-nostdin", + "-y", + "-ss", + sec(at), + "-i", + videoPath, + "-frames:v", + "1", + "-map_metadata", + "-1", + "-fflags", + "+bitexact", + "-flags:v", + "+bitexact", + "-q:v", + "2", + posterPath, + ], + produces: [posterPath], + durationMs: 0, + }); + outputs.push({ kind: "poster", path: posterPath }); + } + + return { quality: input.quality, workDir, steps, outputs }; +} + +function audioClipFilters(clip: EdlAudioClip, lengthMs: number): string[] { + const filters = ["asetpts=PTS-STARTPTS", "aresample=48000", AUDIO_FORMAT]; + if (clip.gainDb !== 0) filters.push(`volume=${num(clip.gainDb, 2)}dB`); + if (clip.fadeInMs > 0) filters.push(`afade=t=in:st=0:d=${sec(clip.fadeInMs)}`); + if (clip.fadeOutMs > 0) { + filters.push( + `afade=t=out:st=${sec(Math.max(0, lengthMs - clip.fadeOutMs))}:d=${sec(clip.fadeOutMs)}`, + ); + } + if (clip.atMs > 0) filters.push(`adelay=${clip.atMs}|${clip.atMs}`); + return filters; } diff --git a/packages/media/src/edl.test.ts b/packages/media/src/edl.test.ts new file mode 100644 index 0000000..f72e9f9 --- /dev/null +++ b/packages/media/src/edl.test.ts @@ -0,0 +1,337 @@ +import type { EdlError } from "@rakazo/contracts"; +import { EdlErrorSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import type { EdlContext } from "./edl.js"; +import { edlDurationMs, locateOutputTime, locateOutputTimeAll, validateEdl } from "./edl.js"; +import { testContext, testEdl, testProfile } from "./test-fixtures.js"; + +/** Raw plan (pre-defaults) so cases can break it in any way. */ +function raw(patch: (plan: Record) => void): unknown { + const plan = JSON.parse(JSON.stringify(testEdl())) as Record; + patch(plan); + return plan; +} + +function codes(input: unknown, context: EdlContext = testContext()): EdlError["code"][] { + const result = validateEdl(input, context); + return result.ok ? [] : result.errors.map((e) => e.code); +} + +describe("validateEdl", () => { + it("accepts a valid plan and returns the parsed EDL", () => { + const result = validateEdl(testEdl(), testContext()); + expect(result.ok).toBe(true); + if (result.ok) expect(result.edl.tracks.video).toHaveLength(2); + }); + + const cases: Array<{ + code: EdlError["code"]; + input: unknown; + context?: Partial; + path?: string; + }> = [ + { code: "schema", input: raw((p) => (p.edlVersion = 2)), path: "edlVersion" }, + { + code: "schema", + input: raw((p) => (p.tracks.video[1].id = "c1")), + path: "tracks.video[1].id", + }, + { + code: "schema", + input: raw((p) => (p.tracks.video[0].id = "Bad Id;[x]")), + path: "tracks.video[0].id", + }, + { + code: "unknown_asset", + input: raw((p) => (p.tracks.video[0].assetId = "missing")), + path: "tracks.video[0].assetId", + }, + { + code: "unknown_clip", + input: raw((p) => { + p.transitions = [{ fromClipId: "c1", toClipId: "c9", kind: "cut", durationMs: 0 }]; + }), + path: "transitions[0].toClipId", + }, + { + code: "unknown_clip", + input: raw((p) => { + p.redactions.regions = [ + { + id: "r1", + clipId: "zz", + keyframes: [{ tMs: 0, box: { x: 0, y: 0, w: 0.1, h: 0.1 } }], + treatment: "blur", + }, + ]; + }), + }, + { + code: "unknown_artifact", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "sfx", artifactId: "nope", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "unknown_artifact", + input: raw((p) => (p.overlays = [{ id: "w", kind: "watermark", atMs: 0, durationMs: 1000 }])), + context: { brandKit: null }, + }, + { + code: "unknown_mark", + input: raw((p) => (p.redactions.marks = [{ markId: "mark-x", treatment: "blur" }])), + }, + { + code: "unknown_person", + input: raw((p) => { + p.redactions.faces = [{ id: "f", target: { personId: "person-x" }, mode: "blur" }]; + }), + }, + { + code: "unknown_track", + input: raw((p) => { + p.redactions.faces = [{ id: "f", target: { trackId: "track-x" }, mode: "pixelate" }]; + }), + }, + { + code: "unknown_template", + input: raw((p) => (p.intro = { templateId: "intro-none", durationMs: 1000 })), + path: "intro.templateId", + }, + { + code: "unknown_template", + input: raw((p) => (p.captions = { mode: "burned", presetId: "missing" })), + path: "captions.presetId", + }, + { + code: "range_out_of_asset", + input: raw((p) => (p.tracks.video[1].outMs = 12_000)), + path: "tracks.video[1].outMs", + }, + { + code: "range_out_of_asset", + input: raw((p) => (p.tracks.video[0].reframe = [{ tMs: 9000, cx: 0.5, cy: 0.5, scale: 1 }])), + }, + { + code: "range_empty", + input: raw((p) => (p.tracks.video[0].outMs = 0)), + }, + { + code: "range_empty", + input: raw( + (p) => (p.overlays = [{ id: "t", kind: "text", atMs: 0, durationMs: 0, text: "x" }]), + ), + }, + { + code: "overlap", + input: raw((p) => (p.tracks.video[1].atMs = 1500)), + path: "tracks.video[1].atMs", + }, + { + code: "overlap", + // A crossfade needs exactly its duration of overlap. + input: raw((p) => { + p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }]; + }), + }, + { + code: "overlap", + input: raw((p) => { + p.tracks.audio = [ + { id: "m1", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 0 }, + { id: "m2", kind: "music", artifactId: "music-1", inMs: 0, outMs: 3000, atMs: 2000 }, + ]; + }), + }, + { + code: "duration_below_min", + input: raw((p) => (p.output.durationHardMinMs = 5000)), + }, + { + code: "duration_below_min", + input: testEdl(), + context: { profile: testProfile({ duration: { hardMinMs: 6000, hardMaxMs: 60_000 } }) }, + }, + { + code: "duration_above_max", + input: raw((p) => (p.output.durationHardMaxMs = 3000)), + }, + { + code: "duration_above_max", + input: testEdl(), + context: { profile: testProfile({ duration: { hardMinMs: 0, hardMaxMs: 3000 } }) }, + }, + { + code: "overlay_outside_safe_zone", + input: raw((p) => { + p.overlays = [ + { + id: "t", + kind: "text", + atMs: 0, + durationMs: 1000, + text: "Hi", + box: { x: 0.1, y: 0.85, w: 0.5, h: 0.1 }, + }, + ]; + }), + path: "overlays[0].box", + }, + { + code: "paid_only_segment", + input: testEdl(), + context: { paidOnly: [{ assetId: "asset-a", startMs: 6500, endMs: 9000 }] }, + path: "tracks.video[1]", + }, + { + code: "paid_only_segment", + input: testEdl({ output: { ...testEdl().output, profileId: "example-paid-video" } }), + context: { + profile: testProfile({ id: "example-paid-video", kind: "paid_video" }), + paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 100 }], + goal: "teaser", + }, + }, + { + code: "profile_mismatch", + input: raw((p) => (p.output.profileId = "other-profile")), + }, + { + code: "profile_mismatch", + input: raw((p) => { + p.output.width = 1920; + p.output.height = 1080; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "vo-1", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + }, + { + code: "music_license", + input: raw((p) => { + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-1", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }), + context: { profile: testProfile({ audio: { music: "platform_library_only" } }) }, + }, + ]; + + it.each(cases)("rejects $code ($path)", ({ code, input, context, path }) => { + const result = validateEdl(input, testContext(context)); + expect(result.ok).toBe(false); + if (result.ok) return; + for (const error of result.errors) expect(EdlErrorSchema.safeParse(error).success).toBe(true); + const match = result.errors.find((e) => e.code === code); + expect(match, JSON.stringify(result.errors)).toBeDefined(); + if (path) expect(match!.path).toBe(path); + }); + + it("covers every error code", () => { + const covered = new Set(cases.map((c) => c.code)); + expect([...covered].sort()).toEqual([...EdlErrorSchema.shape.code.options].sort()); + }); + + it("allows paid-only material in a full release for a paid profile and any music where allowed", () => { + const paid = testProfile({ + id: "example-paid-video", + kind: "paid_video", + audio: { music: "any" }, + }); + const plan = raw((p) => { + p.output.profileId = "example-paid-video"; + p.tracks.audio = [ + { id: "m", kind: "music", artifactId: "music-2", inMs: 0, outMs: 1000, atMs: 0 }, + ]; + }); + expect( + codes( + plan, + testContext({ profile: paid, paidOnly: [{ assetId: "asset-a", startMs: 0, endMs: 9000 }] }), + ), + ).toEqual([]); + }); + + it("accepts photos without a duration and a crossfade with the right overlap", () => { + const plan = raw((p) => { + p.tracks.video[0].assetId = "photo-1"; + p.tracks.video[1].atMs = 1500; + p.transitions = [{ fromClipId: "c1", toClipId: "c2", kind: "crossfade", durationMs: 500 }]; + }); + expect(codes(plan)).toEqual([]); + }); + + it("reports every failure at once", () => { + const plan = raw((p) => { + p.tracks.video[0].assetId = "missing"; + p.tracks.video[1].outMs = 4000; + p.redactions.marks = [{ markId: "mark-x", treatment: "cover" }]; + }); + expect(codes(plan)).toEqual(["unknown_asset", "range_empty", "unknown_mark"]); + }); +}); + +describe("timeline", () => { + const edl = testEdl({ + intro: { templateId: "intro-minimal", durationMs: 1000 }, + outro: { templateId: "outro-simple", durationMs: 500 }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "none" }, + { + id: "c2", + assetId: "asset-b", + inMs: 1000, + outMs: 5000, + atMs: 1500, + speed: 2, + reframe: "none", + }, + ], + }, + transitions: [{ fromClipId: "c1", toClipId: "c2", kind: "dip", durationMs: 500 }], + }); + + it("measures intro, speed, transition overlap and outro", () => { + expect(edlDurationMs(edl)).toBe(1000 + 1500 + 2000 + 500); + expect(edlDurationMs(testEdl())).toBe(4000); + }); + + it("maps output time back to clip and source time", () => { + expect(locateOutputTime(edl, 500)).toBeNull(); + expect(locateOutputTime(edl, 1000)).toEqual({ clipId: "c1", assetId: "asset-a", sourceMs: 0 }); + expect(locateOutputTime(edl, 2200)).toEqual({ + clipId: "c1", + assetId: "asset-a", + sourceMs: 1200, + }); + // Inside the transition both clips are visible; the incoming one wins. + expect(locateOutputTimeAll(edl, 2700).map((h) => h.clipId)).toEqual(["c1", "c2"]); + expect(locateOutputTime(edl, 2700)).toEqual({ + clipId: "c2", + assetId: "asset-b", + sourceMs: 1400, + }); + expect(locateOutputTime(edl, 4400)).toEqual({ + clipId: "c2", + assetId: "asset-b", + sourceMs: 4800, + }); + expect(locateOutputTime(edl, 4600)).toBeNull(); + }); +}); diff --git a/packages/media/src/edl.ts b/packages/media/src/edl.ts index 4c3a06e..384f3d3 100644 --- a/packages/media/src/edl.ts +++ b/packages/media/src/edl.ts @@ -1,5 +1,30 @@ -import type { BrandKit, Edl, EdlError, MediaAssetKind, PlatformProfile } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import type { + AspectRatio, + BrandKit, + Edl, + EdlError, + EdlTransition, + EdlVideoClip, + MediaAssetKind, + MediaProjectGoal, + NormalizedBox, + PlatformProfile, +} from "@rakazo/contracts"; +import { EdlSchema } from "@rakazo/contracts"; + +/* + * Timeline model shared by validation, compilation, gate and auto-fix: + * + * - The output is intro + main program + outro, played back to back. + * - `tracks.video[].atMs` is a position on the main program, which starts + * after the intro. A clip lasts (outMs - inMs) / speed. + * - Clips may only overlap where a crossfade, dip or whip transition joins + * them, and then by exactly the transition duration. Cuts and match cuts + * never overlap. + * - Overlays and audio clips are positioned on the output timeline (intro + * included), so music can run under the intro. + * - Reframe keyframes are in source time of the clip's asset. + */ export interface EdlContext { profile: PlatformProfile; @@ -18,28 +43,384 @@ export interface EdlContext { templateIds: Set; /** Segments the user reserved for the paid release. */ paidOnly: Array<{ assetId: string; startMs: number; endMs: number }>; + /** Project goal; a teaser never uses paid-only segments, whatever the profile. */ + goal?: MediaProjectGoal; } export type EdlValidation = { ok: true; edl: Edl } | { ok: false; errors: EdlError[] }; +/** Transitions rendered as a timed blend (xfade). Cuts and match cuts are plain concatenation. */ +export const BLENDED_TRANSITIONS: ReadonlySet = new Set([ + "crossfade", + "dip", + "whip", +]); + +export const ASPECT_RATIO_VALUES: Record = { + "9:16": 9 / 16, + "4:5": 4 / 5, + "1:1": 1, + "16:9": 16 / 9, + "4:3": 4 / 3, + "21:9": 21 / 9, +}; + +const ASPECT_TOLERANCE = 0.01; + +/** Whether width × height matches one of the allowed aspect ratios within 1 %. */ +export function matchesAspect(width: number, height: number, allowed: AspectRatio[]): boolean { + const actual = width / height; + return allowed.some( + (ratio) => + Math.abs(actual - ASPECT_RATIO_VALUES[ratio]) / ASPECT_RATIO_VALUES[ratio] <= + ASPECT_TOLERANCE, + ); +} + +/** Output length of one video clip after speed change. */ +export function clipLengthMs(clip: Pick): number { + return Math.max(0, Math.round((clip.outMs - clip.inMs) / clip.speed)); +} + +/** End of the main program (excluding intro and outro). */ +export function mainProgramMs(edl: Edl): number { + let end = 0; + for (const clip of edl.tracks.video) end = Math.max(end, clip.atMs + clipLengthMs(clip)); + return end; +} + +export function introMs(edl: Edl): number { + return edl.intro?.durationMs ?? 0; +} + +/** Output length in ms including intro, outro, speed, and transition overlap. */ +export function edlDurationMs(edl: Edl): number { + return introMs(edl) + mainProgramMs(edl) + (edl.outro?.durationMs ?? 0); +} + +/** Every clip visible at an output timestamp (two during a blended transition), earliest first. */ +export function locateOutputTimeAll( + edl: Edl, + outputMs: number, +): Array<{ clipId: string; assetId: string; sourceMs: number }> { + const t = outputMs - introMs(edl); + if (t < 0) return []; + return [...edl.tracks.video] + .filter((clip) => t >= clip.atMs && t < clip.atMs + clipLengthMs(clip)) + .sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)) + .map((clip) => ({ + clipId: clip.id, + assetId: clip.assetId, + sourceMs: Math.min(clip.outMs, Math.round(clip.inMs + (t - clip.atMs) * clip.speed)), + })); +} + +/** + * Map an output timestamp back to the clip and source time it shows. During a + * blended transition the incoming clip wins; intro and outro map to null. + */ +export function locateOutputTime( + edl: Edl, + outputMs: number, +): { clipId: string; assetId: string; sourceMs: number } | null { + return locateOutputTimeAll(edl, outputMs).at(-1) ?? null; +} + +/** Whether a normalized box lies inside the profile's safe zones. */ +export function boxInsideSafeZone(box: NormalizedBox, profile: PlatformProfile): boolean { + const epsilon = 1e-6; + const zones = profile.safeZones; + return ( + box.x >= zones.leftPct / 100 - epsilon && + box.y >= zones.topPct / 100 - epsilon && + box.x + box.w <= 1 - zones.rightPct / 100 + epsilon && + box.y + box.h <= 1 - zones.bottomPct / 100 + epsilon + ); +} + +function formatPath(path: ReadonlyArray): string { + let out = ""; + for (const part of path) { + if (typeof part === "number") out += `[${part}]`; + else out += out ? `.${String(part)}` : String(part); + } + return out || "(root)"; +} + +/** Video clips in timeline order, ties broken by id for determinism. */ +export function sortedVideoClips(edl: Edl): EdlVideoClip[] { + return [...edl.tracks.video].sort((a, b) => a.atMs - b.atMs || a.id.localeCompare(b.id)); +} + /** * Parse with EdlSchema, then check references, ranges, overlaps, durations, * safe zones, paid-only use (teaser-style profiles), and music licensing. * Every failure is reported; nothing is silently corrected. */ -export function validateEdl(_input: unknown, _context: EdlContext): EdlValidation { - return notImplemented("validateEdl"); -} +export function validateEdl(input: unknown, context: EdlContext): EdlValidation { + const parsed = EdlSchema.safeParse(input); + if (!parsed.success) { + return { + ok: false, + errors: parsed.error.issues.map((issue) => ({ + code: "schema", + path: formatPath(issue.path), + message: issue.message, + })), + }; + } + const edl = parsed.data; + const errors: EdlError[] = []; + const fail = (code: EdlError["code"], path: string, message: string) => + errors.push({ code, path, message }); + const { profile } = context; -/** Output length in ms including intro, outro, speed, and transition overlap. */ -export function edlDurationMs(_edl: Edl): number { - return notImplemented("edlDurationMs"); -} + if (edl.output.profileId !== profile.id) { + fail("profile_mismatch", "output.profileId", `plan targets ${edl.output.profileId}`); + } + if (!matchesAspect(edl.output.width, edl.output.height, profile.video.aspectRatios)) { + fail( + "profile_mismatch", + "output", + `${edl.output.width}x${edl.output.height} is not one of ${profile.video.aspectRatios.join(", ")}`, + ); + } + if (edl.output.fps > profile.video.fps * 1.001) { + fail("profile_mismatch", "output.fps", `fps above profile maximum ${profile.video.fps}`); + } -/** Map an output timestamp back to the clip and source time it shows. */ -export function locateOutputTime( - _edl: Edl, - _outputMs: number, -): { clipId: string; assetId: string; sourceMs: number } | null { - return notImplemented("locateOutputTime"); + const seenIds = new Map(); + const claimId = (id: string, path: string) => { + const previous = seenIds.get(id); + if (previous) fail("schema", path, `id ${id} already used at ${previous}`); + else seenIds.set(id, path); + }; + + const clipsById = new Map(); + edl.tracks.video.forEach((clip, index) => { + const path = `tracks.video[${index}]`; + claimId(clip.id, `${path}.id`); + clipsById.set(clip.id, clip); + if (clip.outMs <= clip.inMs) fail("range_empty", path, "outMs must be after inMs"); + const asset = context.assets.get(clip.assetId); + if (!asset) { + fail("unknown_asset", `${path}.assetId`, `asset ${clip.assetId} is not available`); + } else if (asset.durationMs !== null && asset.kind !== "photo") { + if (clip.outMs > asset.durationMs) { + fail( + "range_out_of_asset", + `${path}.outMs`, + `outMs ${clip.outMs} exceeds asset duration ${asset.durationMs}`, + ); + } + if (clip.inMs >= asset.durationMs) { + fail("range_out_of_asset", `${path}.inMs`, `inMs ${clip.inMs} is past the asset end`); + } + } + if (Array.isArray(clip.reframe)) { + clip.reframe.forEach((keyframe, k) => { + if (keyframe.tMs < clip.inMs || keyframe.tMs > clip.outMs) { + fail( + "range_out_of_asset", + `${path}.reframe[${k}].tMs`, + "reframe keyframe lies outside the clip's source range", + ); + } + }); + } + if ( + (profile.kind === "free_social" || context.goal === "teaser") && + context.paidOnly.some( + (segment) => + segment.assetId === clip.assetId && + segment.startMs < clip.outMs && + clip.inMs < segment.endMs, + ) + ) { + fail("paid_only_segment", path, "clip uses material reserved for the paid release"); + } + }); + + // Transitions: known clips, adjacent in time, at most one in and one out per clip. + const blendBetween = new Map(); + const outgoing = new Set(); + const incoming = new Set(); + const order = sortedVideoClips(edl); + const nextOf = new Map(); + for (let i = 0; i + 1 < order.length; i++) nextOf.set(order[i]!.id, order[i + 1]!.id); + edl.transitions.forEach((transition, index) => { + const path = `transitions[${index}]`; + const from = clipsById.get(transition.fromClipId); + const to = clipsById.get(transition.toClipId); + if (!from) fail("unknown_clip", `${path}.fromClipId`, `no clip ${transition.fromClipId}`); + if (!to) fail("unknown_clip", `${path}.toClipId`, `no clip ${transition.toClipId}`); + if (!from || !to) return; + if (nextOf.get(from.id) !== to.id) { + fail("schema", path, "a transition must join a clip to the clip that follows it"); + return; + } + if (outgoing.has(from.id) || incoming.has(to.id)) { + fail("schema", path, "a clip can have only one transition on each side"); + return; + } + outgoing.add(from.id); + incoming.add(to.id); + if (BLENDED_TRANSITIONS.has(transition.kind) && transition.durationMs > 0) { + const shortest = Math.min(clipLengthMs(from), clipLengthMs(to)); + if (transition.durationMs >= shortest) { + fail("overlap", `${path}.durationMs`, "transition is longer than one of its clips"); + return; + } + blendBetween.set(`${from.id}>${to.id}`, transition.durationMs); + } + }); + + // Overlaps on the video track: only blended transitions may overlap, by exactly their length. + for (let i = 0; i + 1 < order.length; i++) { + const a = order[i]!; + const b = order[i + 1]!; + const overlap = a.atMs + clipLengthMs(a) - b.atMs; + const blend = blendBetween.get(`${a.id}>${b.id}`); + const path = `tracks.video[${edl.tracks.video.indexOf(b)}].atMs`; + if (blend !== undefined) { + if (Math.abs(overlap - blend) > 1) { + fail( + "overlap", + path, + `clips ${a.id} and ${b.id} must overlap by the ${blend} ms transition`, + ); + } + } else if (overlap > 1) { + fail("overlap", path, `clip ${b.id} starts ${overlap} ms before ${a.id} ends`); + } + } + for (let i = 0; i + 2 < order.length; i++) { + const a = order[i]!; + const c = order[i + 2]!; + if (a.atMs + clipLengthMs(a) > c.atMs + 1) { + fail("overlap", `tracks.video[${edl.tracks.video.indexOf(c)}].atMs`, "three clips overlap"); + } + } + + // Audio clips. + const audioByKind = new Map>(); + edl.tracks.audio.forEach((clip, index) => { + const path = `tracks.audio[${index}]`; + claimId(clip.id, `${path}.id`); + if (clip.outMs <= clip.inMs) fail("range_empty", path, "outMs must be after inMs"); + if (!context.artifactIds.has(clip.artifactId)) { + fail( + "unknown_artifact", + `${path}.artifactId`, + `artifact ${clip.artifactId} is not available`, + ); + } + if (clip.fadeInMs + clip.fadeOutMs > clip.outMs - clip.inMs && clip.outMs > clip.inMs) { + fail("range_empty", path, "fades are longer than the clip"); + } + if (clip.kind === "music") { + if (profile.audio.music === "platform_library_only") { + fail( + "music_license", + `${path}.artifactId`, + "this platform only allows music added from its own library", + ); + } else if (profile.audio.music === "licensed_only") { + const license = context.music.get(clip.artifactId); + if (!license?.commercialUse) { + fail( + "music_license", + `${path}.artifactId`, + license ? "music is not licensed for commercial use" : "music license is unknown", + ); + } + } + } + if (clip.kind !== "sfx") { + const list = audioByKind.get(clip.kind) ?? []; + list.push({ start: clip.atMs, end: clip.atMs + (clip.outMs - clip.inMs), index }); + audioByKind.set(clip.kind, list); + } + }); + for (const list of audioByKind.values()) { + list.sort((a, b) => a.start - b.start || a.index - b.index); + for (let i = 0; i + 1 < list.length; i++) { + if (list[i]!.end > list[i + 1]!.start) { + fail("overlap", `tracks.audio[${list[i + 1]!.index}].atMs`, "audio clips overlap"); + } + } + } + + // Templates. + const checkTemplate = (templateId: string, path: string) => { + if (!context.templateIds.has(templateId)) { + fail("unknown_template", path, `template ${templateId} is not available`); + } + }; + if (edl.intro) checkTemplate(edl.intro.templateId, "intro.templateId"); + if (edl.outro) checkTemplate(edl.outro.templateId, "outro.templateId"); + + // Overlays. + edl.overlays.forEach((overlay, index) => { + const path = `overlays[${index}]`; + claimId(overlay.id, `${path}.id`); + if (overlay.durationMs <= 0) + fail("range_empty", `${path}.durationMs`, "overlay has no duration"); + if (overlay.motion) checkTemplate(overlay.motion.templateId, `${path}.motion.templateId`); + if (overlay.box && !boxInsideSafeZone(overlay.box, profile)) { + fail("overlay_outside_safe_zone", `${path}.box`, "overlay leaves the profile safe zone"); + } + if (overlay.kind === "watermark" && !context.brandKit?.watermark) { + fail("unknown_artifact", path, "the brand kit has no watermark"); + } + if ((overlay.kind === "text" || overlay.kind === "lower_third") && !overlay.text?.trim()) { + if (!overlay.motion) fail("range_empty", `${path}.text`, "text overlay has no text"); + } + }); + + // Captions preset. + const presetId = edl.captions?.presetId; + if (presetId && context.brandKit) { + if (!context.brandKit.captionPresets.some((preset) => preset.id === presetId)) { + fail("unknown_template", "captions.presetId", `caption preset ${presetId} is not in the kit`); + } + } + + // Redactions. + edl.redactions.faces.forEach((redaction, index) => { + const path = `redactions.faces[${index}]`; + claimId(redaction.id, `${path}.id`); + const target = redaction.target; + if ("trackId" in target && !context.faceTrackIds.has(target.trackId)) { + fail("unknown_track", `${path}.target.trackId`, `face track ${target.trackId} is unknown`); + } + if ("personId" in target && !context.personIds.has(target.personId)) { + fail("unknown_person", `${path}.target.personId`, `person ${target.personId} is unknown`); + } + }); + edl.redactions.marks.forEach((mark, index) => { + if (!context.markIds.has(mark.markId)) { + fail("unknown_mark", `redactions.marks[${index}].markId`, `mark ${mark.markId} is unknown`); + } + }); + edl.redactions.regions.forEach((region, index) => { + const path = `redactions.regions[${index}]`; + claimId(region.id, `${path}.id`); + if (!clipsById.has(region.clipId)) { + fail("unknown_clip", `${path}.clipId`, `no clip ${region.clipId}`); + } + }); + + // Duration against both the plan's and the profile's hard bounds. + const duration = edlDurationMs(edl); + const minMs = Math.max(edl.output.durationHardMinMs, profile.duration.hardMinMs); + const maxMs = Math.min(edl.output.durationHardMaxMs, profile.duration.hardMaxMs); + if (duration < minMs) { + fail("duration_below_min", "output", `duration ${duration} ms is below ${minMs} ms`); + } + if (duration > maxMs) { + fail("duration_above_max", "output", `duration ${duration} ms is above ${maxMs} ms`); + } + + return errors.length > 0 ? { ok: false, errors } : { ok: true, edl }; } diff --git a/packages/media/src/gate.test.ts b/packages/media/src/gate.test.ts new file mode 100644 index 0000000..acb75e3 --- /dev/null +++ b/packages/media/src/gate.test.ts @@ -0,0 +1,531 @@ +import type { FfprobeReport, GateReport } from "@rakazo/contracts"; +import { GateReportSchema, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { edlDurationMs } from "./edl.js"; +import type { GateInput } from "./gate.js"; +import { + checkCopy, + checkTechnical, + evaluateGate, + findMetadataLeaks, + METADATA_ALLOWED_TAGS, + planAutoFix, +} from "./gate.js"; +import { testEdl, testPaidProfile, testProfile } from "./test-fixtures.js"; + +const cleanTags = { + "format.major_brand": "isom", + "format.minor_version": "512", + "format.compatible_brands": "isomiso2avc1mp41", + "format.title": "Example", + "stream.0.language": "und", + "stream.0.handler_name": "VideoHandler", + "stream.0.vendor_id": "[0][0][0][0]", + "stream.0.encoder": "Lavc libx264", + "stream.1.handler_name": "SoundHandler", +}; + +function probe(overrides: Partial = {}): FfprobeReport { + return { + durationMs: 4000, + width: 1080, + height: 1920, + fps: 30, + videoCodec: "h264", + audioCodec: "aac", + sizeBytes: 2_000_000, + integratedLufs: -14.3, + truePeakDb: -1.2, + tags: cleanTags, + ...overrides, + }; +} + +function gateInput(overrides: Partial = {}): GateInput { + return { + profile: testProfile(), + edl: testEdl(), + ffprobe: probe(), + content: [ + { tMs: 0, scores: { explicit_nudity: 0.01 } }, + { tMs: 500, scores: { explicit_nudity: 0.02, violence: 0.1 } }, + ], + faces: [], + trackingGaps: [], + marks: [], + people: [], + copy: { title: "A day out", caption: "Short caption #one", tags: ["two"] }, + aiAlterations: [], + bundleHasDisclosure: false, + watermarkBox: { x: 0.7, y: 0.12, w: 0.15, h: 0.05 }, + sampledFrames: 8, + autoFixRound: 0, + now: new Date("2026-10-01T12:00:00Z"), + ...overrides, + }; +} + +const blockCodes = (report: GateReport) => + report.issues.filter((i) => i.severity === "block").map((i) => i.code); + +describe("metadata", () => { + it("allows only harmless container tags", () => { + expect(METADATA_ALLOWED_TAGS).toContain("title"); + expect(findMetadataLeaks(cleanTags)).toEqual([]); + expect( + findMetadataLeaks({ + ...cleanTags, + "format.creation_time": "2026-01-01T00:00:00Z", + "format.location": "+00.0000+000.0000/", + "format.com.apple.quicktime.make": "Example", + "format.com.apple.quicktime.location.ISO6709": "+00+000/", + "format.make": "Example", + "format.model": "Example", + "format.software": "Example 1.0", + "stream.0.creation_time": "2026-01-01T00:00:00Z", + GPSLatitude: "0", + "format.encoder": "Lavf60.16.100", + "stream:0:encoder": "x264 core 164", + }), + ).toEqual([ + "GPSLatitude", + "format.com.apple.quicktime.location.ISO6709", + "format.com.apple.quicktime.make", + "format.creation_time", + "format.encoder", + "format.location", + "format.make", + "format.model", + "format.software", + "stream.0.creation_time", + "stream:0:encoder", + ]); + expect(findMetadataLeaks({ title: "x", major_brand: "isom" })).toEqual([]); + }); +}); + +describe("evaluateGate", () => { + it("passes a clean render and produces a schema-valid report", () => { + const report = evaluateGate(gateInput()); + expect(report.issues).toEqual([]); + expect(report.passed).toBe(true); + expect(GateReportSchema.parse(report)).toEqual(report); + expect(report.checkedAt).toBe("2026-10-01T12:00:00.000Z"); + }); + + it("blocks technical problems", () => { + const edl = testEdl(); + expect(checkTechnical(testProfile(), edl, probe())).toEqual([]); + const cases: Array<[Partial, string]> = [ + [{ width: 1920, height: 1080 }, "resolution_mismatch"], + [{ width: 1920, height: 1080 }, "aspect_not_allowed"], + [{ fps: 25 }, "fps_mismatch"], + [{ durationMs: 70_000 }, "duration_too_long"], + [{ durationMs: 500 }, "duration_too_short"], + [{ sizeBytes: 600_000_000 }, "file_too_large"], + [{ videoCodec: "vp9" }, "codec_mismatch"], + [{ integratedLufs: -20 }, "loudness_off_target"], + [{ integratedLufs: null }, "loudness_unmeasured"], + [{ truePeakDb: 0 }, "true_peak_exceeded"], + [{ tags: { ...cleanTags, "format.creation_time": "2020" } }, "metadata_leak"], + ]; + for (const [patch, code] of cases) { + const report = evaluateGate(gateInput({ ffprobe: probe(patch) })); + expect(report.passed, code).toBe(false); + expect(blockCodes(report), code).toContain(code); + } + const drift = checkTechnical(testProfile(), edl, probe({ durationMs: 4500 })); + expect(drift).toEqual([expect.objectContaining({ code: "duration_drift", severity: "warn" })]); + expect( + checkTechnical(testProfile(), edl, probe({ audioCodec: null, integratedLufs: null })), + ).toEqual([]); + }); + + it("blocks classifier hits with timestamps, category and score", () => { + const report = evaluateGate( + gateInput({ + content: [ + { tMs: 0, scores: { explicit_nudity: 0.05 } }, + { tMs: 1500, scores: { explicit_nudity: 0.4 } }, + { tMs: 2000, scores: { explicit_nudity: 0.6 } }, + { tMs: 3500, scores: { violence: 0.9, drugs: 1 } }, + ], + }), + ); + expect(report.passed).toBe(false); + expect(report.issues).toEqual([ + expect.objectContaining({ + check: "content", + code: "content_threshold", + category: "explicit_nudity", + atMs: 1500, + endMs: 2000, + score: 0.6, + }), + expect.objectContaining({ category: "violence", atMs: 3500, severity: "block" }), + ]); + }); + + it("only blocks above the ratio when the profile says so", () => { + const profile = testProfile({ + content: { thresholds: { violence: 0.5 }, blockOn: "ratio", blockRatio: 0.25 }, + }); + const samples = Array.from({ length: 10 }, (_, i) => ({ + tMs: i * 500, + scores: { violence: i < 2 ? 0.9 : 0 }, + })); + const below = evaluateGate(gateInput({ profile, content: samples })); + expect(below.passed).toBe(true); + expect(below.issues[0]).toMatchObject({ severity: "warn", category: "violence" }); + const above = samples.map((s, i) => ({ ...s, scores: { violence: i < 4 ? 0.9 : 0 } })); + expect(evaluateGate(gateInput({ profile, content: above })).passed).toBe(false); + }); + + it("blocks anonymization gaps, uncovered and unknown faces, and visible marks", () => { + const people = [ + { id: "person-1", anonymizeByDefault: "blur", consentRecordedAt: null, ageVerifiedAt: null }, + { id: "person-2", anonymizeByDefault: "none", consentRecordedAt: null, ageVerifiedAt: null }, + ]; + const box = { x: 0.4, y: 0.2, w: 0.1, h: 0.1 }; + const report = evaluateGate( + gateInput({ + people, + faces: [ + { tMs: 100, box, personId: "person-1", coverage: 1 }, + { tMs: 200, box, personId: "person-1", coverage: 0.9 }, + { tMs: 300, box, personId: "person-2", coverage: 0 }, + { tMs: 400, box, personId: null, coverage: 0 }, + ], + trackingGaps: [ + { startMs: 1000, endMs: 1400, personId: "person-1" }, + { startMs: 2000, endMs: 2600, personId: "person-1" }, + ], + marks: [ + { markId: "mark-1", tMs: 100, residualVisibility: 0.01 }, + { markId: "mark-1", tMs: 300, residualVisibility: 0.3 }, + ], + }), + ); + expect(report.issues.filter((i) => i.check === "anonymization")).toEqual([ + expect.objectContaining({ code: "face_not_covered", personId: "person-1", atMs: 200 }), + expect.objectContaining({ code: "unknown_face", atMs: 400 }), + expect.objectContaining({ + code: "tracking_gap", + atMs: 2000, + endMs: 2600, + personId: "person-1", + }), + expect.objectContaining({ code: "mark_visible", markId: "mark-1", atMs: 300 }), + ]); + // Without protected people an unmatched face is not an anonymization issue. + const open = evaluateGate(gateInput({ faces: [{ tMs: 0, box, personId: null, coverage: 0 }] })); + expect(open.passed).toBe(true); + }); + + it("requires consent and age verification only where the profile does", () => { + const people = [ + { + id: "p-ok", + anonymizeByDefault: "none", + consentRecordedAt: "2026-01-01T00:00:00Z", + ageVerifiedAt: "2026-01-01T00:00:00Z", + }, + { + id: "p-no-consent", + anonymizeByDefault: "none", + consentRecordedAt: null, + ageVerifiedAt: "2026-01-01T00:00:00Z", + }, + { + id: "p-no-age", + anonymizeByDefault: "none", + consentRecordedAt: "2026-01-01T00:00:00Z", + ageVerifiedAt: null, + }, + ]; + const faces = [{ tMs: 1000, box: { x: 0, y: 0, w: 0.1, h: 0.1 }, personId: null, coverage: 0 }]; + expect(evaluateGate(gateInput({ people, faces })).passed).toBe(true); + const profile = testPaidProfile(); + const edl = testEdl({ + output: { ...testEdl().output, profileId: profile.id, width: 1920, height: 1080 }, + }); + const report = evaluateGate( + gateInput({ profile, edl, people, faces, ffprobe: probe({ width: 1920, height: 1080 }) }), + ); + expect(report.issues.filter((i) => i.check === "consent")).toEqual([ + expect.objectContaining({ code: "age_unverified", personId: "p-no-age" }), + expect.objectContaining({ code: "consent_missing", personId: "p-no-consent" }), + expect.objectContaining({ code: "unknown_performer", atMs: 1000 }), + ]); + expect(report.passed).toBe(false); + }); + + it("requires the AI label when the profile does and warns about the watermark", () => { + const report = evaluateGate( + gateInput({ + aiAlterations: ["synthetic_identity"], + watermarkBox: { x: 0.9, y: 0.01, w: 0.1, h: 0.05 }, + }), + ); + expect(report.issues).toEqual([ + expect.objectContaining({ check: "disclosure", code: "ai_label_missing", severity: "block" }), + expect.objectContaining({ + check: "brand", + code: "watermark_outside_safe_zone", + severity: "warn", + }), + ]); + expect( + evaluateGate(gateInput({ aiAlterations: ["inpaint"], bundleHasDisclosure: true })).passed, + ).toBe(true); + const edl = testEdl({ metadata: { aiAltered: true, aiAlterations: ["voice_disguise"] } }); + expect(blockCodes(evaluateGate(gateInput({ edl })))).toEqual(["ai_label_missing"]); + const relaxed = testProfile({ disclosure: { aiAlteredLabelRequired: false } }); + expect(evaluateGate(gateInput({ profile: relaxed, edl })).passed).toBe(true); + }); +}); + +describe("checkCopy", () => { + const profile = testProfile({ + text: { + titleMaxChars: 20, + captionMaxChars: 60, + hashtagLimit: 2, + linkPolicy: "bio_only", + bannedPatterns: [ + { pattern: "free\\s+money", reason: "misleading claim" }, + { pattern: "([a-z", reason: "broken pattern" }, + ], + }, + }); + + it("checks banned patterns case-insensitively and reports invalid ones without crashing", () => { + const issues = checkCopy(profile, { + title: "FREE Money", + caption: null, + tags: ["free money"], + }); + expect(issues.map((i) => [i.code, i.message])).toEqual([ + ["banned_pattern", "title: misleading claim"], + ["banned_pattern", "tags[0]: misleading claim"], + ["invalid_banned_pattern", "profile pattern cannot be checked: broken pattern"], + ]); + }); + + it("checks lengths, hashtags and links", () => { + const codes = (copy: { title: string | null; caption: string | null; tags: string[] }) => + checkCopy(profile, copy) + .map((i) => i.code) + .filter((c) => c !== "invalid_banned_pattern"); + expect(codes({ title: "x".repeat(21), caption: "y".repeat(61), tags: [] })).toEqual([ + "title_too_long", + "caption_too_long", + ]); + expect(codes({ title: null, caption: "#a #b", tags: ["#c"] })).toEqual(["too_many_hashtags"]); + expect(codes({ title: null, caption: "#a #A", tags: ["a"] })).toEqual([]); + expect(codes({ title: null, caption: "see https://example.com/x", tags: [] })).toEqual([ + "link_bio_only", + ]); + expect(codes({ title: null, caption: "visit example.shop today", tags: [] })).toEqual([ + "link_bio_only", + ]); + expect(codes({ title: "Link in bio", caption: "Ready? Go.", tags: [] })).toEqual([]); + const forbidden = testProfile({ + text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "forbidden" }, + }); + expect( + checkCopy(forbidden, { title: "www.example.com", caption: null, tags: [] })[0]!.code, + ).toBe("link_forbidden"); + const allowed = testProfile({ + text: { captionMaxChars: 100, hashtagLimit: 5, linkPolicy: "allowed" }, + }); + expect(checkCopy(allowed, { title: null, caption: "https://example.com", tags: [] })).toEqual( + [], + ); + }); +}); + +describe("planAutoFix", () => { + const edl = testEdl({ + tracks: { + video: [ + { + id: "c1", + assetId: "asset-a", + inMs: 0, + outMs: 2000, + atMs: 0, + reframe: "auto", + segmentId: "s1", + }, + { + id: "c2", + assetId: "asset-a", + inMs: 5000, + outMs: 7000, + atMs: 2000, + reframe: "auto", + segmentId: "s2", + }, + { + id: "c3", + assetId: "asset-a", + inMs: 8000, + outMs: 10_000, + atMs: 4000, + reframe: "auto", + segmentId: "s3", + }, + ], + }, + transitions: [{ fromClipId: "c2", toClipId: "c3", kind: "cut", durationMs: 0 }], + overlays: [ + { id: "wm", kind: "watermark", atMs: 0, durationMs: 6000 }, + { id: "cta", kind: "cta_end_card", atMs: 4500, durationMs: 1500, text: "Link in bio" }, + ], + redactions: { + faces: [{ id: "f1", target: { personId: "person-1" }, mode: "blur", paddingPct: 20 }], + marks: [{ markId: "mark-1", treatment: "inpaint" }], + }, + metadata: { aiAltered: true, aiAlterations: ["inpaint"] }, + }); + const report = (issues: GateReport["issues"], round = 0): GateReport => ({ + profileId: "example-short-vertical", + passed: !issues.some((i) => i.severity === "block"), + checkedAt: "2026-10-01T12:00:00.000Z", + sampledFrames: 12, + issues, + autoFixRound: round, + }); + const contentHit = (atMs: number, endMs = atMs) => + ({ + check: "content", + severity: "block", + code: "content_threshold", + message: "violence", + atMs, + endMs, + category: "violence", + score: 0.9, + }) as const; + + const assertNoNewOverlays = (fixed: typeof edl) => { + const before = new Map(edl.overlays.map((o) => [o.id, o])); + for (const overlay of fixed.overlays) { + const original = before.get(overlay.id); + expect(original, `overlay ${overlay.id} was added`).toBeDefined(); + expect(overlay.kind).toBe(original!.kind); + expect(overlay.text).toBe(original!.text); + expect(overlay.durationMs).toBeLessThanOrEqual(original!.durationMs); + } + expect(fixed.overlays.length).toBeLessThanOrEqual(edl.overlays.length); + }; + + it("swaps an offending clip for a clean alternative segment", () => { + const fix = planAutoFix(edl, report([contentHit(2500, 3000)]), [ + { segmentId: "s2", assetId: "asset-a", startMs: 5000, endMs: 7000 }, + { segmentId: "s9", assetId: "asset-a", startMs: 5500, endMs: 9000 }, + { segmentId: "s8", assetId: "asset-b", startMs: 1000, endMs: 4000 }, + ])!; + expect(fix.changes).toEqual(["clip c2: swapped to segment s8 (content_threshold)"]); + expect(fix.edl.tracks.video.find((c) => c.id === "c2")).toMatchObject({ + assetId: "asset-b", + inMs: 1000, + outMs: 3000, + segmentId: "s8", + atMs: 2000, + }); + expect(edlDurationMs(fix.edl)).toBe(6000); + assertNoNewOverlays(fix.edl); + }); + + it("trims or drops the clip when no alternative fits, and repacks the timeline", () => { + const trimmed = planAutoFix(edl, report([contentHit(3800)]), [])!; + expect(trimmed.changes).toEqual(["clip c2: trimmed the end at 6300 ms (content_threshold)"]); + expect(trimmed.edl.tracks.video.map((c) => [c.id, c.atMs, c.inMs, c.outMs])).toEqual([ + ["c1", 0, 0, 2000], + ["c2", 2000, 5000, 6300], + ["c3", 3300, 8000, 10_000], + ]); + assertNoNewOverlays(trimmed.edl); + expect(trimmed.edl.overlays.find((o) => o.id === "wm")!.durationMs).toBe(5300); + + const dropped = planAutoFix(edl, report([contentHit(2300, 3700)]), [])!; + expect(dropped.changes).toEqual(["clip c2: removed (content_threshold)"]); + expect(dropped.edl.tracks.video.map((c) => c.id)).toEqual(["c1", "c3"]); + expect(dropped.edl.transitions).toEqual([]); + assertNoNewOverlays(dropped.edl); + }); + + it("cuts tracking gaps, widens face redactions, adds a blur for unknown faces, and covers marks", () => { + const fix = planAutoFix( + edl, + report([ + { + check: "anonymization", + severity: "block", + code: "face_not_covered", + message: "", + atMs: 100, + personId: "person-1", + }, + { check: "anonymization", severity: "block", code: "unknown_face", message: "", atMs: 300 }, + { + check: "anonymization", + severity: "block", + code: "mark_visible", + message: "", + atMs: 300, + markId: "mark-1", + }, + { + check: "anonymization", + severity: "block", + code: "tracking_gap", + message: "", + atMs: 4100, + endMs: 4800, + }, + ]), + [], + )!; + expect(fix.edl.redactions.faces).toEqual([ + expect.objectContaining({ id: "f1", paddingPct: 45 }), + expect.objectContaining({ + id: "fix-face-unassigned", + target: { unassignedFaces: true }, + mode: "blur", + }), + ]); + expect(fix.edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "cover" }]); + expect(fix.edl.metadata.aiAlterations).toEqual([]); + expect(fix.edl.tracks.video.find((c) => c.id === "c3")).toMatchObject({ inMs: 9300 }); + assertNoNewOverlays(fix.edl); + }); + + it("stops at the round limit, on passing reports, and when nothing safe applies", () => { + expect(planAutoFix(edl, report([contentHit(2500)], MAX_AUTO_FIX_ROUNDS), [])).toBeNull(); + expect(planAutoFix(edl, report([]), [])).toBeNull(); + const textOnly = report([ + { check: "text", severity: "block", code: "banned_pattern", message: "" }, + ]); + expect(planAutoFix(edl, textOnly, [])).toBeNull(); + const consent = report([ + { check: "consent", severity: "block", code: "consent_missing", message: "", personId: "p" }, + ]); + expect(planAutoFix(edl, consent, [])).toBeNull(); + }); + + it("runs at most three rounds in a loop", () => { + let current = edl; + let rounds = 0; + for (let round = 0; round < 10; round++) { + const fix = planAutoFix(current, report([contentHit(500)], round), []); + if (!fix) break; + current = fix.edl; + rounds += 1; + assertNoNewOverlays(current); + } + expect(rounds).toBe(MAX_AUTO_FIX_ROUNDS); + }); +}); diff --git a/packages/media/src/gate.ts b/packages/media/src/gate.ts index f3e4610..56d41ea 100644 --- a/packages/media/src/gate.ts +++ b/packages/media/src/gate.ts @@ -2,16 +2,74 @@ import type { AiAlteration, ContentCategory, Edl, + EdlVideoClip, + FaceRedaction, FfprobeReport, GateIssue, GateReport, NormalizedBox, PlatformProfile, } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import { CONTENT_CATEGORIES, MAX_AUTO_FIX_ROUNDS } from "@rakazo/contracts"; +import { + BLENDED_TRANSITIONS, + boxInsideSafeZone, + clipLengthMs, + edlDurationMs, + locateOutputTimeAll, + matchesAspect, + sortedVideoClips, +} from "./edl.js"; -/** Tags a clean export may carry. Everything else is a metadata leak. */ -export const METADATA_ALLOWED_TAGS: readonly string[] = []; +/** + * Tags a clean export may carry. Everything else is a metadata leak. + * + * - `title`: the only descriptive tag the compiler writes, and only when set. + * - `major_brand`, `minor_version`, `compatible_brands`: the MP4 file-type + * box, always written by the muxer; it names the format, not the author. + * - `handler_name`, `vendor_id`, `language`: per-stream boilerplate the MP4 + * muxer writes ("VideoHandler", "[0][0][0][0]", "und"). + * - `encoder`: the MP4 sample description always names the codec. It is + * allowed only without a version number (bitexact output writes + * "Lavc libx264"); a versioned value such as "Lavf60.16.100" is a leak. + * + * `creation_time`, `location`, `com.apple.quicktime.*`, `make`, `model`, + * `software`, GPS keys and anything unknown always count as leaks. + */ +export const METADATA_ALLOWED_TAGS: readonly string[] = [ + "title", + "major_brand", + "minor_version", + "compatible_brands", + "handler_name", + "vendor_id", + "language", + "encoder", +]; + +const ALLOWED = new Set(METADATA_ALLOWED_TAGS); +/** "Lavf60.16.100", "Lavc60.31", "x264 core 164", "HandBrake 1.7.0". */ +const VERSIONED = /\d+\.\d+|lav[cf]\d|core\s*\d/i; + +/** Tag name without a "format." / "stream.0." style prefix, lower-cased. */ +function tagName(key: string): string { + return key + .trim() + .toLowerCase() + .replace(/^(?:format|streams?(?:[.:/]\d+)?)[.:/]/, ""); +} + +export function findMetadataLeaks(tags: Record): string[] { + return Object.entries(tags) + .filter(([key, value]) => { + const name = tagName(key); + if (!ALLOWED.has(name)) return true; + if (name === "encoder") return VERSIONED.test(value); + return false; + }) + .map(([key]) => key) + .sort(); +} export interface GateInput { profile: PlatformProfile; @@ -44,28 +102,370 @@ export interface GateInput { now: Date; } -/** Deterministic gate (spec 6). Any blocking issue fails the report. */ -export function evaluateGate(_input: GateInput): GateReport { - return notImplemented("evaluateGate"); +/** Longest tracking gap the anonymizer may bridge (spec 7). */ +export const MAX_TRACKING_GAP_MS = 500; +/** Residual visibility of a treated mark above which the gate blocks. */ +export const MAX_MARK_RESIDUAL = 0.05; +/** Loudness tolerance around the target, in LU. */ +export const LOUDNESS_TOLERANCE_LU = 2; +const TRUE_PEAK_TOLERANCE_DB = 0.5; + +const ms = (value: number) => Math.max(0, Math.round(value)); + +/** Merge time-sorted hits into ranges when they are at most `gapMs` apart. */ +function ranges(hits: T[], gapMs: number): T[][] { + const sorted = [...hits].sort((a, b) => a.tMs - b.tMs); + const out: T[][] = []; + for (const hit of sorted) { + const last = out.at(-1); + if (last && hit.tMs - last.at(-1)!.tMs <= gapMs) last.push(hit); + else out.push([hit]); + } + return out; } export function checkTechnical( - _profile: PlatformProfile, - _edl: Edl, - _probe: FfprobeReport, + profile: PlatformProfile, + edl: Edl, + probe: FfprobeReport, ): GateIssue[] { - return notImplemented("checkTechnical"); + const issues: GateIssue[] = []; + const block = (code: string, message: string) => + issues.push({ check: "technical", severity: "block", code, message }); + if (probe.width !== edl.output.width || probe.height !== edl.output.height) { + block( + "resolution_mismatch", + `output is ${probe.width}x${probe.height}, plan expects ${edl.output.width}x${edl.output.height}`, + ); + } + if (!matchesAspect(probe.width, probe.height, profile.video.aspectRatios)) { + block("aspect_not_allowed", `${probe.width}x${probe.height} is not an allowed aspect ratio`); + } + if (Math.max(probe.width, probe.height) > Math.max(profile.video.width, profile.video.height)) { + block("resolution_too_high", "output is larger than the profile allows"); + } + if ( + Math.abs(probe.fps - edl.output.fps) > edl.output.fps * 0.01 || + probe.fps > profile.video.fps * 1.001 + ) { + block("fps_mismatch", `output runs at ${probe.fps} fps`); + } + const frameMs = 1000 / Math.max(1, edl.output.fps); + const expected = edlDurationMs(edl); + if (probe.durationMs < profile.duration.hardMinMs - frameMs) { + block("duration_too_short", `${probe.durationMs} ms is below ${profile.duration.hardMinMs} ms`); + } + if (probe.durationMs > profile.duration.hardMaxMs + frameMs) { + block("duration_too_long", `${probe.durationMs} ms is above ${profile.duration.hardMaxMs} ms`); + } + if (Math.abs(probe.durationMs - expected) > Math.max(2 * frameMs, 100)) { + issues.push({ + check: "technical", + severity: "warn", + code: "duration_drift", + message: `output is ${probe.durationMs} ms, plan is ${expected} ms`, + }); + } + if (probe.sizeBytes > profile.video.maxFileSizeMb * 1_000_000) { + block("file_too_large", `file is larger than ${profile.video.maxFileSizeMb} MB`); + } + const codec = probe.videoCodec.toLowerCase(); + const expectedCodec = profile.video.codec === "hevc" ? ["hevc", "h265"] : ["h264", "avc1"]; + if (!expectedCodec.includes(codec)) { + block("codec_mismatch", `video codec ${probe.videoCodec} is not ${profile.video.codec}`); + } + if (probe.audioCodec !== null && probe.audioCodec.toLowerCase() !== "aac") { + issues.push({ + check: "technical", + severity: "warn", + code: "audio_codec", + message: `audio codec ${probe.audioCodec} is not AAC`, + }); + } + const target = edl.targetLufs ?? profile.audio.targetLufs; + if (probe.audioCodec !== null) { + if (probe.integratedLufs === null) { + block("loudness_unmeasured", "loudness was not measured"); + } else if (Math.abs(probe.integratedLufs - target) > LOUDNESS_TOLERANCE_LU) { + block("loudness_off_target", `${probe.integratedLufs} LUFS, target ${target} LUFS`); + } + if ( + probe.truePeakDb !== null && + probe.truePeakDb > profile.audio.truePeakDb + TRUE_PEAK_TOLERANCE_DB + ) { + block("true_peak_exceeded", `true peak ${probe.truePeakDb} dBTP`); + } + } + const leaks = findMetadataLeaks(probe.tags); + if (leaks.length > 0) block("metadata_leak", `metadata not removed: ${leaks.join(", ")}`); + return issues; } -export function findMetadataLeaks(_tags: Record): string[] { - return notImplemented("findMetadataLeaks"); +const URL_PATTERN = + /\b(?:https?:\/\/|www\.)\S+|\b[a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:com|net|org|io|de|at|ch|eu|ly|co|me|link|shop|store|app|tv|gg|xyz)\b(?:\/\S*)?/i; +const HASHTAG_PATTERN = /#[\p{L}\p{N}_]+/gu; + +/** Profile patterns compile case-insensitively; an invalid one is reported, never thrown. */ +function compilePattern(pattern: string): RegExp | null { + try { + return new RegExp(pattern, "i"); + } catch { + return null; + } } export function checkCopy( - _profile: PlatformProfile, - _copy: { title: string | null; caption: string | null; tags: string[] }, + profile: PlatformProfile, + copy: { title: string | null; caption: string | null; tags: string[] }, ): GateIssue[] { - return notImplemented("checkCopy"); + const issues: GateIssue[] = []; + const block = (code: string, message: string) => + issues.push({ check: "text", severity: "block", code, message }); + const { text } = profile; + const title = copy.title ?? ""; + const caption = copy.caption ?? ""; + if (text.titleMaxChars !== undefined && [...title].length > text.titleMaxChars) { + block( + "title_too_long", + `title has ${[...title].length} characters, limit ${text.titleMaxChars}`, + ); + } + if ([...caption].length > text.captionMaxChars) { + block( + "caption_too_long", + `caption has ${[...caption].length} characters, limit ${text.captionMaxChars}`, + ); + } + const hashtags = new Set(); + for (const tag of copy.tags) { + const clean = tag.replace(/^#+/, "").trim().toLowerCase(); + if (clean) hashtags.add(clean); + } + for (const match of caption.matchAll(HASHTAG_PATTERN)) + hashtags.add(match[0].slice(1).toLowerCase()); + if (hashtags.size > text.hashtagLimit) { + block("too_many_hashtags", `${hashtags.size} hashtags, limit ${text.hashtagLimit}`); + } + const fields: Array<[string, string]> = [ + ["title", title], + ["caption", caption], + ...copy.tags.map((tag, i): [string, string] => [`tags[${i}]`, tag]), + ]; + for (const banned of text.bannedPatterns) { + const regex = compilePattern(banned.pattern); + if (!regex) { + block("invalid_banned_pattern", `profile pattern cannot be checked: ${banned.reason}`); + continue; + } + for (const [field, value] of fields) { + if (value && regex.test(value)) block("banned_pattern", `${field}: ${banned.reason}`); + } + } + if (text.linkPolicy !== "allowed") { + for (const [field, value] of fields) { + if (!value || !URL_PATTERN.test(value)) continue; + if (text.linkPolicy === "forbidden") block("link_forbidden", `${field} contains a link`); + else block("link_bio_only", `${field} contains a link; links belong in the bio`); + } + } + return issues; +} + +function checkContent(input: GateInput): GateIssue[] { + const { profile } = input; + const issues: GateIssue[] = []; + const thresholds = profile.content.thresholds; + const gapMs = (1000 / profile.content.sampleFps) * 1.5; + const sampled = Math.max(input.content.length, input.sampledFrames, 1); + for (const category of CONTENT_CATEGORIES) { + const threshold = thresholds[category]; + if (threshold === undefined) continue; + const hits = input.content.flatMap((sample) => { + const score = sample.scores[category]; + return score !== undefined && score > threshold ? [{ tMs: sample.tMs, score }] : []; + }); + if (hits.length === 0) continue; + const blocking = + profile.content.blockOn === "any_frame" || + hits.length / sampled > (profile.content.blockRatio ?? 0); + for (const range of ranges(hits, gapMs)) { + const score = Math.max(...range.map((h) => h.score)); + issues.push({ + check: "content", + severity: blocking ? "block" : "warn", + code: "content_threshold", + message: `${category} ${score.toFixed(2)} above ${threshold}`, + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + category, + score, + }); + } + } + return issues; +} + +function checkAnonymization(input: GateInput): GateIssue[] { + const issues: GateIssue[] = []; + const protectedIds = new Set( + input.people.filter((p) => p.anonymizeByDefault !== "none").map((p) => p.id), + ); + const gapMs = 1000; + const uncovered = input.faces.filter( + (face) => face.personId !== null && protectedIds.has(face.personId) && face.coverage < 1, + ); + const byPerson = new Map(); + for (const face of uncovered) { + const list = byPerson.get(face.personId!) ?? []; + list.push(face); + byPerson.set(face.personId!, list); + } + for (const [personId, faces] of [...byPerson.entries()].sort((a, b) => + a[0].localeCompare(b[0]), + )) { + for (const range of ranges(faces, gapMs)) { + const coverage = Math.min(...range.map((f) => f.coverage)); + issues.push({ + check: "anonymization", + severity: "block", + code: "face_not_covered", + message: `face covered ${(coverage * 100).toFixed(0)} %`, + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + personId, + score: coverage, + }); + } + } + if (protectedIds.size > 0) { + const unknown = input.faces.filter((face) => face.personId === null); + for (const range of ranges(unknown, gapMs)) { + issues.push({ + check: "anonymization", + severity: "block", + code: "unknown_face", + message: "a face that cannot be matched appears next to a protected person", + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + }); + } + } + for (const gap of input.trackingGaps) { + if (gap.endMs - gap.startMs <= MAX_TRACKING_GAP_MS) continue; + issues.push({ + check: "anonymization", + severity: "block", + code: "tracking_gap", + message: `face tracking lost for ${Math.round(gap.endMs - gap.startMs)} ms (needs review)`, + atMs: ms(gap.startMs), + endMs: ms(gap.endMs), + ...(gap.personId ? { personId: gap.personId } : {}), + }); + } + const visibleMarks = input.marks.filter((mark) => mark.residualVisibility > MAX_MARK_RESIDUAL); + const byMark = new Map(); + for (const mark of visibleMarks) { + const list = byMark.get(mark.markId) ?? []; + list.push(mark); + byMark.set(mark.markId, list); + } + for (const [markId, marks] of [...byMark.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + for (const range of ranges(marks, gapMs)) { + const residual = Math.max(...range.map((m) => m.residualVisibility)); + issues.push({ + check: "anonymization", + severity: "block", + code: "mark_visible", + message: `treated mark still ${(residual * 100).toFixed(0)} % visible`, + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + markId, + score: residual, + }); + } + } + return issues; +} + +function checkConsent(input: GateInput): GateIssue[] { + const { consent } = input.profile; + if (!consent.requirePerformerConsent && !consent.requireAgeVerification) return []; + const issues: GateIssue[] = []; + for (const person of [...input.people].sort((a, b) => a.id.localeCompare(b.id))) { + if (consent.requirePerformerConsent && !person.consentRecordedAt) { + issues.push({ + check: "consent", + severity: "block", + code: "consent_missing", + message: "no recorded consent", + personId: person.id, + }); + } + if (consent.requireAgeVerification && !person.ageVerifiedAt) { + issues.push({ + check: "consent", + severity: "block", + code: "age_unverified", + message: "age is not verified", + personId: person.id, + }); + } + } + for (const range of ranges( + input.faces.filter((face) => face.personId === null), + 1000, + )) { + issues.push({ + check: "consent", + severity: "block", + code: "unknown_performer", + message: "a visible face is not assigned to a person with consent", + atMs: ms(range[0]!.tMs), + endMs: ms(range.at(-1)!.tMs), + }); + } + return issues; +} + +/** Deterministic gate (spec 6). Any blocking issue fails the report. */ +export function evaluateGate(input: GateInput): GateReport { + const { profile } = input; + const issues: GateIssue[] = [ + ...checkTechnical(profile, input.edl, input.ffprobe), + ...checkContent(input), + ...checkAnonymization(input), + ...checkConsent(input), + ...checkCopy(profile, input.copy), + ]; + const alterations = new Set([...input.aiAlterations, ...input.edl.metadata.aiAlterations]); + if ( + alterations.size > 0 && + profile.disclosure.aiAlteredLabelRequired && + !input.bundleHasDisclosure + ) { + issues.push({ + check: "disclosure", + severity: "block", + code: "ai_label_missing", + message: `AI alterations (${[...alterations].sort().join(", ")}) need the platform label`, + }); + } + if (input.watermarkBox && !boxInsideSafeZone(input.watermarkBox, profile)) { + issues.push({ + check: "brand", + severity: "warn", + code: "watermark_outside_safe_zone", + message: "watermark is covered by platform interface", + }); + } + return { + profileId: profile.id, + passed: !issues.some((issue) => issue.severity === "block"), + checkedAt: input.now.toISOString(), + sampledFrames: Math.max(0, Math.round(input.sampledFrames)), + issues, + autoFixRound: Math.min(MAX_AUTO_FIX_ROUNDS, Math.max(0, Math.round(input.autoFixRound))), + }; } export interface AutoFixResult { @@ -73,15 +473,227 @@ export interface AutoFixResult { changes: string[]; } +const PADDING_STEP = 25; +const CUT_MARGIN_MS = 500; +const MIN_KEEP_MS = 500; + +/** Lay clips out back to back again, keeping blended transitions' overlap. */ +function repack(edl: Edl): Edl { + const clips = sortedVideoClips(edl); + const ids = new Set(clips.map((c) => c.id)); + const transitions = edl.transitions.filter((t) => ids.has(t.fromClipId) && ids.has(t.toClipId)); + const adjacent = new Set(); + for (let i = 0; i + 1 < clips.length; i++) adjacent.add(`${clips[i]!.id}>${clips[i + 1]!.id}`); + const kept = transitions.filter((t) => adjacent.has(`${t.fromClipId}>${t.toClipId}`)); + const blend = new Map( + kept + .filter((t) => BLENDED_TRANSITIONS.has(t.kind) && t.durationMs > 0) + .map((t) => [`${t.fromClipId}>${t.toClipId}`, t.durationMs]), + ); + let cursor = 0; + const packed: EdlVideoClip[] = clips.map((clip, i) => { + const previous = clips[i - 1]; + let overlap = previous ? (blend.get(`${previous.id}>${clip.id}`) ?? 0) : 0; + overlap = Math.min(overlap, clipLengthMs(clip) - 1, cursor); + const atMs = Math.max(0, cursor - Math.max(0, overlap)); + cursor = atMs + clipLengthMs(clip); + return { ...clip, atMs }; + }); + const regions = edl.redactions.regions.filter((r) => ids.has(r.clipId)); + const next = { + ...edl, + tracks: { ...edl.tracks, video: packed }, + transitions: kept.filter((t) => { + if (!BLENDED_TRANSITIONS.has(t.kind)) return true; + const from = packed.find((c) => c.id === t.fromClipId)!; + const to = packed.find((c) => c.id === t.toClipId)!; + return t.durationMs < Math.min(clipLengthMs(from), clipLengthMs(to)); + }), + redactions: { ...edl.redactions, regions }, + }; + // Overlays never grow or move; they are only clamped to the new end. + const total = edlDurationMs(next); + next.overlays = edl.overlays.flatMap((overlay) => { + if (overlay.atMs >= total) return []; + return [{ ...overlay, durationMs: Math.min(overlay.durationMs, total - overlay.atMs) }]; + }); + return next; +} + /** * Compliance-only repairs: trim or drop offending clips, change crop, widen * redaction padding, swap to another segment. Never adds obfuscation aimed at * classifiers. Returns null when the round limit is reached or nothing safe applies. + * + * - Content blocks and tracking gaps: the clip showing that moment is swapped + * for an alternative segment of at least the same length that does not + * overlap the offending source range, else trimmed (when the range sits at + * one end) or dropped. The timeline is then repacked. + * - Uncovered faces: padding of the person's face redaction grows by 25 + * points; a missing redaction is added as blur. + * - Unknown faces: the unassigned-face redaction grows or is added as blur. + * - Visible marks: the mark treatment falls back to cover. */ export function planAutoFix( - _edl: Edl, - _report: GateReport, - _alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>, + edl: Edl, + report: GateReport, + alternatives: Array<{ segmentId: string; assetId: string; startMs: number; endMs: number }>, ): AutoFixResult | null { - return notImplemented("planAutoFix"); + if (report.autoFixRound >= MAX_AUTO_FIX_ROUNDS || report.passed) return null; + const blocks = report.issues.filter((issue) => issue.severity === "block"); + const changes: string[] = []; + let clips: EdlVideoClip[] = edl.tracks.video.map((clip) => ({ ...clip })); + let faces: FaceRedaction[] = edl.redactions.faces.map((f) => ({ ...f })); + let marks = edl.redactions.marks.map((m) => ({ ...m })); + let clipsChanged = false; + + // Source ranges to remove, per clip. + const offending = new Map>(); + for (const issue of blocks) { + const timeline = + issue.check === "content" || + issue.code === "tracking_gap" || + (issue.code === "mark_visible" && + !edl.redactions.marks.some((m) => m.markId === issue.markId)); + if (!timeline || issue.atMs === undefined) continue; + const endMs = issue.endMs ?? issue.atMs; + const step = Math.max(1, Math.floor(1000 / edl.output.fps)); + const times: number[] = []; + for (let t = issue.atMs; t < endMs; t += step) times.push(t); + times.push(endMs); + for (const t of times) { + for (const hit of locateOutputTimeAll(edl, t)) { + const list = offending.get(hit.clipId) ?? []; + const last = list.at(-1); + if (last && last.reason === issue.code && hit.sourceMs - last.end <= step * 4) { + last.end = Math.max(last.end, hit.sourceMs); + } else { + list.push({ start: hit.sourceMs, end: hit.sourceMs, reason: issue.code }); + } + offending.set(hit.clipId, list); + } + } + } + const usedAlternatives = new Set(); + for (const [clipId, found] of [...offending.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + const clip = clips.find((c) => c.id === clipId); + if (!clip) continue; + const start = Math.max(clip.inMs, Math.min(...found.map((r) => r.start)) - CUT_MARGIN_MS); + const end = Math.min(clip.outMs, Math.max(...found.map((r) => r.end)) + CUT_MARGIN_MS); + const length = clip.outMs - clip.inMs; + const alternative = alternatives + .filter( + (a) => + !usedAlternatives.has(a.segmentId) && + a.segmentId !== clip.segmentId && + a.endMs - a.startMs >= length && + !(a.assetId === clip.assetId && a.startMs < end && start < a.endMs), + ) + .sort((a, b) => a.segmentId.localeCompare(b.segmentId))[0]; + clipsChanged = true; + if (alternative) { + usedAlternatives.add(alternative.segmentId); + Object.assign(clip, { + assetId: alternative.assetId, + segmentId: alternative.segmentId, + inMs: alternative.startMs, + outMs: alternative.startMs + length, + reframe: clip.reframe === "none" ? "none" : "auto", + }); + changes.push( + `clip ${clip.id}: swapped to segment ${alternative.segmentId} (${found[0]!.reason})`, + ); + } else if (start - clip.inMs >= MIN_KEEP_MS && end >= clip.outMs - 1) { + clip.outMs = start; + changes.push(`clip ${clip.id}: trimmed the end at ${start} ms (${found[0]!.reason})`); + } else if (clip.outMs - end >= MIN_KEEP_MS && start <= clip.inMs + 1) { + clip.inMs = end; + changes.push(`clip ${clip.id}: trimmed the start to ${end} ms (${found[0]!.reason})`); + } else { + clips = clips.filter((c) => c.id !== clip.id); + changes.push(`clip ${clip.id}: removed (${found[0]!.reason})`); + } + } + for (const clip of clips) { + if (Array.isArray(clip.reframe)) { + const inside = clip.reframe.filter((k) => k.tMs >= clip.inMs && k.tMs <= clip.outMs); + clip.reframe = inside.length > 0 ? inside : "auto"; + } + } + + const widen = (match: (f: FaceRedaction) => boolean, add: () => FaceRedaction, label: string) => { + const existing = faces.filter(match); + if (existing.length === 0) { + faces = [...faces, add()]; + changes.push(`${label}: added blur redaction`); + return; + } + for (const face of existing) { + if (face.paddingPct >= 100) continue; + face.paddingPct = Math.min(100, face.paddingPct + PADDING_STEP); + changes.push(`${label}: redaction ${face.id} padding ${face.paddingPct} %`); + } + }; + const widened = new Set(); + for (const issue of blocks) { + if (issue.code === "face_not_covered" && issue.personId && !widened.has(issue.personId)) { + const personId = issue.personId; + widened.add(personId); + widen( + (f) => "personId" in f.target && f.target.personId === personId, + () => ({ + id: `fix-face-${widened.size}`, + target: { personId }, + mode: "blur", + paddingPct: 20 + PADDING_STEP, + featherPct: 10, + minSizePct: 2, + }), + `person ${personId}`, + ); + } + if (issue.code === "unknown_face" && !widened.has("(unassigned)")) { + widened.add("(unassigned)"); + widen( + (f) => "unassignedFaces" in f.target, + () => ({ + id: "fix-face-unassigned", + target: { unassignedFaces: true }, + mode: "blur", + paddingPct: 20 + PADDING_STEP, + featherPct: 10, + minSizePct: 2, + }), + "unassigned faces", + ); + } + if (issue.code === "mark_visible" && issue.markId) { + const markId = issue.markId; + marks = marks.map((mark) => { + if (mark.markId !== markId || mark.treatment === "cover") return mark; + changes.push(`mark ${markId}: ${mark.treatment} replaced by cover`); + return { ...mark, treatment: "cover" as const }; + }); + } + } + if (changes.length === 0 || clips.length === 0) return null; + + let next: Edl = { + ...edl, + tracks: { ...edl.tracks, video: clips }, + redactions: { ...edl.redactions, faces, marks }, + }; + if (clipsChanged) next = repack(next); + const stillAltered = new Set( + next.metadata.aiAlterations.filter( + (a) => a !== "inpaint" || marks.some((m) => m.treatment === "inpaint"), + ), + ); + const explicit = edl.metadata.aiAltered && edl.metadata.aiAlterations.length === 0; + next.metadata = { + ...next.metadata, + aiAlterations: [...stillAltered], + aiAltered: stillAltered.size > 0 || explicit, + }; + return { edl: next, changes }; } diff --git a/packages/media/src/marketing.test.ts b/packages/media/src/marketing.test.ts new file mode 100644 index 0000000..f2d43fc --- /dev/null +++ b/packages/media/src/marketing.test.ts @@ -0,0 +1,320 @@ +import type { MetricCsvMapping, MotionTemplateInfo } from "@rakazo/contracts"; +import { MetricCsvMappingSchema } from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { + buildTrackingUrl, + contrastRatio, + introFits, + normalizeHashtags, + parseCsvRecords, + parseMetricsCsv, + parseMoneyToCents, + proposeIntroVariants, + uploadChecklist, + validateBrandKit, +} from "./marketing.js"; +import { testKit, testPaidProfile, testProfile } from "./test-fixtures.js"; + +const utm = { source: "example-short-vertical", medium: "short", campaign: "spring launch" }; + +describe("buildTrackingUrl", () => { + it("appends UTM parameters and keeps the existing query and fragment", () => { + expect(buildTrackingUrl("https://example.com/shop", utm)).toBe( + "https://example.com/shop?utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch", + ); + expect( + buildTrackingUrl("https://example.com/p?ref=a%20b&x=1&utm_source=old#top", { + ...utm, + content: "thumb-2", + term: "ä&b", + }), + ).toBe( + "https://example.com/p?ref=a%20b&x=1&utm_source=example-short-vertical&utm_medium=short&utm_campaign=spring%20launch&utm_content=thumb-2&utm_term=%C3%A4%26b#top", + ); + expect(buildTrackingUrl("http://example.com", utm)).toMatch( + /^http:\/\/example\.com\/\?utm_source=/, + ); + }); + + it("rejects non-http schemes and invalid URLs", () => { + expect(() => buildTrackingUrl("javascript:alert(1)", utm)).toThrow(/http/); + expect(() => buildTrackingUrl("ftp://example.com", utm)).toThrow(/http/); + expect(() => buildTrackingUrl("not a url", utm)).toThrow(/valid/); + }); +}); + +describe("normalizeHashtags", () => { + it("normalizes, dedupes and limits", () => { + expect( + normalizeHashtags(["#Summer", "summer", "##new day", "über-cool!", "", "#"], 10), + ).toEqual(["#Summer", "#newday", "#übercool"]); + expect(normalizeHashtags(["a", "b", "c"], 2)).toEqual(["#a", "#b"]); + expect(normalizeHashtags(["a"], 0)).toEqual([]); + }); +}); + +describe("CSV", () => { + const mapping = (overrides: Partial = {}): MetricCsvMapping => + MetricCsvMappingSchema.parse({ + name: "Example", + columns: { + publication: "Link", + observedAt: "Date", + views: "Views", + clicks: "Clicks", + conversions: "Sales", + revenue: "Revenue", + }, + publicationKey: "tracking_url", + ...overrides, + }); + + it("parses RFC 4180 records with quotes, delimiters and newlines in fields", () => { + expect(parseCsvRecords('a,"b,c","say ""hi""\nthere"\r\n1,2,3\n', ",")).toEqual([ + ["a", "b,c", 'say "hi"\nthere'], + ["1", "2", "3"], + ]); + expect(parseCsvRecords("x;y\n", ";")).toEqual([["x", "y"]]); + expect(parseCsvRecords("", ",")).toEqual([]); + }); + + it("imports metrics with BOM, comma delimiter and major-unit revenue", () => { + const csv = + 'Link,Date,Views,Clicks,Sales,Revenue\n"https://example.com/?utm_content=a,b",2026-09-01,"1,200",30,2,"19.98"\n'; + const result = parseMetricsCsv(csv, mapping()); + expect(result.skipped).toEqual([]); + expect(result.rows).toEqual([ + { + row: 2, + publicationKey: "https://example.com/?utm_content=a,b", + observedAt: new Date("2026-09-01T00:00:00Z"), + views: 1200, + clicks: 30, + conversions: 2, + revenueCents: 1998, + currency: "EUR", + }, + ]); + }); + + it("handles semicolons, tabs, decimal commas, currencies and skipped rows", () => { + const semicolon = [ + "link;date;views;clicks;sales;revenue;currency", + "pub-1;01.09.2026;1.500;12;1;1.234,50 €;", + "pub-2;2026/09/02;10;1;0;12,5;usd", + "pub-3;31.02.2026;1;0;0;0;", + "pub-4;2026-09-03;ten;0;0;0;", + ";2026-09-03;1;0;0;0;", + "pub-5;2026-09-04;1;0;0;-5;", + "pub-6;2026-09-04;1;0;0;1,2,3;", + "pub-7;2026-09-04;1;0;0;5;EURO", + ";;;;;;", + "pub-8;2026-09-05T10:30:00Z;7;;;$3.10;", + ].join("\r\n"); + const result = parseMetricsCsv( + semicolon, + mapping({ + delimiter: ";", + columns: { + publication: "LINK", + observedAt: "date", + views: "views", + clicks: "clicks", + conversions: "sales", + revenue: "revenue", + currency: "currency", + }, + }), + ); + expect(result.rows.map((r) => [r.publicationKey, r.views, r.revenueCents, r.currency])).toEqual( + [ + ["pub-1", 1500, 123_450, "EUR"], + ["pub-2", 10, 1250, "USD"], + ["pub-8", 7, 310, "USD"], + ], + ); + expect(result.rows[2]!.observedAt).toEqual(new Date("2026-09-05T10:30:00Z")); + expect(result.skipped).toEqual([ + { row: 4, reason: "date is not recognized" }, + { row: 5, reason: "views is not a whole number" }, + { row: 6, reason: "publication is empty" }, + { row: 7, reason: "revenue is negative" }, + { row: 8, reason: "revenue is not a number" }, + { row: 9, reason: "currency is not a 3-letter code" }, + ]); + + const tabs = parseMetricsCsv( + "Link\tDate\tRevenue\nx\t2026-09-01\t250\n", + mapping({ + delimiter: "\t", + revenueUnit: "cents", + columns: { publication: "Link", observedAt: "Date", revenue: "Revenue" }, + }), + ); + expect(tabs.rows[0]).toMatchObject({ views: 0, clicks: 0, conversions: 0, revenueCents: 250 }); + }); + + it("reports missing columns and empty files", () => { + expect(parseMetricsCsv("Other,Date\n", mapping())).toEqual({ + rows: [], + skipped: [{ row: 1, reason: "missing columns: Link, Views, Clicks, Sales, Revenue" }], + }); + expect(parseMetricsCsv("", mapping()).skipped).toEqual([{ row: 1, reason: "file is empty" }]); + }); + + it("converts money in either notation to cents", () => { + expect(parseMoneyToCents("1,234.56")).toEqual({ cents: 123_456, currency: null }); + expect(parseMoneyToCents("1.234,56")).toEqual({ cents: 123_456, currency: null }); + expect(parseMoneyToCents("1,234")).toEqual({ cents: 123_400, currency: null }); + expect(parseMoneyToCents("0.995")).toEqual({ cents: 100, currency: null }); + expect(parseMoneyToCents("2.5")).toEqual({ cents: 250, currency: null }); + expect(parseMoneyToCents("CHF 7")).toEqual({ cents: 700, currency: "CHF" }); + expect(parseMoneyToCents("12.345.678,9")).toEqual({ cents: 1_234_567_890, currency: null }); + expect(parseMoneyToCents("1,5,0")).toBeNull(); + }); +}); + +describe("intro", () => { + const templates: MotionTemplateInfo[] = [ + { + id: "intro-glitch", + kind: "intro", + name: "Glitch energy", + minDurationMs: 1000, + maxDurationMs: 5000, + params: [{ name: "primaryColor", type: "color", required: true }], + }, + { + id: "intro-elegant", + kind: "intro", + name: "Elegant fade", + minDurationMs: 2000, + maxDurationMs: 3000, + params: [ + { name: "brandName", type: "string", required: true }, + { name: "sound", type: "string", required: false }, + ], + }, + { + id: "intro-kinetic", + kind: "intro", + name: "Kinetic type", + minDurationMs: 1500, + maxDurationMs: 4000, + params: [], + }, + { + id: "intro-logo", + kind: "intro", + name: "Minimal logo reveal", + minDurationMs: 1500, + maxDurationMs: 4000, + params: [{ name: "logoArtifactId", type: "string", required: true }], + }, + { + id: "outro-a", + kind: "outro", + name: "Outro", + minDurationMs: 1000, + maxDurationMs: 3000, + params: [], + }, + ]; + + it("keeps shorts hook-first and allows short intros on long-form profiles", () => { + expect(introFits(testProfile(), 1500)).toBe(false); + expect(introFits(testProfile(), 0)).toBe(true); + expect(introFits(testPaidProfile(), 3000)).toBe(true); + expect(introFits(testPaidProfile(), 5000)).toBe(false); + }); + + it("proposes three distinct variants ordered by tone", () => { + const kit = testKit({ tone: "calm, premium", logoArtifactIds: ["logo-1"] }); + const variants = proposeIntroVariants(kit, templates, 3); + expect(variants.map((v) => v.template.templateId)).toEqual([ + "intro-elegant", + "intro-logo", + "intro-glitch", + ]); + expect(new Set(variants.map((v) => v.template.templateId)).size).toBe(3); + for (const variant of variants) { + expect(variant.template.durationMs).toBeGreaterThanOrEqual(1500); + expect(variant.template.durationMs).toBeLessThanOrEqual(4000); + } + expect(variants[0]!.template.params).toEqual({ brandName: "Studio Example", sound: "music-1" }); + expect(variants[0]!.rationale).toContain('matches the tone "calm, premium"'); + expect(variants[1]!.template.params).toEqual({ logoArtifactId: "logo-1" }); + expect(proposeIntroVariants(kit, templates, 3)).toEqual(variants); + const energetic = proposeIntroVariants(testKit({ tone: "bold and energetic" }), templates, 3); + expect(energetic[0]!.template).toMatchObject({ + templateId: "intro-glitch", + params: { primaryColor: "#112233" }, + }); + expect(proposeIntroVariants(kit, [], 3)).toEqual([]); + expect( + proposeIntroVariants(kit, templates.slice(0, 1), 3).map((v) => v.template.durationMs), + ).toEqual([2500, 3000, 1500]); + }); +}); + +describe("validateBrandKit", () => { + it("flags low contrast, duplicate presets, unlicensed music and fonts", () => { + expect(validateBrandKit(testKit())).toEqual([]); + expect(contrastRatio("#FFFFFF", "#000000")).toBeCloseTo(21, 5); + const preset = testKit().captionPresets[0]!; + const warnings = validateBrandKit( + testKit({ + colors: { ...testKit().colors, text: "#777777", background: "#888888" }, + captionPresets: [preset, { ...preset, backgroundColor: "#EEEEEE" }], + musicBeds: [ + { artifactId: "m", title: "Borrowed", license: "unknown", commercialUse: false }, + ], + fonts: [{ family: "Nameless", artifactId: "f", role: "body" }], + }), + ); + expect(warnings).toEqual([ + "text on background has contrast 1.3:1, below 4.5:1", + 'caption preset id "clean" is used twice', + 'caption preset "clean" has contrast 1.2:1, below 4.5:1', + 'music bed "Borrowed" is not licensed for commercial use', + 'font "Nameless" has no license note', + ]); + }); +}); + +describe("uploadChecklist", () => { + it("lists the AI label, age restriction, link policy and captions where they apply", () => { + const paid = uploadChecklist({ + profile: testPaidProfile(), + aiAltered: true, + hasCaptions: true, + trackingUrl: "https://example.com/?utm_source=x", + }); + expect(paid).toContain("# Upload checklist: Example paid video"); + expect(paid).toContain("AI-generated or altered content label"); + expect(paid).toContain("age-restricted"); + expect(paid).toContain("Add the tracking link: "); + expect(paid).toContain("SRT or VTT"); + + const short = uploadChecklist({ + profile: testProfile({ captions: { recommended: true, required: true } }), + aiAltered: false, + hasCaptions: false, + trackingUrl: "https://example.com/", + }); + expect(short).not.toContain("AI-generated"); + expect(short).not.toContain("age-restricted"); + expect(short).toContain("in the profile bio, not in the caption"); + expect(short).toContain("Captions are required"); + expect(short).toContain("at most 3 hashtags"); + expect(short.split("\n").filter((l) => l.startsWith("- [ ] "))).toHaveLength(4); + + const relaxed = uploadChecklist({ + profile: testProfile({ disclosure: { aiAlteredLabelRequired: false } }), + aiAltered: true, + hasCaptions: true, + trackingUrl: null, + }); + expect(relaxed).not.toContain("AI-generated"); + }); +}); diff --git a/packages/media/src/marketing.ts b/packages/media/src/marketing.ts index e63db11..2a0ee0c 100644 --- a/packages/media/src/marketing.ts +++ b/packages/media/src/marketing.ts @@ -6,15 +6,70 @@ import type { PlatformProfile, Utm, } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; -/** Append UTM parameters to an http(s) target. Rejects other schemes. */ -export function buildTrackingUrl(_targetUrl: string, _utm: Utm): string { - return notImplemented("buildTrackingUrl"); +const UTM_KEYS: Array<[keyof Utm, string]> = [ + ["source", "utm_source"], + ["medium", "utm_medium"], + ["campaign", "utm_campaign"], + ["content", "utm_content"], + ["term", "utm_term"], +]; + +/** + * Append UTM parameters to an http(s) target. Rejects other schemes. Existing + * query parameters keep their exact encoding; existing utm_* values for the + * keys being set are replaced, and the fragment stays last. + */ +export function buildTrackingUrl(targetUrl: string, utm: Utm): string { + let url: URL; + try { + url = new URL(targetUrl.trim()); + } catch { + throw new Error("tracking target is not a valid URL"); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("tracking target must use http or https"); + } + const setting = UTM_KEYS.filter(([key]) => utm[key]?.trim()); + const replaced = new Set(setting.map(([, name]) => name)); + const kept = url.search + .replace(/^\?/, "") + .split("&") + .filter((part) => { + if (!part) return false; + const rawKey = part.split("=")[0] ?? ""; + let key = rawKey; + try { + key = decodeURIComponent(rawKey.replace(/\+/g, " ")); + } catch { + // keep undecodable keys as they are + } + return !replaced.has(key); + }); + const added = setting.map( + ([key, name]) => `${name}=${encodeURIComponent((utm[key] as string).trim())}`, + ); + const query = [...kept, ...added].join("&"); + return `${url.origin}${url.pathname}${query ? `?${query}` : ""}${url.hash}`; } -export function normalizeHashtags(_tags: string[], _limit: number): string[] { - return notImplemented("normalizeHashtags"); +/** + * Normalize to "#tag": strip leading #, drop spaces and punctuation (letters, + * digits and _ stay, in any script), dedupe case-insensitively, and cap. + */ +export function normalizeHashtags(tags: string[], limit: number): string[] { + const seen = new Set(); + const out: string[] = []; + for (const raw of tags) { + const tag = raw.normalize("NFC").replace(/[^\p{L}\p{N}_]/gu, ""); + if (!tag) continue; + const key = tag.toLocaleLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + out.push(`#${tag}`); + if (out.length >= Math.max(0, limit)) break; + } + return out.slice(0, Math.max(0, limit)); } export interface ParsedMetricRow { @@ -28,39 +83,450 @@ export interface ParsedMetricRow { currency: string; } -/** RFC 4180 CSV parsing with the saved column mapping. */ +/** RFC 4180 records; quoted fields may hold delimiters, quotes ("") and newlines. */ +export function parseCsvRecords(csv: string, delimiter: string): string[][] { + const text = csv.replace(/^/, ""); + const records: string[][] = []; + let record: string[] = []; + let field = ""; + let quoted = false; + let i = 0; + while (i < text.length) { + const char = text[i]!; + if (quoted) { + if (char === '"') { + if (text[i + 1] === '"') { + field += '"'; + i += 2; + continue; + } + quoted = false; + i += 1; + continue; + } + field += char; + i += 1; + continue; + } + if (char === '"' && field.length === 0) { + quoted = true; + i += 1; + } else if (char === delimiter) { + record.push(field); + field = ""; + i += 1; + } else if (char === "\r" || char === "\n") { + record.push(field); + records.push(record); + record = []; + field = ""; + i += char === "\r" && text[i + 1] === "\n" ? 2 : 1; + } else { + field += char; + i += 1; + } + } + if (field.length > 0 || record.length > 0) { + record.push(field); + records.push(record); + } + return records; +} + +const CURRENCY_SYMBOLS: Record = { "€": "EUR", $: "USD", "£": "GBP", "¥": "JPY" }; + +/** Whole, non-negative count with optional thousands separators. */ +function parseCount(raw: string): number | null { + const value = raw.trim().replace(/[\s  ']/g, ""); + if (value === "") return 0; + if (/^\d+$/.test(value)) return Number(value); + if (/^\d{1,3}([.,]\d{3})+$/.test(value) && new Set(value.match(/[.,]/g)).size === 1) { + return Number(value.replace(/[.,]/g, "")); + } + return null; +} + +/** + * Money in major units to cents. With both "," and "." the later one is the + * decimal separator; a single separator followed by one or two digits is + * decimal, by exactly three digits after 1–999 it groups thousands ("1,234"), + * otherwise it is decimal ("0.995"). Extra decimals round half up. + */ +export function parseMoneyToCents(raw: string): { cents: number; currency: string | null } | null { + let value = raw.trim(); + let currency: string | null = null; + for (const [symbol, code] of Object.entries(CURRENCY_SYMBOLS)) { + if (value.includes(symbol)) { + currency = code; + value = value.replace(symbol, ""); + } + } + const code = value.match(/[A-Za-z]{3}/)?.[0]; + if (code) { + currency = code.toUpperCase(); + value = value.replace(code, ""); + } + value = value.replace(/[\s  ']/g, ""); + if (value === "") return { cents: 0, currency }; + if (!/^\d[\d.,]*$/.test(value)) return null; + const lastComma = value.lastIndexOf(","); + const lastDot = value.lastIndexOf("."); + let integer = value; + let fraction = ""; + if (lastComma >= 0 && lastDot >= 0) { + const decimal = Math.max(lastComma, lastDot); + integer = value.slice(0, decimal); + fraction = value.slice(decimal + 1); + } else if (lastComma >= 0 || lastDot >= 0) { + const separator = lastComma >= 0 ? "," : "."; + const parts = value.split(separator); + const tail = parts.at(-1)!; + const decimal = + parts.length === 2 && + tail.length >= 1 && + (tail.length <= 2 || !/^[1-9]\d{0,2}$/.test(parts[0]!)); + if (decimal) { + integer = parts[0]!; + fraction = tail; + } else if (parts.slice(1).every((part) => part.length === 3)) { + integer = parts.join(""); + } else { + return null; + } + } + integer = integer.replace(/[.,]/g, ""); + if (!/^\d+$/.test(integer) || !/^\d*$/.test(fraction)) return null; + const twoDigits = `${fraction}00`.slice(0, 2); + const roundUp = fraction.length > 2 && Number(fraction[2]) >= 5 ? 1 : 0; + return { cents: Number(integer) * 100 + Number(twoDigits) + roundUp, currency }; +} + +/** ISO dates and times, YYYY/MM/DD, and DD.MM.YYYY. Ambiguous slash dates are rejected. */ +function parseDate(raw: string): Date | null { + const value = raw.trim(); + let match = value.match(/^(\d{4})[-/](\d{2})[-/](\d{2})$/); + if (match) return utcDate(Number(match[1]), Number(match[2]), Number(match[3])); + match = value.match(/^(\d{1,2})\.(\d{1,2})\.(\d{4})$/); + if (match) return utcDate(Number(match[3]), Number(match[2]), Number(match[1])); + if (/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(value)) { + const date = new Date(value.replace(" ", "T")); + return Number.isNaN(date.getTime()) ? null : date; + } + return null; +} + +function utcDate(year: number, month: number, day: number): Date | null { + const date = new Date(Date.UTC(year, month - 1, day)); + return date.getUTCFullYear() === year && + date.getUTCMonth() === month - 1 && + date.getUTCDate() === day + ? date + : null; +} + +/** RFC 4180 CSV parsing with the saved column mapping. Rows count from the header as row 1. */ export function parseMetricsCsv( - _csv: string, - _mapping: MetricCsvMapping, + csv: string, + mapping: MetricCsvMapping, ): { rows: ParsedMetricRow[]; skipped: Array<{ row: number; reason: string }> } { - return notImplemented("parseMetricsCsv"); + const records = parseCsvRecords(csv, mapping.delimiter ?? ","); + const header = records[0]; + const rows: ParsedMetricRow[] = []; + const skipped: Array<{ row: number; reason: string }> = []; + if (!header) return { rows, skipped: [{ row: 1, reason: "file is empty" }] }; + const names = header.map((name) => name.trim()); + const column = (name: string | undefined): number => { + if (!name) return -1; + const exact = names.indexOf(name.trim()); + if (exact >= 0) return exact; + const lower = name.trim().toLowerCase(); + return names.findIndex((n) => n.toLowerCase() === lower); + }; + const columns = mapping.columns; + const index = { + publication: column(columns.publication), + observedAt: column(columns.observedAt), + views: column(columns.views), + clicks: column(columns.clicks), + conversions: column(columns.conversions), + revenue: column(columns.revenue), + currency: column(columns.currency), + }; + const missing = (Object.keys(index) as Array).filter( + (key) => columns[key] !== undefined && index[key] < 0, + ); + if (missing.length > 0) { + return { + rows, + skipped: [ + { row: 1, reason: `missing columns: ${missing.map((k) => columns[k]).join(", ")}` }, + ], + }; + } + const defaultCurrency = (mapping.defaultCurrency ?? "EUR").toUpperCase(); + + for (const [offset, record] of records.slice(1).entries()) { + const row = offset + 2; + if (record.every((field) => field.trim() === "")) continue; + const cell = (i: number) => (i >= 0 ? (record[i] ?? "") : ""); + const publicationKey = cell(index.publication).trim(); + if (!publicationKey) { + skipped.push({ row, reason: "publication is empty" }); + continue; + } + const observedAt = parseDate(cell(index.observedAt)); + if (!observedAt) { + skipped.push({ row, reason: "date is not recognized" }); + continue; + } + const counts = { + views: parseCount(cell(index.views)), + clicks: parseCount(cell(index.clicks)), + conversions: parseCount(cell(index.conversions)), + }; + const invalid = (Object.keys(counts) as Array).find( + (key) => counts[key] === null, + ); + if (invalid) { + skipped.push({ row, reason: `${invalid} is not a whole number` }); + continue; + } + let revenueCents = 0; + let currency: string | null = null; + const revenueRaw = cell(index.revenue); + if (revenueRaw.trim()) { + if (/^\s*-/.test(revenueRaw)) { + skipped.push({ row, reason: "revenue is negative" }); + continue; + } + if (mapping.revenueUnit === "cents") { + const value = parseCount(revenueRaw); + if (value === null) { + skipped.push({ row, reason: "revenue is not a whole number" }); + continue; + } + revenueCents = value; + } else { + const money = parseMoneyToCents(revenueRaw); + if (!money) { + skipped.push({ row, reason: "revenue is not a number" }); + continue; + } + revenueCents = money.cents; + currency = money.currency; + } + } + const currencyCell = cell(index.currency).trim().toUpperCase(); + if (currencyCell && !/^[A-Z]{3}$/.test(currencyCell)) { + skipped.push({ row, reason: "currency is not a 3-letter code" }); + continue; + } + rows.push({ + row, + publicationKey, + observedAt, + views: counts.views ?? 0, + clicks: counts.clicks ?? 0, + conversions: counts.conversions ?? 0, + revenueCents, + currency: currencyCell || currency || defaultCurrency, + }); + } + return { rows, skipped }; } -/** Whether the brand intro fits before the hook window; shorts put the hook first. */ -export function introFits(_profile: PlatformProfile, _introMs: number): boolean { - return notImplemented("introFits"); +const SHORT_FORM_MAX_MS = 180_000; +const MAX_INTRO_MS = 4000; + +/** Vertical profiles capped at three minutes: the hook must come before any branding. */ +export function isShortForm(profile: PlatformProfile): boolean { + return ( + profile.video.aspectRatios.includes("9:16") && profile.duration.hardMaxMs <= SHORT_FORM_MAX_MS + ); } -/** Deterministic intro proposals from the brand kit, tone, and available templates. */ +/** + * Whether the brand intro may open the video. Short-form vertical profiles + * always put the hook first (the brand appears as watermark or end card). + * Elsewhere the intro may take at most 4 s and a tenth of the maximum length. + */ +export function introFits(profile: PlatformProfile, introMs: number): boolean { + if (introMs <= 0) return true; + if (isShortForm(profile)) return false; + return introMs <= Math.min(MAX_INTRO_MS, profile.duration.hardMaxMs / 10); +} + +const TONE_STYLES: Array<{ pattern: RegExp; keywords: string[] }> = [ + { + pattern: /energ|bold|dynamic|loud|wild|sport|fast|laut|schnell|kraft/i, + keywords: ["glitch", "energy", "kinetic"], + }, + { + pattern: /elegan|calm|luxur|soft|premium|ruhig|edel|sanft|warm/i, + keywords: ["elegant", "fade", "minimal"], + }, + { + pattern: /minimal|clean|simple|schlicht|klar|modern/i, + keywords: ["minimal", "logo", "fade"], + }, + { pattern: /playful|fun|witzig|verspielt|bunt/i, keywords: ["kinetic", "typo", "glitch"] }, +]; + +const PARAM_SOURCES: Record string | undefined> = { + primarycolor: (kit) => kit.colors.primary, + secondarycolor: (kit) => kit.colors.secondary, + accentcolor: (kit) => kit.colors.accent, + textcolor: (kit) => kit.colors.text, + backgroundcolor: (kit) => kit.colors.background, + brandname: (kit) => kit.name, + title: (kit) => kit.name, + name: (kit) => kit.name, + logoartifactid: (kit) => kit.logoArtifactIds[0], + logo: (kit) => kit.logoArtifactIds[0], + fontfamily: (kit) => (kit.fonts.find((f) => f.role === "heading") ?? kit.fonts[0])?.family, + soundartifactid: (kit) => kit.musicBeds.find((bed) => bed.commercialUse)?.artifactId, + sound: (kit) => kit.musicBeds.find((bed) => bed.commercialUse)?.artifactId, +}; + +const VARIANT_DURATIONS = [2500, 2000, 3000, 3500, 1500, 4000]; + +/** + * Deterministic intro proposals from the brand kit, tone, and available + * templates: intro templates are ranked by how well their id and name match + * the tone's style keywords (then by id), each proposal uses a distinct + * template while there are enough, and durations stay within 1.5–4 s and the + * template's own limits. Parameters are filled from the kit by name. + */ export function proposeIntroVariants( - _kit: BrandKit, - _templates: MotionTemplateInfo[], - _count: number, + kit: BrandKit, + templates: MotionTemplateInfo[], + count: number, ): Array<{ template: MotionTemplateRef; rationale: string }> { - return notImplemented("proposeIntroVariants"); + const intros = templates + .filter((t) => t.kind === "intro") + .sort((a, b) => a.id.localeCompare(b.id)); + if (intros.length === 0 || count <= 0) return []; + const tone = `${kit.tone ?? ""} ${kit.audience ?? ""}`; + const keywords = TONE_STYLES.filter((style) => style.pattern.test(tone)).flatMap( + (style) => style.keywords, + ); + const affinity = (template: MotionTemplateInfo) => { + const haystack = `${template.id} ${template.name}`.toLowerCase(); + return keywords.reduce( + (score, keyword, index) => score + (haystack.includes(keyword) ? keywords.length - index : 0), + 0, + ); + }; + const ranked = [...intros].sort((a, b) => affinity(b) - affinity(a) || a.id.localeCompare(b.id)); + + return Array.from({ length: count }, (_, index) => { + const template = ranked[index % ranked.length]!; + const round = Math.floor(index / ranked.length); + const wanted = VARIANT_DURATIONS[(index + round) % VARIANT_DURATIONS.length]!; + const durationMs = Math.min( + Math.max(wanted, 1500, template.minDurationMs), + Math.min(4000, template.maxDurationMs), + ); + const params: Record = {}; + for (const param of template.params) { + const source = PARAM_SOURCES[param.name.toLowerCase().replace(/[^a-z]/g, "")]; + const value = source?.(kit); + if (value !== undefined) params[param.name] = value.slice(0, 200); + } + const matched = affinity(template) > 0 && kit.tone ? ` matches the tone "${kit.tone}"` : ""; + return { + template: { templateId: template.id, params, durationMs: Math.max(0, durationMs) }, + rationale: `${template.name}${matched}, ${(durationMs / 1000).toFixed(1)} s`, + }; + }); } +function luminance(hex: string): number { + const channel = (offset: number) => { + const value = Number.parseInt(hex.slice(offset, offset + 2), 16) / 255; + return value <= 0.03928 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4; + }; + return 0.2126 * channel(1) + 0.7152 * channel(3) + 0.0722 * channel(5); +} + +/** WCAG contrast ratio of two #RRGGBB colours. */ +export function contrastRatio(a: string, b: string): number { + const [light, dark] = [luminance(a), luminance(b)].sort((x, y) => y - x) as [number, number]; + return (light + 0.05) / (dark + 0.05); +} + +const MIN_CONTRAST = 4.5; + /** Structural brand-kit checks beyond the schema (contrast, duplicate presets). */ -export function validateBrandKit(_kit: BrandKit): string[] { - return notImplemented("validateBrandKit"); +export function validateBrandKit(kit: BrandKit): string[] { + const warnings: string[] = []; + const ratio = contrastRatio(kit.colors.text, kit.colors.background); + if (ratio < MIN_CONTRAST) { + warnings.push(`text on background has contrast ${ratio.toFixed(1)}:1, below ${MIN_CONTRAST}:1`); + } + const seen = new Set(); + for (const preset of kit.captionPresets) { + if (seen.has(preset.id)) warnings.push(`caption preset id "${preset.id}" is used twice`); + seen.add(preset.id); + if (preset.backgroundColor) { + const presetRatio = contrastRatio(preset.textColor, preset.backgroundColor); + if (presetRatio < MIN_CONTRAST) { + warnings.push( + `caption preset "${preset.id}" has contrast ${presetRatio.toFixed(1)}:1, below ${MIN_CONTRAST}:1`, + ); + } + } + } + for (const bed of kit.musicBeds) { + if (!bed.commercialUse) + warnings.push(`music bed "${bed.title}" is not licensed for commercial use`); + } + for (const font of kit.fonts) { + if (!font.license?.trim()) warnings.push(`font "${font.family}" has no license note`); + } + if (kit.watermark && !kit.watermark.artifactId) warnings.push("watermark has no image"); + return warnings; } /** Upload checklist markdown for an export bundle. */ -export function uploadChecklist(_input: { +export function uploadChecklist(input: { profile: PlatformProfile; aiAltered: boolean; hasCaptions: boolean; trackingUrl: string | null; }): string { - return notImplemented("uploadChecklist"); + const { profile } = input; + const items: string[] = ["Upload the video file and choose the poster as cover image."]; + if (input.aiAltered && profile.disclosure.aiAlteredLabelRequired) { + items.push( + "Set the platform's AI-generated or altered content label: this video contains AI alterations.", + ); + } + if (profile.disclosure.ageRestrictedFlag) items.push("Mark the upload as age-restricted."); + if (profile.disclosure.paidPartnershipLabel) { + items.push("Turn on the paid partnership label if this post is sponsored."); + } + if (input.trackingUrl) { + const link = `<${input.trackingUrl}>`; + if (profile.text.linkPolicy === "allowed") items.push(`Add the tracking link: ${link}`); + else if (profile.text.linkPolicy === "bio_only") { + items.push(`Put the tracking link in the profile bio, not in the caption: ${link}`); + } else items.push("Do not add links: this platform does not allow them."); + } else if (profile.text.linkPolicy === "forbidden") { + items.push("Do not add links: this platform does not allow them."); + } + if (input.hasCaptions) { + items.push("Upload the SRT or VTT subtitles, or confirm the burned-in captions are readable."); + } else if (profile.captions.required) { + items.push("Captions are required on this platform: add them before publishing."); + } + const limits = [ + profile.text.titleMaxChars ? `title up to ${profile.text.titleMaxChars} characters` : null, + `caption up to ${profile.text.captionMaxChars} characters`, + `at most ${profile.text.hashtagLimit} hashtags`, + ].filter(Boolean); + items.push(`Paste the caption from caption.txt (${limits.join(", ")}).`); + return `# Upload checklist: ${profile.name.replace(/[\r\n]+/g, " ")}\n\n${items + .map((item) => `- [ ] ${item}`) + .join("\n")}\n`; } diff --git a/packages/media/src/not-implemented.ts b/packages/media/src/not-implemented.ts deleted file mode 100644 index 0c4ab30..0000000 --- a/packages/media/src/not-implemented.ts +++ /dev/null @@ -1,4 +0,0 @@ -/** Phase A scaffold marker. No call site may remain once Media Studio ships. */ -export function notImplemented(name: string): never { - throw new Error(`not implemented: ${name}`); -} diff --git a/packages/media/src/teaser.test.ts b/packages/media/src/teaser.test.ts new file mode 100644 index 0000000..e2f218c --- /dev/null +++ b/packages/media/src/teaser.test.ts @@ -0,0 +1,334 @@ +import { + MediaProjectGoalSchema, + TEASER_FEATURES, + TEASER_LEARNING_MIN_SAMPLES, +} from "@rakazo/contracts"; +import { describe, expect, it } from "vitest"; +import { validateEdl } from "./edl.js"; +import type { ScoredSegmentInput, SegmentFeatures } from "./teaser.js"; +import { + autoPlanEdl, + defaultTeaserWeights, + fitTeaserWeights, + rankThumbnailFrames, + scoreSegment, + selectTeaserSegments, + speechHookScore, +} from "./teaser.js"; +import { testContext, testKit, testPaidProfile, testProfile } from "./test-fixtures.js"; + +function segment( + id: string, + assetId: string, + startMs: number, + endMs: number, + features: SegmentFeatures, + extra: Partial = {}, +): ScoredSegmentInput { + return { + id, + assetId, + startMs, + endMs, + capturedAt: null, + features, + paidOnly: false, + transcript: null, + ...extra, + }; +} + +/** One 30 s asset building up to a climax at 20–24 s, plus a second asset. */ +const segments: ScoredSegmentInput[] = [ + segment("s1", "asset-a", 0, 4000, { motion: 0.2, sharpness: 0.8 }), + segment( + "s2", + "asset-a", + 4000, + 8000, + { motion: 0.3, audioPeak: 0.3 }, + { transcript: "Wait, you won't believe what happens next?" }, + ), + segment("s3", "asset-a", 8000, 12_000, { motion: 0.5, faces: 0.6 }), + segment("s4", "asset-a", 12_000, 16_000, { motion: 0.6, audioPeak: 0.5 }), + segment("s5", "asset-a", 16_000, 20_000, { motion: 0.7, novelty: 0.6 }, { paidOnly: true }), + segment("s6", "asset-a", 20_000, 24_000, { + motion: 1, + audioPeak: 1, + faces: 1, + novelty: 1, + aesthetics: 1, + }), + segment("s7", "asset-a", 24_000, 28_000, { motion: 0.4 }), + segment("s8", "asset-b", 0, 3000, { motion: 0.5, aesthetics: 0.7 }), +]; + +describe("weights and scores", () => { + it("defines every feature for every goal", () => { + for (const goal of MediaProjectGoalSchema.options) { + const weights = defaultTeaserWeights(goal); + expect(Object.keys(weights).sort()).toEqual([...TEASER_FEATURES].sort()); + expect(Object.values(weights).reduce((a, b) => a + b, 0)).toBeCloseTo(1, 6); + } + expect(defaultTeaserWeights("teaser").speechHook).toBeGreaterThan( + defaultTeaserWeights("highlight").speechHook, + ); + }); + + it("scores as a weighted mean of clamped features", () => { + const weights = defaultTeaserWeights("teaser"); + expect(scoreSegment({}, weights)).toBe(0); + const all = Object.fromEntries(TEASER_FEATURES.map((f) => [f, 1])); + expect(scoreSegment(all, weights)).toBeCloseTo(1, 6); + expect(scoreSegment({ motion: 5 }, weights)).toBeCloseTo(weights.motion, 6); + expect(scoreSegment({ motion: Number.NaN }, weights)).toBe(0); + expect(scoreSegment({ motion: 1 }, { ...weights, motion: 0 })).toBe(0); + }); + + it("detects German and English speech hooks", () => { + expect(speechHookScore("")).toBe(0); + expect(speechHookScore("the weather was fine")).toBe(0); + expect(speechHookScore("Wait, you won't believe this!", "en")).toBeGreaterThan(0.6); + expect(speechHookScore("Warum macht das niemand?", "de")).toBeGreaterThan(0.5); + expect(speechHookScore("Schau mal, 3 Tricks!", "de-DE")).toBeGreaterThan(0.6); + expect(speechHookScore("Here are 5 tips")).toBeGreaterThan(speechHookScore("here are tips")); + expect(speechHookScore("Wie geht das? Pass auf! 10 Sekunden, du!")).toBe(1); + }); +}); + +describe("selectTeaserSegments", () => { + const profile = testProfile(); + const select = (overrides: Partial[0]> = {}) => + selectTeaserSegments({ + segments, + profile, + goal: "teaser", + targetMs: 15_000, + weights: defaultTeaserWeights("teaser"), + endCardMs: 2000, + ...overrides, + }); + + it("opens with the hook inside the hook window", () => { + const selection = select(); + expect(selection.hookSegmentId).toBe("s2"); + expect(selection.clips[0]).toMatchObject({ segmentId: "s2", inMs: 4000 }); + expect(selection.notes[0]).toContain("hook window"); + }); + + it("never uses paid-only segments and ends before the payoff", () => { + const selection = select(); + const ids = selection.clips.map((c) => c.segmentId); + expect(ids).not.toContain("s5"); + expect(ids).not.toContain("s6"); + expect(ids).not.toContain("s7"); + const last = selection.clips.at(-1)!; + expect(last.assetId).toBe("asset-a"); + expect(last.outMs).toBeLessThanOrEqual(20_000); + expect(selection.notes.some((n) => n.startsWith("open loop"))).toBe(true); + }); + + it("builds tension and fits the duration bounds minus the end card", () => { + const selection = select(); + const total = selection.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0); + expect(total).toBeLessThanOrEqual(15_000 - 2000); + expect(total).toBeGreaterThanOrEqual(profile.duration.hardMinMs - 2000); + for (const clip of selection.clips) expect(clip.outMs).toBeGreaterThan(clip.inMs); + const tight = select({ targetMs: 5000 }); + expect(tight.clips.reduce((sum, c) => sum + (c.outMs - c.inMs), 0)).toBeLessThanOrEqual(3000); + }); + + it("returns nothing when only paid-only material exists", () => { + const paid = segments.map((s) => ({ ...s, paidOnly: true })); + expect(select({ segments: paid })).toEqual({ + clips: [], + hookSegmentId: null, + notes: ["no usable segments"], + }); + }); + + it("is deterministic", () => { + expect(select({ segments: [...segments].reverse() })).toEqual(select()); + }); +}); + +describe("fitTeaserWeights", () => { + const prior = defaultTeaserWeights("teaser"); + const samples = (count: number) => + Array.from({ length: count }, (_, i) => { + const hook = (i % 10) / 9; + const motion = ((i * 7) % 10) / 9; + return { + features: { speechHook: hook, motion }, + views: 1000, + conversions: Math.round(1000 * (0.01 + 0.2 * hook)), + }; + }); + + it("stays off below the minimum sample count", () => { + expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES - 1), prior)).toBeNull(); + const withoutViews = [ + ...samples(TEASER_LEARNING_MIN_SAMPLES - 1), + { features: {}, views: 0, conversions: 0 }, + ]; + expect(fitTeaserWeights(withoutViews, prior)).toBeNull(); + }); + + it("shifts weight toward features that drive conversion, deterministically", () => { + const learned = fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)!; + expect(learned).not.toBeNull(); + expect(learned.speechHook).toBeGreaterThan(prior.speechHook); + expect(learned.motion).toBeLessThan(prior.motion); + expect(fitTeaserWeights(samples(TEASER_LEARNING_MIN_SAMPLES), prior)).toEqual(learned); + const more = fitTeaserWeights(samples(200), prior)!; + expect(more.speechHook).toBeGreaterThan(learned.speechHook); + }); +}); + +describe("autoPlanEdl", () => { + const assets = new Map([ + ["asset-a", { durationMs: 30_000, width: 1920, height: 1080, hasSpeech: true }], + ["asset-b", { durationMs: 8000, width: 1080, height: 1920, hasSpeech: false }], + ]); + const context = testContext({ + assets: new Map([ + ["asset-a", { kind: "video", durationMs: 30_000, width: 1920, height: 1080 }], + ["asset-b", { kind: "video", durationMs: 8000, width: 1080, height: 1920 }], + ]), + paidOnly: [{ assetId: "asset-a", startMs: 16_000, endMs: 20_000 }], + templateIds: new Set(["intro-minimal", "end-card-a"]), + goal: "teaser", + }); + + it("plans a valid teaser with hook first, captions, watermark, CTA and redactions", () => { + const kit = testKit({ + intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 }, + endCard: { templateId: "end-card-a", params: { style: "a" }, durationMs: 2000 }, + lutArtifactId: "lut-1", + }); + const edl = autoPlanEdl({ + goal: "teaser", + profile: testProfile(), + brandKit: kit, + segments, + weights: defaultTeaserWeights("teaser"), + assets, + people: [ + { id: "person-1", anonymizeByDefault: "synthetic_identity" }, + { id: "person-2", anonymizeByDefault: "none" }, + ], + marks: [ + { id: "mark-1", treatment: "inpaint", assetId: "asset-a", personId: null }, + { id: "mark-2", treatment: "blur", assetId: "asset-z", personId: null }, + { id: "mark-3", treatment: "none", assetId: "asset-a", personId: null }, + ], + ctaText: "Full video: link in bio", + }); + const result = validateEdl(edl, { ...context, brandKit: kit }); + expect(result.ok ? [] : result.errors).toEqual([]); + // Short vertical profile: no intro, the hook comes first. + expect(edl.intro).toBeUndefined(); + expect(edl.tracks.video[0]!.segmentId).toBe("s2"); + expect(edl.tracks.video[0]!.reframe).toBe("auto"); + expect(edl.tracks.video[0]!.color?.useBrandLut).toBe(true); + expect(edl.outro).toMatchObject({ + templateId: "end-card-a", + params: { style: "a", cta: "Full video: link in bio" }, + }); + expect(edl.captions).toMatchObject({ mode: "both", presetId: "clean", maxCharsPerLine: 24 }); + expect(edl.overlays.map((o) => o.kind)).toEqual(["watermark"]); + expect(edl.redactions.faces.map((f) => [f.target, f.mode])).toEqual([ + [{ personId: "person-1" }, "synthetic_identity"], + [{ unassignedFaces: true }, "blur"], + ]); + expect(edl.redactions.marks).toEqual([{ markId: "mark-1", treatment: "inpaint" }]); + expect(edl.metadata).toMatchObject({ + aiAltered: true, + aiAlterations: ["synthetic_identity", "inpaint"], + }); + }); + + it("uses a CTA text card without an end-card template and keeps the intro on long-form profiles", () => { + const profile = testPaidProfile(); + const kit = testKit({ + intro: { templateId: "intro-minimal", params: {}, durationMs: 2000 }, + watermark: undefined, + }); + const teaser = autoPlanEdl({ + goal: "teaser", + profile, + brandKit: kit, + segments, + weights: defaultTeaserWeights("teaser"), + assets, + people: [], + marks: [], + ctaText: "Watch the full video", + }); + expect(teaser.intro?.templateId).toBe("intro-minimal"); + expect(teaser.overlays).toEqual([ + expect.objectContaining({ kind: "cta_end_card", text: "Watch the full video" }), + ]); + expect(teaser.redactions.faces).toEqual([]); + expect(teaser.metadata.aiAltered).toBe(false); + const result = validateEdl(teaser, { ...context, profile, brandKit: kit }); + expect(result.ok ? [] : result.errors).toEqual([]); + }); + + it("plans a full release in capture order including paid-only material on paid profiles", () => { + const profile = testPaidProfile(); + const edl = autoPlanEdl({ + goal: "full_release", + profile, + brandKit: null, + segments, + weights: defaultTeaserWeights("full_release"), + assets, + people: [], + marks: [], + ctaText: null, + }); + expect(edl.tracks.video.map((c) => c.segmentId)).toEqual([ + "s1", + "s2", + "s3", + "s4", + "s5", + "s6", + "s7", + "s8", + ]); + expect(edl.tracks.video.at(-1)!.reframe).toBe("auto"); + expect(edl.tracks.video[0]!.reframe).toBe("none"); + const result = validateEdl(edl, { ...context, profile, brandKit: null, goal: "full_release" }); + expect(result.ok ? [] : result.errors).toEqual([]); + expect(() => + autoPlanEdl({ + goal: "teaser", + profile, + brandKit: null, + segments: [], + weights: defaultTeaserWeights("teaser"), + assets, + people: [], + marks: [], + ctaText: null, + }), + ).toThrow(/no usable segments/); + }); +}); + +describe("rankThumbnailFrames", () => { + it("returns the best frames spread over the video", () => { + const candidates = [ + { tMs: 0, sharpness: 0.5, face: 0, thirds: 0.5, contrast: 0.5 }, + { tMs: 1000, sharpness: 1, face: 1, thirds: 1, contrast: 1 }, + { tMs: 1200, sharpness: 1, face: 1, thirds: 0.9, contrast: 1 }, + { tMs: 5000, sharpness: 0.8, face: 0.5, thirds: 0.5, contrast: 0.5 }, + { tMs: 9000, sharpness: 0.7, face: 0.5, thirds: 0.5, contrast: 0.5 }, + ]; + expect(rankThumbnailFrames(candidates, 3)).toEqual([1000, 5000, 9000]); + expect(rankThumbnailFrames(candidates.slice(1, 3), 3)).toEqual([1000, 1200]); + expect(rankThumbnailFrames(candidates, 0)).toEqual([]); + }); +}); diff --git a/packages/media/src/teaser.ts b/packages/media/src/teaser.ts index d52418f..e8c7818 100644 --- a/packages/media/src/teaser.ts +++ b/packages/media/src/teaser.ts @@ -1,4 +1,6 @@ import type { + AiAlteration, + AnonymizeMode, BrandKit, Edl, MediaProjectGoal, @@ -6,12 +8,88 @@ import type { TeaserFeature, TeaserWeights, } from "@rakazo/contracts"; -import { notImplemented } from "./not-implemented.js"; +import { + AnonymizeModeSchema, + EdlSchema, + TEASER_FEATURES, + TEASER_LEARNING_MIN_SAMPLES, +} from "@rakazo/contracts"; +import { introFits } from "./marketing.js"; export type SegmentFeatures = Partial>; -export function defaultTeaserWeights(_goal: MediaProjectGoal): TeaserWeights { - return notImplemented("defaultTeaserWeights"); +const GOAL_WEIGHTS: Record> = { + teaser: { + motion: 0.14, + sharpness: 0.06, + faces: 0.08, + gaze: 0.08, + audioPeak: 0.14, + speechHook: 0.24, + novelty: 0.12, + beatProximity: 0.06, + aesthetics: 0.08, + }, + trailer: { + motion: 0.16, + sharpness: 0.08, + faces: 0.08, + gaze: 0.06, + audioPeak: 0.14, + speechHook: 0.12, + novelty: 0.12, + beatProximity: 0.12, + aesthetics: 0.12, + }, + highlight: { + motion: 0.2, + sharpness: 0.08, + faces: 0.12, + gaze: 0.06, + audioPeak: 0.18, + speechHook: 0.08, + novelty: 0.1, + beatProximity: 0.08, + aesthetics: 0.1, + }, + story: { + motion: 0.08, + sharpness: 0.08, + faces: 0.16, + gaze: 0.12, + audioPeak: 0.06, + speechHook: 0.22, + novelty: 0.12, + beatProximity: 0.04, + aesthetics: 0.12, + }, + full_release: { + motion: 0.1, + sharpness: 0.14, + faces: 0.1, + gaze: 0.08, + audioPeak: 0.08, + speechHook: 0.1, + novelty: 0.12, + beatProximity: 0.06, + aesthetics: 0.22, + }, + custom: { + motion: 0.12, + sharpness: 0.1, + faces: 0.1, + gaze: 0.1, + audioPeak: 0.12, + speechHook: 0.14, + novelty: 0.12, + beatProximity: 0.08, + aesthetics: 0.12, + }, +}; + +/** Default per-goal weights over every teaser feature; each set sums to 1. */ +export function defaultTeaserWeights(goal: MediaProjectGoal): TeaserWeights { + return { ...GOAL_WEIGHTS[goal] }; } export interface ScoredSegmentInput { @@ -25,13 +103,75 @@ export interface ScoredSegmentInput { transcript: string | null; } -export function scoreSegment(_features: SegmentFeatures, _weights: TeaserWeights): number { - return notImplemented("scoreSegment"); +const clamp01 = (value: number) => (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0); + +/** Weighted mean of 0..1 features; missing features count as 0. */ +export function scoreSegment(features: SegmentFeatures, weights: TeaserWeights): number { + let total = 0; + let norm = 0; + for (const feature of TEASER_FEATURES) { + const weight = weights[feature] ?? 0; + norm += Math.abs(weight); + total += weight * clamp01(features[feature] ?? 0); + } + return norm === 0 ? 0 : total / norm; } +const HOOK_LEXICON = { + en: { + questionWords: + /^(what|why|how|who|when|where|which|did|do|does|can|could|would|should|is|are|have|ever)\b/, + announcements: + /\b(wait|watch|look|listen|check this|you won'?t believe|here'?s|secret|how to|never|finally|the moment|let me show|guess|imagine|stop|nobody|everyone|biggest|best|worst|first time|new|today)\b/, + address: /\b(you|your)\b/, + numbers: /\b(one|two|three|five|ten|hundred|thousand|million|first|second|third|top)\b/, + }, + de: { + questionWords: + /^(was|warum|wieso|weshalb|wie|wer|wann|wo|welche[rsmn]?|hast|habt|kannst|könnt|willst|würdest|weißt|ist|sind)\b/, + announcements: + /\b(warte|wartet|schau|schaut|guck|hör zu|pass auf|achtung|du wirst nicht glauben|geheimnis|so geht|niemals|endlich|gleich|hier ist|zeig ich dir|stell dir vor|rate mal|niemand|alle|größte|beste|schlimmste|zum ersten mal|neu|heute)\b/, + address: /\b(du|dich|dir|dein|deine|ihr|euch|euer)\b/, + numbers: /\b(eins|zwei|drei|fünf|zehn|hundert|tausend|million|erste[rsn]?|zweite[rsn]?|top)\b/, + }, +} as const; + /** 0..1 speech-hook strength from a transcript excerpt (questions, exclamations, numbers, announcements). */ -export function speechHookScore(_text: string, _language?: string): number { - return notImplemented("speechHookScore"); +export function speechHookScore(text: string, language?: string): number { + const normalized = text.trim().toLocaleLowerCase(language); + if (!normalized) return 0; + const code = language?.slice(0, 2).toLowerCase(); + const lexicons = + code === "de" + ? [HOOK_LEXICON.de] + : code === "en" + ? [HOOK_LEXICON.en] + : Object.values(HOOK_LEXICON); + let score = 0; + if (normalized.includes("?")) score += 0.35; + if (normalized.includes("!")) score += 0.25; + const hasNumber = + /\d/.test(normalized) || lexicons.some((lexicon) => lexicon.numbers.test(normalized)); + if (hasNumber) score += 0.2; + if (lexicons.some((lexicon) => lexicon.announcements.test(normalized))) score += 0.3; + if (lexicons.some((lexicon) => lexicon.questionWords.test(normalized))) score += 0.15; + if (lexicons.some((lexicon) => lexicon.address.test(normalized))) score += 0.1; + return Math.min(1, Math.round(score * 1000) / 1000); +} + +function segmentFeatures(segment: ScoredSegmentInput): SegmentFeatures { + if (segment.features.speechHook !== undefined || !segment.transcript) return segment.features; + return { ...segment.features, speechHook: speechHookScore(segment.transcript) }; +} + +/** Strength of a segment as an opener: speech hooks, audio peaks, motion, eye contact. */ +function hookStrength(features: SegmentFeatures): number { + return ( + 0.45 * clamp01(features.speechHook ?? 0) + + 0.25 * clamp01(features.audioPeak ?? 0) + + 0.15 * clamp01(features.motion ?? 0) + + 0.15 * clamp01(features.gaze ?? 0) + ); } export interface TeaserSelection { @@ -41,12 +181,20 @@ export interface TeaserSelection { notes: string[]; } +const MIN_CLIP_MS = 1000; + /** * Pick clips for a goal and profile: a hook inside hookWindowMs, rising * tension, cut before the payoff (open loop), never paid-only segments, * total within the profile's duration bounds. + * + * The primary asset is the one holding the best segment overall; its payoff + * is its best segment (paid-only included, since that is usually the payoff). + * Nothing from the primary asset at or after the payoff is used. The hook + * opens the teaser, the rest follows in ascending score, and the lead-in to + * the payoff closes it, leaving `endCardMs` for the end card. */ -export function selectTeaserSegments(_input: { +export function selectTeaserSegments(input: { segments: ScoredSegmentInput[]; profile: PlatformProfile; goal: MediaProjectGoal; @@ -54,19 +202,227 @@ export function selectTeaserSegments(_input: { weights: TeaserWeights; endCardMs: number; }): TeaserSelection { - return notImplemented("selectTeaserSegments"); + const { profile } = input; + const notes: string[] = []; + const scored = input.segments + .filter((s) => s.endMs > s.startMs) + .map((segment) => { + const features = segmentFeatures(segment); + return { segment, features, score: scoreSegment(features, input.weights) }; + }) + .sort( + (a, b) => + b.score - a.score || + a.segment.assetId.localeCompare(b.segment.assetId) || + a.segment.startMs - b.segment.startMs || + a.segment.id.localeCompare(b.segment.id), + ); + const maxMs = Math.max( + 0, + Math.min(input.targetMs, profile.duration.hardMaxMs) - Math.max(0, input.endCardMs), + ); + const minMs = Math.max(0, profile.duration.hardMinMs - Math.max(0, input.endCardMs)); + const usable = scored.filter((s) => !s.segment.paidOnly); + if (usable.length === 0 || maxMs < MIN_CLIP_MS) { + return { clips: [], hookSegmentId: null, notes: ["no usable segments"] }; + } + + const primaryAsset = scored[0]!.segment.assetId; + const payoff = scored.find((s) => s.segment.assetId === primaryAsset)!; + const pool = usable.filter( + (s) => + s.segment.assetId !== primaryAsset || + (s.segment.id !== payoff.segment.id && s.segment.endMs <= payoff.segment.startMs), + ); + const openLoop = pool.length < usable.length || payoff.segment.paidOnly; + if (pool.length === 0) { + return { clips: [], hookSegmentId: null, notes: ["only the payoff is usable"] }; + } + + const hook = [...pool].sort( + (a, b) => + hookStrength(b.features) - hookStrength(a.features) || + b.score - a.score || + a.segment.id.localeCompare(b.segment.id), + )[0]!; + const hookWindow = profile.duration.hookWindowMs; + const clipCap = Math.max(MIN_CLIP_MS, Math.min(5000, Math.round(maxMs / 4))); + const lengthOf = (s: ScoredSegmentInput, cap: number) => Math.min(s.endMs - s.startMs, cap); + + // Lead-in: the last primary-asset segment before the payoff closes the teaser. + const leadIn = openLoop + ? pool + .filter((s) => s.segment.assetId === primaryAsset && s !== hook) + .sort((a, b) => b.segment.endMs - a.segment.endMs)[0] + : undefined; + + const chosen: typeof pool = [hook]; + let total = lengthOf(hook.segment, Math.max(clipCap, Math.min(hookWindow, maxMs))); + const reserve = leadIn ? lengthOf(leadIn.segment, clipCap) : 0; + for (const candidate of pool) { + if (candidate === hook || candidate === leadIn) continue; + const length = lengthOf(candidate.segment, clipCap); + if (length < Math.min(MIN_CLIP_MS, candidate.segment.endMs - candidate.segment.startMs)) + continue; + if (total + length + reserve > maxMs) continue; + chosen.push(candidate); + total += length; + } + if (leadIn && total + reserve <= maxMs) { + chosen.push(leadIn); + total += reserve; + } + + const middle = chosen + .slice(1) + .filter((s) => s !== leadIn) + .sort((a, b) => a.score - b.score || a.segment.id.localeCompare(b.segment.id)); + const ordered = [hook, ...middle, ...(chosen.includes(leadIn!) ? [leadIn!] : [])]; + + // Lengths: start from the cap, then grow toward the minimum duration if needed. + const lengths = ordered.map((s, i) => + lengthOf(s.segment, i === 0 ? Math.max(clipCap, Math.min(hookWindow, maxMs)) : clipCap), + ); + let sum = lengths.reduce((a, b) => a + b, 0); + for (let i = 0; i < ordered.length && sum < minMs; i++) { + const full = ordered[i]!.segment.endMs - ordered[i]!.segment.startMs; + const grow = Math.min(full - lengths[i]!, minMs - sum, maxMs - sum); + if (grow > 0) { + lengths[i]! += grow; + sum += grow; + } + } + while (sum > maxMs && lengths.length > 0) { + const over = sum - maxMs; + const last = lengths.length - 1; + if (lengths[last]! - over >= MIN_CLIP_MS) { + lengths[last]! -= over; + sum -= over; + } else { + sum -= lengths.pop()!; + ordered.pop(); + } + } + + const clips = ordered.map((s, i) => { + const length = lengths[i]!; + // The lead-in ends where the payoff would begin; other clips open at their start. + const isLeadIn = s === leadIn; + const outMs = isLeadIn ? s.segment.endMs : s.segment.startMs + length; + return { + segmentId: s.segment.id, + assetId: s.segment.assetId, + inMs: outMs - length, + outMs, + }; + }); + + notes.push( + `hook: segment ${hook.segment.id} opens inside the ${hookWindow} ms hook window`, + ...middle.map((s) => `build: segment ${s.segment.id} (score ${s.score.toFixed(2)})`), + ); + if (leadIn && ordered.includes(leadIn)) { + notes.push(`open loop: segment ${leadIn.segment.id} cuts before payoff ${payoff.segment.id}`); + } else if (openLoop) { + notes.push(`open loop: payoff ${payoff.segment.id} is left out`); + } + if (input.segments.some((s) => s.paidOnly)) notes.push("paid-only segments excluded"); + if (sum < minMs) notes.push(`only ${sum} ms of usable material; profile minimum is ${minMs} ms`); + return { clips, hookSegmentId: hook.segment.id, notes }; } -/** Regularized logistic regression of conversion on features; null below TEASER_LEARNING_MIN_SAMPLES. */ +const LEARNING_RATE = 0.5; +const LEARNING_ITERATIONS = 400; +const L2 = 0.05; +const SHRINKAGE = 50; + +/** + * Regularized logistic regression of conversion on features; null below + * TEASER_LEARNING_MIN_SAMPLES samples with views. + * + * Each sample contributes `conversions` successes out of `views` trials, + * weighted so every publication counts equally. Full-batch gradient descent + * from zero with a fixed iteration count keeps it deterministic. Positive + * coefficients of features that vary in the data redistribute those features' + * prior mass; features without variation keep their prior. The result is + * shrunk toward the prior by n / (n + 50) so a small sample cannot overturn it. + */ export function fitTeaserWeights( - _samples: Array<{ features: SegmentFeatures; views: number; conversions: number }>, - _prior: TeaserWeights, + samples: Array<{ features: SegmentFeatures; views: number; conversions: number }>, + prior: TeaserWeights, ): TeaserWeights | null { - return notImplemented("fitTeaserWeights"); + const usable = samples.filter((s) => s.views > 0 && s.conversions >= 0); + if (usable.length < TEASER_LEARNING_MIN_SAMPLES) return null; + const k = TEASER_FEATURES.length; + const xs = usable.map((s) => TEASER_FEATURES.map((f) => clamp01(s.features[f] ?? 0))); + const ys = usable.map((s) => Math.min(1, s.conversions / s.views)); + const beta = new Array(k).fill(0); + let intercept = 0; + for (let iteration = 0; iteration < LEARNING_ITERATIONS; iteration++) { + const gradient = new Array(k).fill(0); + let gradientIntercept = 0; + for (let i = 0; i < xs.length; i++) { + const x = xs[i]!; + let z = intercept; + for (let j = 0; j < k; j++) z += beta[j]! * x[j]!; + const error = 1 / (1 + Math.exp(-z)) - ys[i]!; + gradientIntercept += error; + for (let j = 0; j < k; j++) gradient[j]! += error * x[j]!; + } + intercept -= (LEARNING_RATE * gradientIntercept) / xs.length; + for (let j = 0; j < k; j++) { + beta[j]! -= LEARNING_RATE * (gradient[j]! / xs.length + L2 * beta[j]!); + } + } + // Only features that vary in the data are re-weighted; they share their prior mass. + const varies = TEASER_FEATURES.map((_, j) => xs.some((x) => x[j] !== xs[0]![j])); + const mass = TEASER_FEATURES.reduce( + (sum, f, j) => sum + (varies[j] ? Math.max(0, prior[f] ?? 0) : 0), + 0, + ); + const positive = beta.map((b, j) => (varies[j] ? Math.max(0, b) : 0)); + const positiveTotal = positive.reduce((a, b) => a + b, 0); + const alpha = usable.length / (usable.length + SHRINKAGE); + return Object.fromEntries( + TEASER_FEATURES.map((feature, j) => { + const base = prior[feature] ?? 0; + const learned = varies[j] && positiveTotal > 0 ? (positive[j]! / positiveTotal) * mass : base; + const value = (1 - alpha) * base + alpha * learned; + return [feature, Math.round(value * 1e6) / 1e6]; + }), + ) as TeaserWeights; } -/** Deterministic first-cut EDL for a project and profile; the agent refines it. */ -export function autoPlanEdl(_input: { +const CTA_CARD_MS = 2500; + +function pickTarget(profile: PlatformProfile, goal: MediaProjectGoal, availableMs: number): number { + const { hardMinMs, hardMaxMs, recommended } = profile.duration; + const options = [...recommended] + .filter((r) => r >= hardMinMs && r <= hardMaxMs) + .sort((a, b) => a - b); + let target: number; + if (goal === "full_release" || goal === "custom") { + target = Math.min(hardMaxMs, availableMs); + } else if (options.length > 0) { + target = options[Math.floor((options.length - 1) / 2)]!; + } else { + target = Math.min(hardMaxMs, 30_000); + } + return Math.max(hardMinMs, Math.min(hardMaxMs, target)); +} + +/** + * Deterministic first-cut EDL for a project and profile; the agent refines it. + * + * Teaser-style goals use selectTeaserSegments; full releases and custom + * projects take segments in order (paid-only ones only for non-social + * profiles). Reframing is "auto" whenever the source aspect differs from the + * output, captions follow the profile when the material has speech, the + * brand intro only plays where introFits allows it, and a teaser ends on the + * brand end card (or a CTA text card). Person and mark defaults become + * redactions, and the AI-alteration metadata follows from them. + */ +export function autoPlanEdl(input: { goal: MediaProjectGoal; profile: PlatformProfile; brandKit: BrandKit | null; @@ -81,18 +437,223 @@ export function autoPlanEdl(_input: { marks: Array<{ id: string; treatment: string; assetId: string | null; personId: string | null }>; ctaText: string | null; }): Edl { - return notImplemented("autoPlanEdl"); + const { goal, profile, brandKit: kit } = input; + const W = profile.video.width; + const H = profile.video.height; + const segments = input.segments.filter((s) => { + const asset = input.assets.get(s.assetId); + return ( + asset && s.endMs > s.startMs && (asset.durationMs === null || s.endMs <= asset.durationMs) + ); + }); + const available = segments.reduce((sum, s) => sum + (s.endMs - s.startMs), 0); + const targetMs = pickTarget(profile, goal, available); + + const intro = kit?.intro && introFits(profile, kit.intro.durationMs) ? kit.intro : undefined; + let outro = goal === "teaser" ? undefined : kit?.outro; + const ctaText = input.ctaText?.trim() || null; + let ctaCard = false; + if (goal === "teaser") { + if (kit?.endCard) { + outro = { + ...kit.endCard, + params: { ...kit.endCard.params, ...(ctaText ? { cta: ctaText.slice(0, 200) } : {}) }, + }; + } else if (ctaText) { + ctaCard = true; + } + } + const bookendsMs = (intro?.durationMs ?? 0) + (outro?.durationMs ?? 0); + const budget = Math.max(MIN_CLIP_MS, targetMs - bookendsMs); + + let picks: Array<{ segmentId: string; assetId: string; inMs: number; outMs: number }>; + if (goal === "full_release" || goal === "custom") { + const allowPaid = profile.kind !== "free_social"; + const ordered = segments + .filter((s) => allowPaid || !s.paidOnly) + .sort( + (a, b) => + (a.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) - + (b.capturedAt?.getTime() ?? Number.MAX_SAFE_INTEGER) || + a.assetId.localeCompare(b.assetId) || + a.startMs - b.startMs, + ); + picks = []; + let used = 0; + for (const s of ordered) { + const length = Math.min(s.endMs - s.startMs, budget - used); + if (length < MIN_CLIP_MS) break; + picks.push({ + segmentId: s.id, + assetId: s.assetId, + inMs: s.startMs, + outMs: s.startMs + length, + }); + used += length; + } + } else { + picks = selectTeaserSegments({ + segments, + profile, + goal, + targetMs: budget, + weights: input.weights, + endCardMs: 0, + }).clips; + } + if (picks.length === 0) throw new Error("no usable segments for this plan"); + + let atMs = 0; + const lut = Boolean(kit?.lutArtifactId); + const video = picks.map((pick, index) => { + const asset = input.assets.get(pick.assetId)!; + const sameAspect = + !asset.width || !asset.height || Math.abs(asset.width / asset.height / (W / H) - 1) <= 0.01; + const clip = { + id: `c${index + 1}`, + assetId: pick.assetId, + inMs: pick.inMs, + outMs: pick.outMs, + atMs, + speed: 1, + reframe: sameAspect ? ("none" as const) : ("auto" as const), + ...(lut ? { color: { useBrandLut: true } } : {}), + segmentId: pick.segmentId, + }; + atMs += pick.outMs - pick.inMs; + return clip; + }); + const mainMs = atMs; + const introLength = intro?.durationMs ?? 0; + const totalMs = introLength + mainMs + (outro?.durationMs ?? 0); + + const usedAssets = new Set(picks.map((p) => p.assetId)); + const hasSpeech = [...usedAssets].some((id) => input.assets.get(id)?.hasSpeech); + const preset = kit?.captionPresets[0]; + const captions = + (profile.captions.recommended || profile.captions.required) && hasSpeech + ? { + mode: profile.export.subtitles ? ("both" as const) : ("burned" as const), + ...(preset ? { presetId: preset.id } : {}), + maxCharsPerLine: W < H ? 24 : 32, + wordHighlight: preset?.template === "karaoke", + position: "lower" as const, + } + : undefined; + + const zones = profile.safeZones; + const overlays: Array> = []; + if (kit?.watermark) { + overlays.push({ id: "watermark", kind: "watermark", atMs: 0, durationMs: totalMs }); + } + if (ctaCard && ctaText) { + const cardMs = Math.min(CTA_CARD_MS, mainMs); + const left = zones.leftPct / 100 + 0.02; + const right = zones.rightPct / 100 + 0.02; + overlays.push({ + id: "cta", + kind: "cta_end_card", + atMs: introLength + mainMs - cardMs, + durationMs: cardMs, + text: ctaText.slice(0, 200), + box: { x: left, y: 0.42, w: Math.max(0.1, 1 - left - right), h: 0.12 }, + }); + } + + const protectedPeople = input.people.flatMap((person) => { + const mode = AnonymizeModeSchema.safeParse(person.anonymizeByDefault); + return mode.success && mode.data !== "none" ? [{ id: person.id, mode: mode.data }] : []; + }); + const faces: Array<{ + id: string; + target: { personId: string } | { unassignedFaces: true }; + mode: Exclude; + }> = protectedPeople.map((person, index) => ({ + id: `face-${index + 1}`, + target: { personId: person.id }, + mode: person.mode, + })); + if (faces.length > 0) { + // Unknown faces next to a protected person are blurred too; the gate blocks them otherwise. + faces.push({ id: "face-unassigned", target: { unassignedFaces: true }, mode: "blur" }); + } + const peopleIds = new Set(input.people.map((p) => p.id)); + const marks = input.marks.flatMap((mark) => { + if (mark.treatment === "none") return []; + const relevant = + (mark.assetId !== null && usedAssets.has(mark.assetId)) || + (mark.personId !== null && peopleIds.has(mark.personId)) || + (mark.assetId === null && mark.personId === null); + if (!relevant) return []; + const treatment = mark.treatment as "inpaint" | "blur" | "cover" | "reframe_out"; + return [{ markId: mark.id, treatment }]; + }); + + const aiAlterations: AiAlteration[] = []; + if (faces.some((f) => f.mode === "synthetic_identity")) aiAlterations.push("synthetic_identity"); + if (marks.some((m) => m.treatment === "inpaint")) aiAlterations.push("inpaint"); + + return EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: profile.id, + width: W, + height: H, + fps: profile.video.fps, + durationTargetMs: targetMs, + durationHardMinMs: profile.duration.hardMinMs, + durationHardMaxMs: profile.duration.hardMaxMs, + }, + tracks: { video, audio: [] }, + transitions: [], + ...(captions ? { captions } : {}), + overlays, + ...(intro ? { intro } : {}), + ...(outro ? { outro } : {}), + redactions: { faces, marks, regions: [] }, + metadata: { tags: [], aiAltered: aiAlterations.length > 0, aiAlterations }, + }); } +const THUMBNAIL_WEIGHTS = { sharpness: 0.4, face: 0.25, thirds: 0.2, contrast: 0.15 }; + +/** + * Best thumbnail frame times, spread out: candidates are ranked by sharpness, + * face presence, rule of thirds and contrast, and each pick must be at least + * a tenth of the candidate span (min 1 s) away from earlier picks while + * enough candidates remain. + */ export function rankThumbnailFrames( - _candidates: Array<{ + candidates: Array<{ tMs: number; sharpness: number; face: number; thirds: number; contrast: number; }>, - _count: number, + count: number, ): number[] { - return notImplemented("rankThumbnailFrames"); + if (count <= 0 || candidates.length === 0) return []; + const ranked = candidates + .map((c) => ({ + tMs: c.tMs, + score: + THUMBNAIL_WEIGHTS.sharpness * clamp01(c.sharpness) + + THUMBNAIL_WEIGHTS.face * clamp01(c.face) + + THUMBNAIL_WEIGHTS.thirds * clamp01(c.thirds) + + THUMBNAIL_WEIGHTS.contrast * clamp01(c.contrast), + })) + .sort((a, b) => b.score - a.score || a.tMs - b.tMs); + const times = candidates.map((c) => c.tMs); + const spacing = Math.max(1000, (Math.max(...times) - Math.min(...times)) / 10); + const picked: number[] = []; + for (const candidate of ranked) { + if (picked.length >= count) break; + if (picked.every((t) => Math.abs(t - candidate.tMs) >= spacing)) picked.push(candidate.tMs); + } + for (const candidate of ranked) { + if (picked.length >= count) break; + if (!picked.includes(candidate.tMs)) picked.push(candidate.tMs); + } + return picked; } diff --git a/packages/media/src/test-fixtures.ts b/packages/media/src/test-fixtures.ts new file mode 100644 index 0000000..105843f --- /dev/null +++ b/packages/media/src/test-fixtures.ts @@ -0,0 +1,135 @@ +import type { BrandKit, Edl, PlatformProfile } from "@rakazo/contracts"; +import { BrandKitSchema, EdlSchema, PlatformProfileSchema } from "@rakazo/contracts"; +import type { EdlContext } from "./edl.js"; + +/* Synthetic fixtures shared by the unit tests. No real people, places, or platforms. */ + +export function testProfile(overrides: Record = {}): PlatformProfile { + return PlatformProfileSchema.parse({ + id: "example-short-vertical", + name: "Example short vertical", + kind: "free_social", + reviewedAt: "2026-09-01", + video: { + aspectRatios: ["9:16"], + width: 1080, + height: 1920, + fps: 30, + codec: "h264", + maxBitrateKbps: 8000, + maxFileSizeMb: 500, + }, + duration: { hardMinMs: 1000, hardMaxMs: 60_000, recommended: [7000, 15_000, 30_000] }, + safeZones: { topPct: 10, bottomPct: 20, leftPct: 5, rightPct: 12 }, + captions: { recommended: true, maxLines: 2 }, + audio: {}, + text: { + titleMaxChars: 100, + captionMaxChars: 150, + hashtagLimit: 3, + linkPolicy: "bio_only", + bannedPatterns: [{ pattern: "free\\s+money", reason: "misleading claim" }], + }, + content: { thresholds: { explicit_nudity: 0.2, violence: 0.5 } }, + disclosure: { aiAlteredLabelRequired: true }, + consent: { requirePerformerConsent: false, requireAgeVerification: false }, + export: {}, + ...overrides, + }); +} + +export function testPaidProfile(): PlatformProfile { + return testProfile({ + id: "example-paid-video", + name: "Example paid video", + kind: "paid_video", + video: { + aspectRatios: ["16:9"], + width: 1920, + height: 1080, + fps: 30, + codec: "h264", + maxBitrateKbps: 12_000, + maxFileSizeMb: 4000, + }, + duration: { hardMinMs: 1000, hardMaxMs: 3_600_000, recommended: [] }, + safeZones: { topPct: 0, bottomPct: 0, leftPct: 0, rightPct: 0 }, + text: { captionMaxChars: 2000, hashtagLimit: 10, linkPolicy: "allowed" }, + consent: { requirePerformerConsent: true, requireAgeVerification: true }, + disclosure: { aiAlteredLabelRequired: true, ageRestrictedFlag: true }, + }); +} + +export function testKit(overrides: Partial = {}): BrandKit { + return BrandKitSchema.parse({ + name: "Studio Example", + colors: { + primary: "#112233", + secondary: "#445566", + accent: "#FFAA00", + text: "#FFFFFF", + background: "#000000", + }, + fonts: [{ family: "Example Sans", artifactId: "font-1", role: "caption", license: "OFL" }], + watermark: { artifactId: "logo-1", position: "top_right", opacity: 0.6, scale: 0.15 }, + captionPresets: [ + { + id: "clean", + name: "Clean", + template: "clean", + fontFamily: "Example Sans", + sizePct: 5, + textColor: "#FFFFFF", + }, + ], + musicBeds: [{ artifactId: "music-1", title: "Bed", license: "own", commercialUse: true }], + ...overrides, + }); +} + +/** Two 16:9 clips of a 10 s asset, 4 s total, on a vertical profile. */ +export function testEdl(overrides: Record = {}): Edl { + return EdlSchema.parse({ + edlVersion: 1, + output: { + profileId: "example-short-vertical", + width: 1080, + height: 1920, + fps: 30, + durationTargetMs: 4000, + durationHardMinMs: 1000, + durationHardMaxMs: 60_000, + }, + tracks: { + video: [ + { id: "c1", assetId: "asset-a", inMs: 0, outMs: 2000, atMs: 0, reframe: "auto" }, + { id: "c2", assetId: "asset-a", inMs: 5000, outMs: 7000, atMs: 2000, reframe: "auto" }, + ], + }, + metadata: {}, + ...overrides, + }); +} + +export function testContext(overrides: Partial = {}): EdlContext { + return { + profile: testProfile(), + brandKit: testKit(), + assets: new Map([ + ["asset-a", { kind: "video", durationMs: 10_000, width: 1920, height: 1080 }], + ["asset-b", { kind: "video", durationMs: 8000, width: 1920, height: 1080 }], + ["photo-1", { kind: "photo", durationMs: null, width: 4000, height: 3000 }], + ]), + artifactIds: new Set(["music-1", "music-2", "vo-1"]), + music: new Map([ + ["music-1", { commercialUse: true }], + ["music-2", { commercialUse: false }], + ]), + markIds: new Set(["mark-1"]), + personIds: new Set(["person-1"]), + faceTrackIds: new Set(["track-1"]), + templateIds: new Set(["intro-minimal", "outro-simple", "lower-third-a"]), + paidOnly: [], + ...overrides, + }; +}